From d3360cd2ef53e4d83f96d751190df6ce2795eaa7 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 21:46:58 -0800 Subject: [PATCH 01/24] feat(repo): probe events for commands that ask the host for its facts A command that runs nproc, lscpu, nvidia-smi, pip list, printenv, a python -c one-liner calling os.cpu_count(), or a PowerShell Get-CimInstance Win32_Processor gets a probe event after its command event, one per pattern it matches. Patterns match the program the shell runs, never argument text, so grep nproc README is no probe. The defaults ship in repos/probes.py. A manifest's probes key and repo run --probes FILE replace a target's list; [] turns it off. Existing event kinds are unchanged. Signed-off-by: Nikolay Petrov --- .../20261114_120000_napetrov_probe-events.md | 14 + dokimi/docs/reference/cli-interface.json | 1 + dokimi/docs/repo-evaluation-contract.md | 23 ++ dokimi/dokimi/commands/repo_run.py | 7 +- dokimi/dokimi/repos/events.py | 4 + dokimi/dokimi/repos/manifest.py | 14 + dokimi/dokimi/repos/probes.py | 249 +++++++++++++++++ dokimi/dokimi/repos/runner.py | 6 +- dokimi/dokimi/repos/transcript.py | 4 +- dokimi/dokimi/schemas/repo_manifest.v1.json | 6 + dokimi/tests/test_repo_probes.py | 255 ++++++++++++++++++ 11 files changed, 579 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120000_napetrov_probe-events.md create mode 100644 dokimi/dokimi/repos/probes.py create mode 100644 dokimi/tests/test_repo_probes.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md new file mode 100644 index 000000000..1903e829a --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md @@ -0,0 +1,14 @@ + + +### Added + +- **`repo run`: a `probe` event for a command that asks the host for its facts.** A command + such as `nproc`, `lscpu`, `nvidia-smi`, `pip list`, `printenv` or a `python -c` one-liner that + calls `os.cpu_count()` gets a `probe` event after its `command` event, with its `target` + (`cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`) and the `pattern` it matched. The + patterns match the program the shell runs, not argument text (`grep nproc README` is no probe). + The defaults ship in `dokimi/repos/probes.py`; a manifest's `probes` key or `repo run --probes + FILE` replaces a target's list, and `[]` turns it off. Existing event kinds are unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..d6257c963 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1394,6 +1394,7 @@ {"key": "--max-turns", "action": "_StoreAction", "choices": null, "const": null, "default": 200, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--model", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-runs", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--probes", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c1c0de52..ec570a8b8 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -244,6 +244,9 @@ build_systems: [bazel, make] # selects L0 claim-check adapters process_metrics: # optional repository counters, result.json `process.counters` (§4.1) bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any + +probes: # optional: target -> patterns of `probe` events (§4.1) + cpu: [nproc, "py:os.cpu_count"] # a listed target replaces its defaults; [] turns it off ``` Rules: @@ -527,6 +530,26 @@ reads the upstream base (`base_sha`) and the workspace start commit - A `read` event may have no `path`: a refused Claude Code Bash read of a relative path after an earlier call moved the shell, whose file is not known (`{"kind": "read", "denied": true}`). +- A `probe` event follows the `command` event of a command that asks the host + for its facts, one per pattern it matches (a command may give several): + `{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}`. + `target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`; + `pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the + command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`, + `/proc/cpuinfo`, `nproc`, `sysctl -a | grep machdep.cpu`, `Get-CimInstance Win32_Processor`, + `py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`, + `py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`; + `memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`, + `conda list`; `env` `env`, `printenv`. A pattern is matched on the program the shell or + PowerShell runs, with the words it names in any order and case (`grep nproc README` runs + `grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a + path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A + `$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched: + a Python heredoc, a script the agent runs from a file, and a pattern with a `|` that is + not a pipeline of the same commands. + A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` + (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, + and the `process_metrics` counters do not count probes. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 3f9368856..a6621b9cb 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -319,6 +319,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.bases import check_mcp_configs from dokimi.repos.container import docker_network from dokimi.repos.harness_profile import load_profile + from dokimi.repos.probes import load_probe_file from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) from dokimi.repos.selfcheck import ANCESTOR_NAMES, isolation_probe @@ -331,6 +332,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock try: parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) + probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -344,7 +346,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, - max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] + max_turns=args.max_turns, budget_usd=args.budget, probes=probes) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -547,6 +549,9 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str help="Seconds per agent run unless task.json sets timeout_s (default 2700)") run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns", help="Unless task.json sets max_turns (default 200)") + run_p.add_argument("--probes", default=None, metavar="FILE", + help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the " + "framework's list; contract §4.1)") run_p.add_argument("--budget", type=_usd, default=2.0, help="USD cap per agent run unless task.json sets budget_usd (default 2.0)") run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 956171d08..05ec07ae0 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -10,6 +10,10 @@ {"t": 5.0, "kind": "edit", "path": "src/foo.cpp"} {"t": 20.2, "kind": "tool", "name": "WebFetch", "denied": true} {"t": 2.0, "kind": "tool", "name": "Skill", "skill": "lint-fix"} + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per +pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). ``t`` is seconds since the first timestamped trace line, or ``null`` when the format has no timestamps. Paths inside the workspace are made relative to it. diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 9dc481491..884bdd504 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -35,6 +35,7 @@ from .graders.hidden_tests import HIDDEN_DIR, hidden_files from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter +from .probes import validate_config from .source import RepoTree, SourceError from .useful_spec import UsefulSpecError, parse_useful @@ -137,6 +138,7 @@ class RepoManifest: explore_tasks: str | None = None build_systems: tuple[str, ...] = () process_metrics: dict[str, Counter] = field(default_factory=dict) + probes: dict[str, tuple[str, ...]] = field(default_factory=dict) # target -> patterns (repos/probes.py) @property def runnable(self) -> bool: @@ -175,6 +177,7 @@ def to_dict(self) -> dict[str, Any]: "build_systems": list(self.build_systems), "process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field} for name, c in self.process_metrics.items()}, + **({"probes": {t: list(p) for t, p in self.probes.items()}} if self.probes else {}), } @@ -291,6 +294,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: if duplicates: errors.append(f"image/caches: duplicate cache name(s) {duplicates}") counters = _parse_counters(raw.get("process_metrics") or {}, errors) + probes = _parse_probes(raw.get("probes"), errors) if errors: raise ManifestError(path, errors) @@ -312,9 +316,19 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None, build_systems=tuple(raw.get("build_systems") or ()), process_metrics=counters, + probes=probes, ) +def _parse_probes(raw: Any, errors: list[str]) -> dict[str, tuple[str, ...]]: + """The manifest's ``probes`` (repos/probes.py); a bad one is appended to ``errors``.""" + try: + return validate_config(raw, "probes") + except ValueError as exc: + errors.append(str(exc)) + return {} + + def _test_entry_problems(key: str, found: set[str]) -> list[str]: """``{tests}`` belongs to ``entry_points.test`` alone, which must name it and ``{out}``, its report.""" if key != _TEST_ENTRY_POINT: diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py new file mode 100644 index 000000000..d872761e4 --- /dev/null +++ b/dokimi/dokimi/repos/probes.py @@ -0,0 +1,249 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events: a command that asks the host for its facts gets a ``probe`` event beside its ``command`` event. + + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +One event per pattern the command matches, right after the command event, so one command may give both. A +pattern is matched on the commands the shell runs (the words of each simple command, its pipeline stages, the +code of ``python -c`` and of ``$( )`` substitutions), never on the text of an argument: ``grep nproc README`` +runs ``grep``, not ``nproc``. The pattern syntax, in a manifest's ``probes`` or a run's ``--probes`` file: + +- ``nproc``, ``pip list``, ``Get-CimInstance Win32_Processor``: a program, with the words that follow it (in + any order, any case); ``sysctl -a | grep machdep.cpu``: consecutive pipeline stages; +- ``/proc/cpuinfo``: a file a reading command names (``cat``, ``grep``, ``head``, ...); +- ``py:os.cpu_count``: a call of that name in the code of a ``python -c`` one-liner. + +A target listed in a manifest replaces that target's defaults, and a run's ``--probes`` replaces them again; +an empty list turns the target off. See ``docs/repo-evaluation-contract.md`` §2 and §4.1. +""" + +from __future__ import annotations + +import posixpath +import re +import shlex +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from .shell_split import split_script + +TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") + +DEFAULT_PROBES: dict[str, tuple[str, ...]] = { + "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl -a | grep machdep.cpu", "Get-CimInstance Win32_Processor", + "py:os.cpu_count", "py:platform.processor"), + "gpu": ("nvidia-smi", "rocm-smi", "py:torch.cuda.is_available"), + "os": ("uname", "/etc/os-release", "sw_vers", "systeminfo"), + "memory": ("free", "/proc/meminfo"), + "packages": ("pip list", "pip show", "pip freeze", "conda list"), + "env": ("env", "printenv"), + "network": (), +} + +_READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", + "strings", "wc", "rg"}) +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_ASSIGN = re.compile(r"[A-Za-z_]\w*=") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_SUBSTITUTION = re.compile(r"\$\(([^()]*)\)|`([^`]*)`") +_SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) +_SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script +_LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command +_SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command + +Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words + + +@dataclass(frozen=True) +class Probe: + """One compiled pattern. ``kind``: ``command`` (``stages``, consecutive pipeline stages), ``path`` (``text``, the + file's path) or ``python`` (``call``, the regex the code of a ``-c`` one-liner must match).""" + + target: str + text: str + kind: str + stages: tuple[Stage, ...] = () + call: re.Pattern[str] | None = None + + +def validate_config(raw: Any, where: str) -> dict[str, tuple[str, ...]]: + """A ``probes`` mapping (a manifest's or a run file's) as target -> patterns; ``ValueError`` names what is wrong.""" + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of target to patterns, got {type(raw).__name__}") + config: dict[str, tuple[str, ...]] = {} + for target, patterns in raw.items(): + if target not in TARGETS: + raise ValueError(f"{where}: unknown target {target!r} (known: {', '.join(TARGETS)})") + if not isinstance(patterns, list) or not all(isinstance(p, str) and p.strip() for p in patterns): + raise ValueError(f"{where}: {target} must be a list of non-empty pattern strings") + for pattern in patterns: + _compile(target, pattern.strip()) # a pattern that cannot be read is refused here, not per cell + config[target] = tuple(p.strip() for p in patterns) + return config + + +def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: + """A run's ``--probes`` file (YAML, or JSON, which is YAML) through :func:`validate_config`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"--probes {path}: {exc}") from exc + return validate_config(raw, f"--probes {path}") + + +def probes_for(*layers: Mapping[str, Sequence[str]] | None) -> tuple[Probe, ...]: + """The compiled patterns of the defaults with each layer applied in turn: a target a layer names is replaced.""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return tuple(_compile(target, text.strip()) for target in TARGETS for text in config.get(target, ())) + + +def with_probes(events: list[dict[str, Any]], probes: Sequence[Probe]) -> list[dict[str, Any]]: + """``events`` with a ``probe`` event after each ``command`` event whose command matches a pattern.""" + if not probes: + return events + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if event.get("kind") == "command": + out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) + return out + + +def _probe_event(command: dict[str, Any], probe: Probe) -> dict[str, Any]: + event: dict[str, Any] = {"t": command.get("t"), "kind": "probe", "target": probe.target, "pattern": probe.text, + "text": command.get("text", ""), "rc": command.get("rc")} + if "agent" in command: + event["agent"] = command["agent"] + return event + + +def _compile(target: str, text: str) -> Probe: + if target not in TARGETS: + raise ValueError(f"unknown probe target {target!r}") + if text.startswith("py:"): + name = text[3:].strip() + if not _DOTTED.fullmatch(name): + raise ValueError(f"{text!r}: py: takes a dotted Python name such as os.cpu_count") + return Probe(target, text, "python", call=re.compile(r"(? Stage: + if any(c in part for c in ";&<>"): + raise ValueError(f"{text!r}: separate commands and redirections are not part of a pattern; use '|' for stages") + try: + words = [w for w in shlex.split(part) if w not in _LEX_SKIP] + except ValueError as exc: + raise ValueError(f"{text!r}: {exc}") from exc + if not words: + raise ValueError(f"{text!r}: a pattern names a program") + return _program(words[0]), tuple(words[1:]) + + +def _program(word: str) -> str: + """A program's name as the shell or PowerShell finds it: no directory, no ``.exe``, no case.""" + name = posixpath.basename(word.replace("\\", "/")).casefold() + return name[:-4] if name.endswith(".exe") else name + + +def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: + """The probes the command runs, each once, in the order they are configured.""" + pipelines = [[list(_views(words)) for words in pipe] for pipe in _pipelines(command)] + return [p for p in dict.fromkeys(probes) if any(_hits(p, pipe) for pipe in pipelines)] + + +def _pipelines(script: str) -> Iterator[list[list[str]]]: + """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" + for part, _after, _in_subshell, _outer in split_script(script): + for inner in _SUBSTITUTION.finditer(part): + yield from _pipelines(inner.group(1) if inner.group(1) is not None else inner.group(2)) + stages = _stages(_SUBSTITUTION.sub(" ", part)) # the substitutions ran as their own commands, above + if stages: + yield stages + for stage in stages: # ``bash -lc 'nproc'``: the script a shell runs is a script of its own + if len(stage) > 2 and _program(stage[0]) in _SHELLS and _SHELL_C.fullmatch(stage[1]): + yield from _pipelines(stage[2]) + + +def _stages(part: str) -> list[list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped).""" + try: + lexer = shlex.shlex(part, posix=True, punctuation_chars=True) + lexer.whitespace_split = True + tokens = list(lexer) + except ValueError: # an unbalanced quote: the words as they are typed + tokens = part.split() + stages: list[list[str]] = [[]] + for token in tokens: + if token == "|": + stages.append([]) + elif token not in _SEPARATORS and token not in _LEX_SKIP: + stages[-1].append(token) + return [stage for stage in stages if stage] + + +def _views(words: list[str]) -> Iterator[Stage]: + """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" + i = 0 + while i < len(words): + while i < len(words) and _ASSIGN.match(words[i]): # NAME=value words set variables for the command after them + i += 1 + if i >= len(words): + return + yield _head(words[i], words[i + 1:]) + if _program(words[i]) not in _WRAPPERS: + return + i += 1 + + +def _head(word: str, args: list[str]) -> Stage: + """A program and its words; ``python -m pip list`` is the program ``pip``.""" + program = _program(word) + if _PYTHON.fullmatch(program) and args[:1] == ["-m"] and len(args) > 1 and _PIP.fullmatch(args[1]): + return "pip", tuple(args[2:]) + return ("pip" if _PIP.fullmatch(program) else program), tuple(args) + + +def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: + if probe.kind == "command": + return _in_order(probe.stages, pipeline) + if probe.kind == "path": + return any(prog in _READERS and probe.text in args for views in pipeline for prog, args in views) + return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) + + +def _calls(args: tuple[str, ...], probe: Probe) -> bool: + """Whether the code of a ``python -c`` one-liner calls the probe's name.""" + if "-c" not in args or probe.call is None: + return False + code = args[args.index("-c") + 1:args.index("-c") + 2] + return bool(code) and probe.call.search(code[0]) is not None + + +def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: + """Whether the pattern's stages are consecutive stages of the pipeline (one stage: any stage of it).""" + n = len(want) + return any(all(_stage_runs(want[j], pipeline[i + j]) for j in range(n)) for i in range(len(pipeline) - n + 1)) + + +def _stage_runs(want: Stage, views: list[Stage]) -> bool: + """Whether one of the views is the wanted program with every wanted word among its words, any case.""" + program, words = want + return any(prog == program and all(w.casefold() in {a.casefold() for a in args} for w in words) + for prog, args in views) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a5809f29a..95f11623a 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -70,6 +70,7 @@ ) from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert +from .probes import probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry @@ -98,6 +99,7 @@ class AgentOptions: timeout_s: float = 2700 max_turns: int = 200 budget_usd: float = 2.0 + probes: dict[str, tuple[str, ...]] | None = None # the run's --probes, over the manifest's (repos/probes.py) def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: @@ -259,7 +261,9 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) - (rd / "events.jsonl").write_text(transcript.events_jsonl(), encoding="utf-8") + # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events + probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + (rd / "events.jsonl").write_text(transcript.events_jsonl(probed), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, messages=transcript.agent_messages) diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 7d9408625..bd56e981e 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -59,8 +59,8 @@ def reads(self) -> set[str]: """The paths the agent was shown (:func:`shown_read`); a refused or failed read stays in the events only.""" return {e["path"] for e in self.events if shown_read(e) and e.get("path")} - def events_jsonl(self) -> str: - return "".join(json.dumps(e, sort_keys=True) + "\n" for e in self.events) + def events_jsonl(self, events: list[dict[str, Any]] | None = None) -> str: + return "".join(json.dumps(e, sort_keys=True) + "\n" for e in (self.events if events is None else events)) def shown_read(event: dict[str, Any]) -> bool: diff --git a/dokimi/dokimi/schemas/repo_manifest.v1.json b/dokimi/dokimi/schemas/repo_manifest.v1.json index be85eb5ae..11608ffb0 100644 --- a/dokimi/dokimi/schemas/repo_manifest.v1.json +++ b/dokimi/dokimi/schemas/repo_manifest.v1.json @@ -76,6 +76,12 @@ "uniqueItems": true, "items": {"enum": ["bazel", "make", "cmake", "pytest", "pre-commit"]} }, + "probes": { + "description": "Target -> command, file or python patterns whose commands get a probe event (docs/repo-evaluation-contract.md §2, §4.1). A listed target replaces its defaults; [] turns it off.", + "type": "object", + "propertyNames": {"enum": ["cpu", "gpu", "os", "memory", "packages", "env", "network"]}, + "additionalProperties": {"type": "array", "items": {"type": "string", "minLength": 1}} + }, "process_metrics": { "type": "object", "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py new file mode 100644 index 000000000..3c9b96a7c --- /dev/null +++ b/dokimi/tests/test_repo_probes.py @@ -0,0 +1,255 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events (repos/probes.py, contract §4.1): a command that asks the host for its facts gets a ``probe`` event.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.manifest import ManifestError, parse_manifest +from dokimi.repos.probes import DEFAULT_PROBES, load_probe_file, probes_for, validate_config, with_probes + +WS = "/ws" + + +def _claude_call(tool_id: str, command: str, t: str, name: str = "Bash", agent: str | None = None) -> dict: + rec = {"type": "assistant", "timestamp": t, "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": command}}]}} + return {**rec, "parent_tool_use_id": agent} if agent else rec + + +def _claude_result(tool_id: str, text: str = "ok", t: str = "2026-01-01T00:00:06Z") -> dict: + return {"type": "user", "timestamp": t, "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": False, "content": text}]}} + + +def _claude(*lines: dict) -> list[dict]: + """The command and probe events of a Claude Code stream whose first line starts the clock.""" + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *lines)) + return with_probes(convert("claude-stream-json", raw, WS).events, probes_for()) + + +def _commands(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] in ("command", "probe")] + + +def _probe(target: str, pattern: str, text: str, t: float | None, rc: int | None) -> dict: + return {"t": t, "kind": "probe", "target": target, "pattern": pattern, "text": text, "rc": rc} + + +def test_nproc_then_a_threaded_solver_gives_the_probe_after_its_command(): + events = _claude( + _claude_call("a", "nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "8", "2026-01-01T00:00:05Z"), + _claude_call("b", "python solve.py --threads 8", "2026-01-01T00:00:09Z"), + _claude_result("b", "done", "2026-01-01T00:00:20Z"), + ) + assert events == [ + {"t": 5.0, "kind": "command", "text": "nproc", "rc": 0, "output": "8", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", 5.0, 0), + {"t": 9.0, "kind": "command", "text": "python solve.py --threads 8", "rc": 0, "output": "done", + "output_truncated": False}, + ] + + +def test_a_grep_of_the_word_is_no_probe(): + events = _claude(_claude_call("a", "grep nproc README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_a_probe_is_matched_on_the_program_not_on_an_argument_of_another_one(): + events = _claude(_claude_call("a", "pip install nproc && echo lscpu", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_an_agent_that_never_probes_gives_no_probe_event(): + events = _claude(_claude_call("a", "ls -la", "2026-01-01T00:00:05Z"), _claude_result("a"), + _claude_call("b", "git status", "2026-01-01T00:00:06Z"), _claude_result("b")) + assert all(e["kind"] != "probe" for e in events) + + +def test_one_command_gives_the_command_and_every_probe_it_runs(): + events = _claude(_claude_call("a", "grep -c processor /proc/cpuinfo | cat && nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "4")) + assert _commands(events)[1:] == [ + _probe("cpu", "/proc/cpuinfo", events[0]["text"], 5.0, 0), + _probe("cpu", "nproc", events[0]["text"], 5.0, 0), + ] + assert events[0]["kind"] == "command" + + +def test_a_probe_in_a_substitution_is_run_by_the_command(): + events = _claude(_claude_call("a", "make -j$(nproc)", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert _commands(events)[1] == _probe("cpu", "nproc", "make -j$(nproc)", 5.0, 0) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("lscpu", "lscpu"), + ("time nproc", "nproc"), + ("uname -a", "uname"), + ("free -h", "free"), + ("nvidia-smi --query-gpu=name --format=csv", "nvidia-smi"), + ("/usr/bin/nvidia-smi", "nvidia-smi"), + ("head -n 3 /etc/os-release", "/etc/os-release"), + ("cat \"/proc/meminfo\"", "/proc/meminfo"), + ("pip list --format=json", "pip list"), + ("python -m pip freeze", "pip freeze"), + ("pip3 show numpy", "pip show"), + ("conda list -n base", "conda list"), + ("printenv PATH", "printenv"), + ("env", "env"), + ("sysctl -a | grep machdep.cpu", "sysctl -a | grep machdep.cpu"), +]) +def test_default_command_patterns_match_the_program_that_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize(("command", "target"), [ + ('python -c "import os; print(os.cpu_count())"', "cpu"), + ("python3 -c 'import torch; print(torch.cuda.is_available())'", "gpu"), + ('python -c "import platform; print(platform.processor())"', "cpu"), +]) +def test_a_python_one_liner_that_calls_a_probe_is_a_probe(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +@pytest.mark.parametrize("command", [ + 'echo "os.cpu_count()"', + "python -c 'print(1)' # os.cpu_count()", + 'python -c "import os; os.makedirs(\'x\')"', + "cat README | grep 'nproc'", + "git diff | grep nproc", +]) +def test_text_that_is_not_run_code_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): + events = _claude( + _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", + "2026-01-01T00:00:05Z", name="PowerShell"), + _claude_result("a", "8"), + _claude_call("b", "(get-ciminstance win32_processor).NumberOfCores", "2026-01-01T00:00:07Z", + name="PowerShell"), + _claude_result("b", "8", "2026-01-01T00:00:07Z"), + _claude_call("c", "Get-CimInstance Win32_ComputerSystem", "2026-01-01T00:00:08Z", name="PowerShell"), + _claude_result("c", "", "2026-01-01T00:00:08Z"), + ) + assert [(e["kind"], e.get("target")) for e in events] == [ + ("command", None), ("probe", "cpu"), ("command", None), ("probe", "cpu"), ("command", None)] + + +def test_a_subagents_probe_carries_its_agent(): + events = _claude(_claude_call("a", "nproc", "2026-01-01T00:00:05Z", agent="task-1"), _claude_result("a")) + assert events[1] == {**_probe("cpu", "nproc", "nproc", 5.0, 0), "agent": "task-1"} + + +def _codex(*items: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in items) + return with_probes(convert("codex-json", raw, WS).events, probes_for()) + + +def test_codex_unwraps_the_shell_before_it_matches_a_probe(): + events = _codex( + {"type": "item.completed", "item": {"type": "command_execution", "command": "/usr/bin/bash -lc 'nproc'", + "aggregated_output": "4\n", "exit_code": 0}}, + {"type": "item.completed", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'nvidia-smi'", + "aggregated_output": "", "exit_code": 127}}, + ) + assert events == [ + {"t": None, "kind": "command", "text": "nproc", "rc": 0, "output": "4\n", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", None, 0), + {"t": None, "kind": "command", "text": "nvidia-smi", "rc": 127}, + _probe("gpu", "nvidia-smi", "nvidia-smi", None, 127), + ] + + +def _copilot(*records: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in records) + return with_probes(convert("copilot-json", raw, WS).events, probes_for()) + + +def test_copilot_bash_and_powershell_probes_give_probe_events(): + events = _copilot( + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:01Z", + "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "nvidia-smi"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "a", "success": True, + "shellExecution": {"exitCode": 0}, + "result": {"content": "GPU 0"}}}, + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:02Z", + "data": {"toolCallId": "b", "toolName": "powershell", + "arguments": {"command": "Get-CimInstance Win32_Processor"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "b", "success": True, + "shellExecution": {"exitCode": 0}, "result": {"content": "x"}}}, + ) + assert [(e["kind"], e.get("target"), e.get("pattern")) for e in events] == [ + ("command", None, None), ("probe", "gpu", "nvidia-smi"), + ("command", None, None), ("probe", "cpu", "Get-CimInstance Win32_Processor"), + ] + + +def test_a_manifest_replaces_a_target_and_an_empty_list_turns_it_off(): + probes = probes_for({"cpu": ["lscpu"], "env": []}) + assert [p.text for p in probes if p.target == "cpu"] == ["lscpu"] + assert all(p.target != "env" for p in probes) + assert {p.text for p in probes if p.target == "gpu"} == set(DEFAULT_PROBES["gpu"]) + + +def test_a_run_layer_replaces_the_manifest_layer(): + manifest = {"cpu": ["lscpu"]} + run = {"cpu": ["nproc"]} + assert [p.text for p in probes_for(manifest, run) if p.target == "cpu"] == ["nproc"] + + +def test_no_probes_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + assert with_probes(events, ()) == events + + +def test_probe_events_go_to_a_copy_so_the_trace_events_the_counters_read_are_unchanged(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + out = with_probes(events, probes_for()) + assert [e["kind"] for e in out] == ["command", "probe"] + assert events == [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"gpus": ["nvidia-smi"]}, "unknown target 'gpus'"), + ({"cpu": "nproc"}, "cpu must be a list"), + ({"cpu": [" "]}, "cpu must be a list"), + ({"cpu": ["py:os.cpu_count()"]}, "dotted Python name"), + ({"cpu": ['nproc "']}, "No closing quotation"), + ({"cpu": ["|"]}, "names a program"), + ({"cpu": ["nproc;lscpu"]}, "separate commands"), + ({"cpu": ["/proc/cpu info"]}, "one word with no spaces"), + (["cpu"], "expected a mapping"), +]) +def test_a_bad_pattern_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_config(raw, "probes") + + +def test_a_run_probe_file_is_yaml_or_json_and_checked(tmp_path): + good = tmp_path / "probes.yaml" + good.write_text("cpu: [lscpu]\nnetwork: [curl]\n", encoding="utf-8") + assert load_probe_file(str(good)) == {"cpu": ("lscpu",), "network": ("curl",)} + bad = tmp_path / "bad.json" + bad.write_text('{"cpu": ["nproc"', encoding="utf-8") + with pytest.raises(ValueError, match="--probes .*bad.json"): + load_probe_file(str(bad)) + + +def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): + base = {"schema": "repo_manifest.v1", "id": "x", "guidance": {"globs": ["AGENTS.md"]}} + assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} + with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): + parse_manifest({**base, "probes": {"disk": ["df"]}}) From 6617895ec94df7785e315ea98d6d1c16cb6b42bf Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 22:27:59 -0800 Subject: [PATCH 02/24] feat(repo): choice events for a named candidate's install, import, use or removal Signed-off-by: Nikolay Petrov --- .../20261114_120100_napetrov_choice-events.md | 13 ++ dokimi/docs/reference/cli-interface.json | 1 + dokimi/docs/repo-evaluation-contract.md | 8 + dokimi/dokimi/commands/repo_run.py | 8 +- dokimi/dokimi/repos/choices.py | 196 ++++++++++++++++++ dokimi/dokimi/repos/events.py | 14 +- dokimi/dokimi/repos/runner.py | 5 +- dokimi/dokimi/repos/transcript.py | 1 + dokimi/tests/data/cli_flags_allowlist.txt | 1 + dokimi/tests/test_repo_choices.py | 158 ++++++++++++++ 10 files changed, 402 insertions(+), 3 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120100_napetrov_choice-events.md create mode 100644 dokimi/dokimi/repos/choices.py create mode 100644 dokimi/tests/test_repo_choices.py diff --git a/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md new file mode 100644 index 000000000..0f2f3bd2a --- /dev/null +++ b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md @@ -0,0 +1,13 @@ + + +### Added + +- **`repo run --candidates FILE`: a `choice` event for an install, import, use or removal of a candidate.** A file + maps each candidate id to package names (`install`, `remove`), modules an added `import` line names (`import`) and + dotted names an added line mentions (`use`). A `choice` event follows the `command` or `edit` event that does it, with + its `candidate`, `act` and, for an edit, its `path`. Only added text counts: an edit's new side, a write's content. + Copilot and Codex edits carry no added text, so they give install and remove events only. Existing event kinds are + unchanged. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index d6257c963..8ba74e20f 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1380,6 +1380,7 @@ {"key": "--cache", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--cache-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--cache-root", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--candidates", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--delegation", "action": "_StoreAction", "choices": ["'native'", "'none'"], "const": null, "default": "native", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--docker", "action": "_StoreAction", "choices": null, "const": null, "default": "docker", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--force", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index ec570a8b8..16446039c 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -550,6 +550,14 @@ reads the upstream base (`base_sha`) and the workspace start commit A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, and the `process_metrics` counters do not count probes. +- A `choice` event follows the `command` or `edit` event of a call that installs, imports, uses or removes a + candidate the run names: `{"t": 5.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": null}` + (`repo run --candidates FILE`, YAML or JSON, `dokimi/repos/choices.py`). The file maps a candidate id to + `install` and `remove` (package names a package manager is given; a removal matches both lists), `import` + (modules an added `import` or `from` line names) and `use` (dotted names an added line mentions, textually: a comment counts too). Only added text + counts: the new side of an `Edit`, the content of a `Write`, never what the edit replaced. A harness whose edit + events carry no added text (Copilot's `apply_patch`, Codex's `file_change`) gives install and remove events only. + Choice events are written to `events.jsonl`; the counters and `first_useful` do not read them. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index a6621b9cb..4bcbbad9b 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -319,6 +319,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.bases import check_mcp_configs from dokimi.repos.container import docker_network from dokimi.repos.harness_profile import load_profile + from dokimi.repos.choices import load_candidate_file from dokimi.repos.probes import load_probe_file from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) @@ -333,6 +334,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start + candidates = load_candidate_file(args.candidates) if args.candidates else None profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -346,7 +348,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, - max_turns=args.max_turns, budget_usd=args.budget, probes=probes) for m in args.model] + max_turns=args.max_turns, budget_usd=args.budget, probes=probes, + candidates=candidates) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -552,6 +555,9 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str run_p.add_argument("--probes", default=None, metavar="FILE", help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the " "framework's list; contract §4.1)") + run_p.add_argument("--candidates", default=None, metavar="FILE", + help="YAML or JSON: candidate id -> install/import/use/remove names; adds choice events " + "(contract §4.1)") run_p.add_argument("--budget", type=_usd, default=2.0, help="USD cap per agent run unless task.json sets budget_usd (default 2.0)") run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py new file mode 100644 index 000000000..59140a017 --- /dev/null +++ b/dokimi/dokimi/repos/choices.py @@ -0,0 +1,196 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Choice events: an agent installs, imports, uses or removes a candidate the caller names (contract §4.1). + + {"t": 4.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": null} + +A candidate is ``{id: {act: [names]}}``. ``install`` and ``remove`` name packages a package manager is given (``pip +install``, ``uv pip uninstall``, ``conda remove``); a removal matches the names of both lists. ``import`` names modules +an added ``import``/``from`` line names; +``use`` names dotted names an added line mentions (a comment too: the match is textual). Only added text counts: the new side of an edit, never what it +replaced. The events go to ``events.jsonl`` beside the trace's own events; the counters do not read them. +""" + +from __future__ import annotations + +import posixpath +import re +import shlex +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import yaml + +from .shell_split import split_script + +ACTS = ("install", "import", "use", "remove") +_PACKAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_INSTALL_VERBS = frozenset({"install", "add"}) +_REMOVE_VERBS = frozenset({"uninstall", "remove"}) +_PYTHON = re.compile(r"python\d*(?:\.\d+)?") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_REQUIREMENT_END = re.compile(r"[<>=!~\[;@]") # where a requirement's name ends: a version, extras, a URL + + +def validate_candidates(raw: Any, where: str = "--candidates") -> dict[str, dict[str, tuple[str, ...]]]: + """A ``{id: {act: [names]}}`` mapping as ``{id: {act: (names,)}}``; ``ValueError`` names what is wrong.""" + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of candidate to acts, got {type(raw).__name__}") + config: dict[str, dict[str, tuple[str, ...]]] = {} + for cid, acts in raw.items(): + if not isinstance(cid, str) or not cid.strip(): + raise ValueError(f"{where}: a candidate id must be a non-empty string") + if not isinstance(acts, Mapping) or not acts: + raise ValueError(f"{where}: {cid} must map some of {', '.join(ACTS)} to lists of names") + config[cid] = {act: _names(where, cid, act, names) for act, names in acts.items()} + return config + + +def _names(where: str, cid: str, act: str, names: Any) -> tuple[str, ...]: + if act not in ACTS: + raise ValueError(f"{where}: {cid}: unknown act {act!r} (known: {', '.join(ACTS)})") + if not isinstance(names, list) or not all(isinstance(n, str) and n.strip() for n in names): + raise ValueError(f"{where}: {cid}.{act} must be a list of non-empty strings") + shape = _PACKAGE if act in ("install", "remove") else _DOTTED + bad = [n for n in names if not shape.fullmatch(n.strip())] + if bad: + kind = "a package name" if act in ("install", "remove") else "a dotted Python name" + raise ValueError(f"{where}: {cid}.{act} takes {kind}: {bad[0]!r}") + return tuple(n.strip() for n in names) + + +def load_candidate_file(path: str) -> dict[str, dict[str, tuple[str, ...]]]: + """A run's ``--candidates`` file (YAML, or JSON, which is YAML) through :func:`validate_candidates`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"--candidates {path}: {exc}") from exc + return validate_candidates(raw, f"--candidates {path}") + + +def with_choices(events: list[dict[str, Any]], candidates: Mapping[str, Mapping[str, Sequence[str]]], + added: Sequence[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]: + """``events`` with the ``choice`` events each command or edit gives, after it. ``added`` pairs an edit event with + the text it adds (``Transcript.added_text``); an edit without one gives no import or use.""" + if not candidates: + return events + texts = {id(event): text for event, text in added} + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if not _ran(event): + continue + if event.get("kind") == "command": + out.extend(_command_choices(event, candidates)) + elif event.get("kind") == "edit" and id(event) in texts: + out.extend(_text_choices(event, texts[id(event)], candidates)) + return out + + +def _ran(event: dict[str, Any]) -> bool: + """Whether the call took effect: a denied one or one that failed did not install, import, use or remove anything.""" + return not event.get("denied") and event.get("rc") in (None, 0) + + +def choice_timeline(events: Sequence[dict[str, Any]]) -> list[tuple[float | None, str, str]]: + """The ``(t, candidate, act)`` of each choice event, in the order the trace gives them.""" + return [(e.get("t"), str(e["candidate"]), str(e["act"])) for e in events if e.get("kind") == "choice"] + + +def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str, Sequence[str]]]) -> list[dict]: + found: dict[tuple[str, str], None] = {} + for part, *_ in split_script(str(event.get("text", ""))): + call = _package_call(_words(part)) + if call is None: + continue + act, packages = call + for cid, acts in candidates.items(): + if _wanted(acts, act) & set(packages): + found[(cid, act)] = None + return [_choice(event, cid, act, None) for cid, act in found] + + +def _text_choices(event: dict[str, Any], text: str, + candidates: Mapping[str, Mapping[str, Sequence[str]]]) -> list[dict]: + out: list[dict] = [] + for cid, acts in candidates.items(): + if any(_imports(name, text) for name in acts.get("import", ())): + out.append(_choice(event, cid, "import", event.get("path"))) + if any(_calls(name, text) for name in acts.get("use", ())): + out.append(_choice(event, cid, "use", event.get("path"))) + return out + + +def _wanted(acts: Mapping[str, Sequence[str]], act: str) -> set[str]: + """The normalised package names an install or a removal of the candidate names; a removal also takes ``install``.""" + names = [*acts.get("install", ()), *acts.get("remove", ())] if act == "remove" else list(acts.get("install", ())) + return {_package(n) for n in names} + + +def _choice(event: dict[str, Any], cid: str, act: str, path: str | None) -> dict[str, Any]: + out: dict[str, Any] = {"t": event.get("t"), "kind": "choice", "candidate": cid, "act": act, "path": path} + if "agent" in event: # a subagent's choice names its agent, as its probe and command events do + out["agent"] = event["agent"] + return out + + +def _imports(name: str, text: str) -> bool: + """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line.""" + if re.search(rf"^\s*from\s+{re.escape(name)}(?!\w)", text, re.M): + return True + for line in re.finditer(r"^\s*import\s+([^\n#]+)", text, re.M): + for part in line.group(1).split(","): + module = re.split(r"\s+as\s+", part.strip())[0] + if module == name or module.startswith(name + "."): + return True + return False + + +def _calls(name: str, text: str) -> bool: + return re.search(rf"(? list[str]: + try: + return shlex.split(part) + except ValueError: # an unbalanced quote: the words as they are typed + return part.split() + + +def _package_call(words: list[str]) -> tuple[str, list[str]] | None: + """The act and the package names of a package manager's install, add, uninstall or remove call.""" + rest = _manager_rest(words) + while rest and rest[0].startswith("-"): # ``pip --quiet install``: the options before the verb + rest = rest[1:] + if not rest: + return None + verb = rest[0].casefold() + act = "install" if verb in _INSTALL_VERBS else "remove" if verb in _REMOVE_VERBS else None + if act is None: + return None + return act, [_package(w) for w in rest[1:] if not w.startswith("-") and _package(w)] + + +def _manager_rest(words: list[str]) -> list[str]: + """The words after the package manager's program (``pip``, ``python -m pip``, ``uv pip``, ``conda``...).""" + if not words: + return [] + head = posixpath.basename(words[0].replace("\\", "/")).casefold() + if _PYTHON.fullmatch(head) and words[1:3] == ["-m", "pip"]: + return words[3:] + if head == "uv" and words[1:2] == ["pip"]: + return words[2:] + if _PIP.fullmatch(head) or head in ("conda", "mamba", "poetry"): + return words[1:] + return [] + + +def _package(word: str) -> str: + """A requirement's name as the package manager compares it: no version, extras or URL; case and ``-_.`` folded.""" + name = _REQUIREMENT_END.split(word, maxsplit=1)[0] + return re.sub(r"[-_.]+", "-", name).casefold() diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 05ec07ae0..d336bd16f 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -13,7 +13,8 @@ {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per -pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). +pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). A ``choice`` event follows the ``command`` or +``edit`` event of an install, import, use or removal of a named candidate (:mod:`dokimi.repos.choices`). ``t`` is seconds since the first timestamped trace line, or ``null`` when the format has no timestamps. Paths inside the workspace are made relative to it. @@ -239,6 +240,15 @@ def _deny_read(read: dict[str, Any], unknown: bool) -> None: del read["path"] +def _added_text(args: Any) -> str: + """The text an edit tool adds: the new side of each replacement, or the whole file of a write (choices.py).""" + if not isinstance(args, dict): + return "" + if isinstance(args.get("edits"), list): # MultiEdit + return "\n".join(str(e.get("new_string", "")) for e in args["edits"] if isinstance(e, dict)) + return str(args.get("new_string") or args.get("content") or args.get("new_source") or "") + + def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: """The event of one tool_use block; a Bash call's derived reads come before it when the trace ends (:func:`_walk_bash`).""" event = _tool_event(block, t, stream.workdir) @@ -248,6 +258,8 @@ def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: st stream.bash[id(event)] = _BashCall(str(event["text"]), event) if block.get("name") in _AGENT_TOOLS: out.subagent_calls = (out.subagent_calls or 0) + 1 + if block.get("name") in _EDIT_TOOLS: + out.added_text.append((event, _added_text(block.get("input")))) out.events.append(event) if block.get("id"): stream.by_id[block["id"]] = event diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 95f11623a..5857adbc2 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -70,6 +70,7 @@ ) from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert +from .choices import with_choices from .probes import probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful @@ -100,6 +101,7 @@ class AgentOptions: max_turns: int = 200 budget_usd: float = 2.0 probes: dict[str, tuple[str, ...]] | None = None # the run's --probes, over the manifest's (repos/probes.py) + candidates: dict[str, dict[str, tuple[str, ...]]] | None = None # the run's --candidates (repos/choices.py) def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: @@ -263,7 +265,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, transcript = convert(opts.profile.trace, raw, WORKSPACE) # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) - (rd / "events.jsonl").write_text(transcript.events_jsonl(probed), encoding="utf-8") + shown = with_choices(probed, opts.candidates or {}, transcript.added_text) # choice events: choices.py + (rd / "events.jsonl").write_text(transcript.events_jsonl(shown), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, messages=transcript.agent_messages) diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index bd56e981e..f74251d94 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -46,6 +46,7 @@ class Transcript: dropped_lines: int = 0 # trace lines that did not parse or were of no known shape: their events are missing init_model: str | None = None # the model the harness says it starts with (``system``/``init``) answered_by: set[str] = field(default_factory=set) # ``message.model`` of the main thread's replies + added_text: list[tuple[dict[str, Any], str]] = field(default_factory=list) # edit event, the text it adds (choices.py) def model_resolved(self) -> str | list[str] | None: """The model that answered: one name, a sorted list when several did, ``None`` when the trace cannot tell. diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..52c5282b9 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -35,6 +35,7 @@ --cache-root --calls --candidate +--candidates # repo run's candidate file (repos/choices.py): no registry concept yet; a F1b migration names it --catalog --checklist --claude # catalog trigger-probe (#217) adds it: backend choice has no registry concept yet diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py new file mode 100644 index 000000000..144005aa8 --- /dev/null +++ b/dokimi/tests/test_repo_choices.py @@ -0,0 +1,158 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Choice events (repos/choices.py, contract §4.1): an install, import, use or removal of a named candidate.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.choices import choice_timeline, validate_candidates, with_choices +from dokimi.repos.events import convert + +WS = "/ws" +SKLEARN = {"sklearn": {"install": ["scikit-learn-intelex"], "import": ["sklearnex"], "use": ["patch_sklearn"]}} +SCIPY = {"lstsq": {"use": ["scipy.linalg.lstsq"]}} + + +def _call(tool_id: str, name: str, args: dict, t: str) -> dict: + return {"type": "assistant", "timestamp": t, + "message": {"content": [{"type": "tool_use", "id": tool_id, "name": name, "input": args}]}} + + +def _result(tool_id: str, t: str) -> dict: + return {"type": "user", "timestamp": t, + "message": {"content": [{"type": "tool_result", "tool_use_id": tool_id, "is_error": False, + "content": ""}]}} + + +def _bash(tool_id: str, command: str, t: str) -> list[dict]: + return [_call(tool_id, "Bash", {"command": command}, t), _result(tool_id, t)] + + +def _edit(tool_id: str, path: str, new: str, t: str, old: str = "", name: str = "Edit") -> list[dict]: + args = {"file_path": f"{WS}/{path}", "new_string": new, "old_string": old} if name == "Edit" else { + "file_path": f"{WS}/{path}", "content": new} + return [_call(tool_id, name, args, t), _result(tool_id, t)] + + +def _run(candidates: dict, *records: list[dict] | dict) -> list[dict]: + flat = [r for rec in records for r in (rec if isinstance(rec, list) else [rec])] + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *flat)) + out = convert("claude-stream-json", raw, WS) + return with_choices(out.events, candidates, out.added_text) + + +def _choices(events: list[dict]) -> list[tuple]: + return [(e["candidate"], e["act"], e["path"]) for e in events if e["kind"] == "choice"] + + +def test_install_then_import_then_use_give_one_choice_each_in_order(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _edit("b", "app.py", "from sklearnex import patch_sklearn\npatch_sklearn()\n", + "2026-01-01T00:00:09Z")) + assert [(e["kind"], e.get("act")) for e in events] == [ + ("command", None), ("choice", "install"), ("edit", None), ("choice", "import"), ("choice", "use")] + assert events[1] == {"t": 5.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": None} + assert events[3]["path"] == "app.py" and events[4]["path"] == "app.py" + + +def test_install_then_remove_are_two_choice_events(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _bash("b", "pip uninstall scikit-learn-intelex", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("sklearn", "install", None), ("sklearn", "remove", None)] + + +def test_a_use_without_an_install_is_a_use_and_no_install(): + events = _run(SCIPY, _edit("a", "solve.py", "x = scipy.linalg.lstsq(A, b)\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("lstsq", "use", "solve.py")] + + +def test_only_the_added_side_of_an_edit_counts(): + events = _run(SKLEARN, _edit("a", "app.py", "import numpy\n", "2026-01-01T00:00:05Z", + old="from sklearnex import patch_sklearn\n")) + assert _choices(events) == [] + + +def test_a_write_counts_its_whole_content(): + events = _run(SKLEARN, _edit("a", "app.py", "import sklearnex\n", "2026-01-01T00:00:05Z", name="Write")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + +@pytest.mark.parametrize("command", [ + "pip install 'scikit-learn-intelex==2024.7'", + "uv pip install scikit_learn_intelex", + "python3 -m pip install --upgrade scikit-learn-intelex", +]) +def test_a_package_manager_install_names_the_package_whatever_the_spelling(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + "grep 'pip install scikit-learn-intelex' README", + "echo pip install scikit-learn-intelex", + "pip install numpy", +]) +def test_a_command_that_only_mentions_the_package_is_no_choice(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def test_no_candidates_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "pip install scikit-learn-intelex", "rc": 0}] + assert with_choices(events, {}, []) == events + + +def test_the_timeline_lists_each_choice_in_trace_order(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _edit("b", "app.py", "patch_sklearn()\n", "2026-01-01T00:00:09Z")) + assert choice_timeline(events) == [(5.0, "sklearn", "install"), (9.0, "sklearn", "use")] + + +def test_a_name_listed_only_for_removal_is_no_install(): + events = _run({"x": {"remove": ["foo"]}}, _bash("a", "pip install foo", "2026-01-01T00:00:05Z"), + _bash("b", "pip uninstall foo", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("x", "remove", None)] + + +def test_a_subagents_choice_names_its_agent(): + record = {"type": "assistant", "timestamp": "2026-01-01T00:00:05Z", "parent_tool_use_id": "task-1", + "message": {"content": [{"type": "tool_use", "id": "a", "name": "Bash", + "input": {"command": "pip install scikit-learn-intelex"}}]}} + events = _run(SKLEARN, record, _result("a", "2026-01-01T00:00:06Z")) + assert [e for e in events if e["kind"] == "choice"][0]["agent"] == "task-1" + + +def test_a_failed_install_gives_no_choice(): + failed = [_call("a", "Bash", {"command": "pip install scikit-learn-intelex"}, "2026-01-01T00:00:05Z"), + {"type": "user", "timestamp": "2026-01-01T00:00:06Z", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "a", "is_error": True, "content": "no such package"}]}}] + assert _choices(_run(SKLEARN, failed)) == [] + + +def test_a_module_in_a_multi_module_import_line_is_an_import(): + events = _run(SKLEARN, _edit("a", "app.py", "import os, sklearnex as sk\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + +@pytest.mark.parametrize("command", ["pip --quiet install scikit-learn-intelex", "pip3.11 install scikit-learn-intelex"]) +def test_a_pip_option_or_a_versioned_pip_still_names_the_install(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"sk": {"fit": ["x"]}}, "unknown act 'fit'"), + ({"sk": {"import": ["sklearn learn"]}}, "takes a dotted Python name"), + ({"sk": {"install": ["bad name;rm"]}}, "takes a package name"), + ({"sk": {"install": "scikit-learn"}}, "must be a list of non-empty strings"), + ({"sk": []}, "must map some of"), + (["sk"], "expected a mapping"), +]) +def test_a_bad_candidate_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_candidates(raw, "candidates") From 2991eb7b582fb64931be35dae0973e1381da5639 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 23:17:57 -0800 Subject: [PATCH 03/24] fix(repo): count an install choice only where its exit code is its own The harness reports one exit code per command, so an install after `||`, `;` or a pipe may not have run, or may have failed, while the command still exits 0. Such installs now give no choice. A `--dry-run` installs nothing and gives none. An import after `;` on the same line is seen, and `uv add` and `uv remove` name their package like `uv pip` does. Contract section 4.1 says which installs count and which do not. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 4 ++++ dokimi/dokimi/repos/choices.py | 22 ++++++++++++++---- dokimi/tests/test_repo_choices.py | 31 +++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 16446039c..4067d9dad 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -557,6 +557,10 @@ reads the upstream base (`base_sha`) and the workspace start commit (modules an added `import` or `from` line names) and `use` (dotted names an added line mentions, textually: a comment counts too). Only added text counts: the new side of an `Edit`, the content of a `Write`, never what the edit replaced. A harness whose edit events carry no added text (Copilot's `apply_patch`, Codex's `file_change`) gives install and remove events only. + An install or removal gives a choice only when its command joins every part to the next with `&&` and has no pipe: + the harness reports one exit code per command, so a `;`, `||` or `|` part could have failed or never run. A + `--dry-run` installs nothing and gives none. An install in a `bash -c` wrapper, or whose first word carries a + parenthesis (`(pip install X)`), gives none. Choice events are written to `events.jsonl`; the counters and `first_useful` do not read them. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 59140a017..3a0c0a439 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -103,9 +103,16 @@ def choice_timeline(events: Sequence[dict[str, Any]]) -> list[tuple[float | None def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str, Sequence[str]]]) -> list[dict]: + """An install or removal counts only where the command's exit code is its own: every part joined to the next by + ``&&`` (rc 0 then means each ran), and not in a pipeline (the last stage's code). A ``;``, ``||`` or ``&`` part may + have failed or never run, so its install gives no choice.""" + parts = split_script(str(event.get("text", ""))) + if any(part.after not in ("", "&&") for part in parts): + return [] found: dict[tuple[str, str], None] = {} - for part, *_ in split_script(str(event.get("text", ""))): - call = _package_call(_words(part)) + for part, *_ in parts: + words = _words(part) + call = None if "|" in words else _package_call(words) if call is None: continue act, packages = call @@ -140,10 +147,11 @@ def _choice(event: dict[str, Any], cid: str, act: str, path: str | None) -> dict def _imports(name: str, text: str) -> bool: - """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line.""" - if re.search(rf"^\s*from\s+{re.escape(name)}(?!\w)", text, re.M): + """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line; a + statement may follow another on its line after a ``;``.""" + if re.search(rf"(?:^|;)[ \t]*from\s+{re.escape(name)}(?!\w)", text, re.M): return True - for line in re.finditer(r"^\s*import\s+([^\n#]+)", text, re.M): + for line in re.finditer(r"(?:^|;)[ \t]*import\s+([^\n#;]+)", text, re.M): for part in line.group(1).split(","): module = re.split(r"\s+as\s+", part.strip())[0] if module == name or module.startswith(name + "."): @@ -164,6 +172,8 @@ def _words(part: str) -> list[str]: def _package_call(words: list[str]) -> tuple[str, list[str]] | None: """The act and the package names of a package manager's install, add, uninstall or remove call.""" + if "--dry-run" in words: # a dry run installs nothing + return None rest = _manager_rest(words) while rest and rest[0].startswith("-"): # ``pip --quiet install``: the options before the verb rest = rest[1:] @@ -185,6 +195,8 @@ def _manager_rest(words: list[str]) -> list[str]: return words[3:] if head == "uv" and words[1:2] == ["pip"]: return words[2:] + if head == "uv" and words[1:2] in (["add"], ["remove"]): # uv's project verbs: ``uv add`` installs a requirement + return words[1:] if _PIP.fullmatch(head) or head in ("conda", "mamba", "poetry"): return words[1:] return [] diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 144005aa8..2b500f401 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -145,6 +145,37 @@ def test_a_pip_option_or_a_versioned_pip_still_names_the_install(command): assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] +def test_uvs_project_verbs_add_and_remove_name_the_package(): + events = _run(SKLEARN, _bash("a", "uv add scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _bash("b", "uv remove scikit-learn-intelex", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("sklearn", "install", None), ("sklearn", "remove", None)] + + +def test_an_install_in_an_and_chain_names_the_install(): + assert _choices(_run(SKLEARN, _bash("a", "cd /ws && pip install scikit-learn-intelex", + "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + # the exit code is the last part's: the install may have failed, or never run + "python -c 'import sklearnex' || pip install scikit-learn-intelex", + "pip install scikit-learn-intelex; echo done", + "pip install scikit-learn-intelex 2>&1 | tail -3", + "pip install scikit-learn-intelex &", + "pip install --dry-run scikit-learn-intelex", +]) +def test_an_install_whose_exit_code_is_not_its_own_gives_no_choice(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def test_an_import_after_a_semicolon_on_the_same_line_is_an_import(): + events = _run(SKLEARN, _edit("a", "app.py", "import os; from sklearnex import patch_sklearn\n", + "2026-01-01T00:00:05Z")) + assert ("sklearn", "import", "app.py") in _choices(events) + events = _run(SKLEARN, _edit("a", "app.py", "import os; import sklearnex\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + @pytest.mark.parametrize(("raw", "message"), [ ({"sk": {"fit": ["x"]}}, "unknown act 'fit'"), ({"sk": {"import": ["sklearn learn"]}}, "takes a dotted Python name"), From b3bd1d82c5d47b0fdc650a2e44e49e2f33f331f8 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:48:07 -0600 Subject: [PATCH 04/24] fix(repo): read choice and probe commands as the shell runs them Choice events: - `pip install X |cat` and a quoted `'|'` no longer read as a pipeline by whitespace alone; quoting decides. - `--help`, `-h`, `--version` give no install or removal choice. - A value of `--target`, `-r`, `-c`, `--index-url`, `--name` ... is not a package operand. - `sudo`, `time`, `nice` before the manager no longer hide the call. - An import is read from code: a `#` comment is dropped, a `\` continuation is joined, `if x: import m` counts. Probe events: - A heredoc body is data unless a shell reads it. - `$( )` and backticks inside single quotes or escaped are text; nested substitutions are walked. - A quoted `|` is not a stage break. - Wrapper options (`sudo -n`, `nice -n 5`, `time -p`) are skipped before the wrapped program. - A path probe matches a file operand, not a search pattern or an option value. - A `py:NAME` probe matches a call in the parsed `-c` code, not a string or a comment. Tests pin each rule, the denied-call check and the `run_cell` wiring of `--candidates` and `--probes`. The contract names Copilot's edit tools among those that give no import or use choice. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- .../20261114_120100_napetrov_choice-events.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 +- dokimi/dokimi/repos/choices.py | 51 +++-- dokimi/dokimi/repos/probes.py | 190 +++++++++++++++--- dokimi/tests/test_repo_choices.py | 66 ++++++ dokimi/tests/test_repo_probes.py | 65 ++++++ dokimi/tests/test_repo_run_events.py | 43 ++++ 7 files changed, 372 insertions(+), 49 deletions(-) create mode 100644 dokimi/tests/test_repo_run_events.py diff --git a/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md index 0f2f3bd2a..79b4ea225 100644 --- a/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md +++ b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md @@ -9,5 +9,5 @@ SPDX-License-Identifier: Apache-2.0 maps each candidate id to package names (`install`, `remove`), modules an added `import` line names (`import`) and dotted names an added line mentions (`use`). A `choice` event follows the `command` or `edit` event that does it, with its `candidate`, `act` and, for an edit, its `path`. Only added text counts: an edit's new side, a write's content. - Copilot and Codex edits carry no added text, so they give install and remove events only. Existing event kinds are + Copilot's edit tools and Codex's `file_change` carry no added text, so they give install and remove events only. Existing event kinds are unchanged. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 4067d9dad..5e7c68f76 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -556,7 +556,9 @@ reads the upstream base (`base_sha`) and the workspace start commit `install` and `remove` (package names a package manager is given; a removal matches both lists), `import` (modules an added `import` or `from` line names) and `use` (dotted names an added line mentions, textually: a comment counts too). Only added text counts: the new side of an `Edit`, the content of a `Write`, never what the edit replaced. A harness whose edit - events carry no added text (Copilot's `apply_patch`, Codex's `file_change`) gives install and remove events only. + events carry no added text (Copilot's `edit`, `create`, `str_replace_editor`, `write` and `apply_patch`, Codex's + `file_change`) gives install and remove events only: no import or use choice there does not mean the agent did not + choose the candidate. An install or removal gives a choice only when its command joins every part to the next with `&&` and has no pipe: the harness reports one exit code per command, so a `;`, `||` or `|` part could have failed or never run. A `--dry-run` installs nothing and gives none. An install in a `bash -c` wrapper, or whose first word carries a diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 3a0c0a439..4d29e8a41 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -16,13 +16,13 @@ import posixpath import re -import shlex from collections.abc import Mapping, Sequence from pathlib import Path from typing import Any import yaml +from .probes import command_stages, command_words from .shell_split import split_script ACTS = ("install", "import", "use", "remove") @@ -33,12 +33,19 @@ _PYTHON = re.compile(r"python\d*(?:\.\d+)?") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _REQUIREMENT_END = re.compile(r"[<>=!~\[;@]") # where a requirement's name ends: a version, extras, a URL +_INFO_FLAGS = frozenset({"-h", "--help", "-V", "--version"}) # the manager prints text and exits 0, changing nothing +_VALUE_OPTIONS = frozenset({ + "-r", "--requirement", "-c", "--constraint", "-e", "--editable", "-t", "--target", "-i", "--index-url", + "--extra-index-url", "-f", "--find-links", "--prefix", "--root", "--python", "--name", "-n", "-p", "--platform", + "--python-version", "--abi", "--implementation", "--src", "--cache-dir", "--log", "--proxy", "--retries", + "--timeout", "--cert", "--client-cert", "--trusted-host", "--config-settings", "-C", "--channel", "--group", + "--with"}) # options that take the next word, which is not a package def validate_candidates(raw: Any, where: str = "--candidates") -> dict[str, dict[str, tuple[str, ...]]]: """A ``{id: {act: [names]}}`` mapping as ``{id: {act: (names,)}}``; ``ValueError`` names what is wrong.""" if raw is None: - return {} + raise ValueError(f"{where}: the file names no candidate") if not isinstance(raw, Mapping): raise ValueError(f"{where}: expected a mapping of candidate to acts, got {type(raw).__name__}") config: dict[str, dict[str, tuple[str, ...]]] = {} @@ -111,8 +118,13 @@ def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str return [] found: dict[tuple[str, str], None] = {} for part, *_ in parts: - words = _words(part) - call = None if "|" in words else _package_call(words) + stages, _inner = command_stages(part) + if len(stages) > 1: # a pipeline reports its last stage's exit code + continue + words = stages[0] if stages else [] + if words[:1] in (["exit"], ["exec"]): # exit or exec: no later part runs + break + call = next(filter(None, map(_package_call, command_words(words))), None) if call is None: continue act, packages = call @@ -148,10 +160,11 @@ def _choice(event: dict[str, Any], cid: str, act: str, path: str | None) -> dict def _imports(name: str, text: str) -> bool: """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line; a - statement may follow another on its line after a ``;``.""" - if re.search(rf"(?:^|;)[ \t]*from\s+{re.escape(name)}(?!\w)", text, re.M): + statement may follow another on its line after a ``;`` or a ``:``, a ``\\`` continues a line, a comment is no code.""" + text = re.sub(r"\\\n[ \t]*", " ", re.sub(r"#[^\n]*", "", text)) + if re.search(rf"(?:^|[;:])[ \t]*from\s+{re.escape(name)}(?!\w)", text, re.M): return True - for line in re.finditer(r"(?:^|;)[ \t]*import\s+([^\n#;]+)", text, re.M): + for line in re.finditer(r"(?:^|[;:])[ \t]*import\s+([^\n;]+)", text, re.M): for part in line.group(1).split(","): module = re.split(r"\s+as\s+", part.strip())[0] if module == name or module.startswith(name + "."): @@ -163,34 +176,36 @@ def _calls(name: str, text: str) -> bool: return re.search(rf"(? list[str]: - try: - return shlex.split(part) - except ValueError: # an unbalanced quote: the words as they are typed - return part.split() - - def _package_call(words: list[str]) -> tuple[str, list[str]] | None: """The act and the package names of a package manager's install, add, uninstall or remove call.""" - if "--dry-run" in words: # a dry run installs nothing + if "--dry-run" in words or _INFO_FLAGS & set(words): # a dry run installs nothing, nor does a help request return None rest = _manager_rest(words) while rest and rest[0].startswith("-"): # ``pip --quiet install``: the options before the verb - rest = rest[1:] + rest = rest[2:] if rest[0] in _VALUE_OPTIONS else rest[1:] if not rest: return None verb = rest[0].casefold() act = "install" if verb in _INSTALL_VERBS else "remove" if verb in _REMOVE_VERBS else None if act is None: return None - return act, [_package(w) for w in rest[1:] if not w.startswith("-") and _package(w)] + names: list[str] = [] + skip = False + for word in rest[1:]: + if skip: + skip = False + elif word.startswith("-"): + skip = word in _VALUE_OPTIONS + elif _package(word): + names.append(_package(word)) + return act, names def _manager_rest(words: list[str]) -> list[str]: """The words after the package manager's program (``pip``, ``python -m pip``, ``uv pip``, ``conda``...).""" if not words: return [] - head = posixpath.basename(words[0].replace("\\", "/")).casefold() + head = posixpath.basename(words[0].replace("\\", "/")).casefold().removesuffix(".exe") if _PYTHON.fullmatch(head) and words[1:3] == ["-m", "pip"]: return words[3:] if head == "uv" and words[1:2] == ["pip"]: diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index d872761e4..d00354ff3 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -21,6 +21,7 @@ from __future__ import annotations +import ast import posixpath import re import shlex @@ -31,7 +32,7 @@ import yaml -from .shell_split import split_script +from .shell_split import split_script, substitution_end TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") @@ -49,11 +50,22 @@ _READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", "strings", "wc", "rg"}) _WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_WRAPPER_VALUES = {"sudo": frozenset({"-u", "-g", "-h", "-p", "-r", "-t", "-C", "-D", "-R", "-T", "-U"}), + "nice": frozenset({"-n", "--adjustment"}), "exec": frozenset({"-a"}), + "time": frozenset({"-f", "-o", "--format", "--output"})} # options that take the next word +_PATTERN_READERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is a pattern or script +_READER_VALUES = {"head": {"-n", "-c"}, "tail": {"-n", "-c"}, "grep": {"-e", "-f", "-m", "-A", "-B", "-C"}, + "egrep": {"-e", "-f", "-m", "-A", "-B", "-C"}, "fgrep": {"-e", "-f", "-m", "-A", "-B", "-C"}, + "rg": {"-e", "-f", "-m", "-A", "-B", "-C", "-g", "-t", "-T"}, "sed": {"-e", "-f", "-l"}, + "awk": {"-F", "-v", "-f"}} # options that take the next word +_PATTERN_OPTIONS = frozenset({"-e", "--regexp"}) # the pattern or script comes from the option, not the first operand _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") _DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") -_SUBSTITUTION = re.compile(r"\$\(([^()]*)\)|`([^`]*)`") +_HEREDOC = re.compile(r"<<-?[ \t]*(['\"]?)([A-Za-z_]\w*)\1") +_SHELL_LINE = re.compile(r"\s*(?:(?:sudo|exec|command|time)\s+)*(?:bash|sh|zsh|dash)\b") # a heredoc fed to a shell runs +_PUNCT = frozenset("();<>&|") _SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) _SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script _LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command @@ -65,13 +77,13 @@ @dataclass(frozen=True) class Probe: """One compiled pattern. ``kind``: ``command`` (``stages``, consecutive pipeline stages), ``path`` (``text``, the - file's path) or ``python`` (``call``, the regex the code of a ``-c`` one-liner must match).""" + file's path) or ``python`` (``call``, the dotted name a call in the code of a ``-c`` one-liner must have).""" target: str text: str kind: str stages: tuple[Stage, ...] = () - call: re.Pattern[str] | None = None + call: str | None = None def validate_config(raw: Any, where: str) -> dict[str, tuple[str, ...]]: @@ -136,7 +148,7 @@ def _compile(target: str, text: str) -> Probe: name = text[3:].strip() if not _DOTTED.fullmatch(name): raise ValueError(f"{text!r}: py: takes a dotted Python name such as os.cpu_count") - return Probe(target, text, "python", call=re.compile(r"(? list[Probe]: def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" - for part, _after, _in_subshell, _outer in split_script(script): - for inner in _SUBSTITUTION.finditer(part): - yield from _pipelines(inner.group(1) if inner.group(1) is not None else inner.group(2)) - stages = _stages(_SUBSTITUTION.sub(" ", part)) # the substitutions ran as their own commands, above + for part, _after, _in_subshell, _outer in split_script(_without_heredocs(script)): + stages, inner = command_stages(part) + for body in inner: + yield from _pipelines(body) if stages: yield stages for stage in stages: # ``bash -lc 'nproc'``: the script a shell runs is a script of its own @@ -181,35 +193,119 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: yield from _pipelines(stage[2]) -def _stages(part: str) -> list[list[str]]: - """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped).""" - try: - lexer = shlex.shlex(part, posix=True, punctuation_chars=True) - lexer.whitespace_split = True - tokens = list(lexer) - except ValueError: # an unbalanced quote: the words as they are typed - tokens = part.split() +def _without_heredocs(script: str) -> str: + """The script without the bodies of its heredocs, which are data (``cat < run.sh``) unless a shell reads them.""" + if "<<" not in script: + return script + lines, out, end = script.split("\n"), [], None + for line in lines: + if end is not None: + if line.strip() == end: + end = None + continue + out.append(line) + m = _HEREDOC.search(line) + if m and not _SHELL_LINE.match(line): + end = m.group(2) + return "\n".join(out) + + +def command_stages(part: str) -> tuple[list[list[str]], list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped), and + the bodies of the ``$( )`` and backtick substitutions outside single quotes: they ran as commands of their own.""" stages: list[list[str]] = [[]] - for token in tokens: - if token == "|": - stages.append([]) - elif token not in _SEPARATORS and token not in _LEX_SKIP: - stages[-1].append(token) - return [stage for stage in stages if stage] + bodies: list[str] = [] + word: list[str] = [] + started = False # a quoted empty string is a word + quote, i = "", 0 + while i < len(part): + c = part[i] + if c == "'" and not quote: # nothing inside single quotes is special + end = part.find("'", i + 1) + end = len(part) if end < 0 else end + word.append(part[i + 1:end]) + started = True + i = end + elif c == "\\" and part[i + 1:i + 2] in ("$", "`", '"'): + word.append(part[i + 1]) # an escaped ``$(`` is text; any other backslash is a path separator + started = True + i += 1 + elif c in "$`" and (sub := _substitution(part, i)): + bodies.append(sub[0]) + i = sub[1] + elif c == quote: + quote = "" + elif c == '"' and not quote: + quote, started = c, True + elif quote or c not in " \t\n" + "".join(_PUNCT): + word.append(c) + started = True + else: + _end_word(stages, word, started) + started = False + i = _operator(part, i, stages) + i += 1 + _end_word(stages, word, started) + return [stage for stage in stages if stage], bodies + + +def _end_word(stages: list[list[str]], word: list[str], started: bool) -> None: + if started and "".join(word) not in _LEX_SKIP: + stages[-1].append("".join(word)) + word.clear() + + +def _substitution(part: str, i: int) -> tuple[str, int] | None: + """The body of the ``$( )`` or backtick substitution at ``i`` and the index of its last character; ``None`` + where ``$`` or a backtick starts none.""" + if part[i] == "$": + if part[i + 1:i + 2] != "(": + return None + end = substitution_end(part, i + 2) + return part[i + 2:end - 1], end - 1 + end = part.find("`", i + 1) + end = len(part) if end < 0 else end + return part[i + 1:end], end + + +def _operator(part: str, i: int, stages: list[list[str]]) -> int: + """Take the run of operator characters at ``i`` (a blank is none): a pipe starts a stage, a list separator or a + grouping parenthesis is dropped, a redirection stays a word; the index of the run's last character.""" + run = part[i].strip() + if not run: + return i + while part[i + 1:i + 2] in _PUNCT: + i += 1 + run += part[i] + if "|" in run: + stages.append([]) + elif run and run not in _SEPARATORS and not set(run) <= set("()"): + stages[-1].append(run) + return i def _views(words: list[str]) -> Iterator[Stage]: """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" + for rest in command_words(words): + yield _head(rest[0], rest[1:]) + + +def command_words(words: list[str]) -> Iterator[list[str]]: + """The words of the command a simple command runs, from its program on, then those of the command each wrapper + (``time``, ``sudo -n``, ``nice -n 5``...) runs: its options are not the program.""" i = 0 while i < len(words): while i < len(words) and _ASSIGN.match(words[i]): # NAME=value words set variables for the command after them i += 1 if i >= len(words): return - yield _head(words[i], words[i + 1:]) - if _program(words[i]) not in _WRAPPERS: + yield words[i:] + wrapper = _program(words[i]) + if wrapper not in _WRAPPERS: return i += 1 + while i < len(words) and words[i].startswith("-") and len(words[i]) > 1: + i += 2 if words[i] in _WRAPPER_VALUES.get(wrapper, ()) else 1 def _head(word: str, args: list[str]) -> Stage: @@ -224,16 +320,52 @@ def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: if probe.kind == "command": return _in_order(probe.stages, pipeline) if probe.kind == "path": - return any(prog in _READERS and probe.text in args for views in pipeline for prog, args in views) + return any(prog in _READERS and probe.text in _operands(prog, args) for views in pipeline for prog, args in views) return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) +def _operands(program: str, args: tuple[str, ...]) -> list[str]: + """The files a reading command names: not its options, their values, or the pattern ``grep`` and ``sed`` search for.""" + values = _READER_VALUES.get(program, ()) + needs_pattern = program in _PATTERN_READERS + files: list[str] = [] + i = 0 + while i < len(args): + word = args[i] + if word in _PATTERN_OPTIONS and needs_pattern: + needs_pattern = False + if word.startswith("-") and len(word) > 1: + i += 2 if word in values else 1 + continue + if needs_pattern: + needs_pattern = False + else: + files.append(word) + i += 1 + return files + + def _calls(args: tuple[str, ...], probe: Probe) -> bool: - """Whether the code of a ``python -c`` one-liner calls the probe's name.""" + """Whether the ``-c`` code of a Python command calls the probe's name, as code (not in a string or a comment).""" if "-c" not in args or probe.call is None: return False - code = args[args.index("-c") + 1:args.index("-c") + 2] - return bool(code) and probe.call.search(code[0]) is not None + at = args.index("-c") + if any(not w.startswith("-") for w in args[:at]): # after a script name, ``-c`` is the script's own + return False + try: + tree = ast.parse(args[at + 1]) if at + 1 < len(args) else None + except (SyntaxError, ValueError): + return False + return tree is not None and any(_dotted(node.func) == probe.call for node in ast.walk(tree) if isinstance(node, ast.Call)) + + +def _dotted(node: ast.expr) -> str: + """``os.cpu_count`` for the expression ``os.cpu_count``; ``""`` for anything that is not a dotted name.""" + parts: list[str] = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + return ".".join([node.id, *reversed(parts)]) if isinstance(node, ast.Name) else "" def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 2b500f401..04847215f 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -176,6 +176,72 @@ def test_an_import_after_a_semicolon_on_the_same_line_is_an_import(): assert _choices(events) == [("sklearn", "import", "app.py")] +def _bash_rc(command: str, **extra) -> list[dict]: + return [{"t": 1.0, "kind": "command", "text": command, **extra}] + + +def test_a_denied_install_gives_no_choice(): + events = with_choices(_bash_rc("pip install scikit-learn-intelex", rc=0, denied=True), SKLEARN, []) + assert _choices(events) == [] + assert _choices(with_choices(_bash_rc("pip install scikit-learn-intelex", rc=0), SKLEARN, [])) == [ + ("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + "pip install scikit-learn-intelex |cat", + "pip install scikit-learn-intelex|tail -3", + "pip install scikit-learn-intelex --help", + "pip install -h scikit-learn-intelex", + "pip --version install scikit-learn-intelex", + "pip uninstall scikit-learn-intelex --help", + "pip install --target scikit-learn-intelex numpy", + "pip install -r scikit-learn-intelex numpy", + "pip install -c scikit-learn-intelex numpy", + "pip install --index-url scikit-learn-intelex numpy", + "conda install --name scikit-learn-intelex numpy", + "exit 0 && pip install scikit-learn-intelex", +]) +def test_a_command_that_installs_nothing_the_candidate_names_gives_no_choice(command): + assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [] + + +@pytest.mark.parametrize("command", [ + "sudo pip install scikit-learn-intelex", + "time pip install scikit-learn-intelex", + "sudo -n pip install scikit-learn-intelex", + "nice -n 5 python -m pip install scikit-learn-intelex", + "pip.exe install scikit-learn-intelex", + "python.exe -m pip install scikit-learn-intelex", + "pip install --target lib scikit-learn-intelex", + "echo '|' && pip install scikit-learn-intelex", +]) +def test_a_wrapped_or_option_laden_install_still_gives_its_choice(command): + assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize(("text", "found"), [ + ("# example; import sklearnex\n", False), + ("# example; from sklearnex import patch_sklearn\n", False), + ("import os, \\\n sklearnex\n", True), + ("if True: import sklearnex\n", True), + ("from sklearnex.svm import SVC\n", True), + ("from sklearnexx import y\n", False), + ("import sklearnex.patches\n", True), + ("import sklearnexx\n", False), +]) +def test_an_import_is_read_from_code_not_from_a_comment(text, found): + event = {"t": 1.0, "kind": "edit", "path": "a.py", "rc": 0} + out = with_choices([event], SKLEARN, [(event, text)]) + assert (("sklearn", "import", "a.py") in _choices(out)) is found + + +def test_use_matches_a_dotted_name_only_at_a_name_boundary(): + event = {"t": 1.0, "kind": "edit", "path": "a.py", "rc": 0} + for text, found in (("x = scipy.linalg.lstsq(a)\n", True), ("x = myscipy.linalg.lstsq(a)\n", False), + ("x = a.scipy.linalg.lstsq(a)\n", False), ("x = scipy.linalg.lstsqx(a)\n", False)): + assert bool(_choices(with_choices([event], SCIPY, [(event, text)]))) is found, text + + @pytest.mark.parametrize(("raw", "message"), [ ({"sk": {"fit": ["x"]}}, "unknown act 'fit'"), ({"sk": {"import": ["sklearn learn"]}}, "takes a dotted Python name"), diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 3c9b96a7c..8c43d064e 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -253,3 +253,68 @@ def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): parse_manifest({**base, "probes": {"disk": ["df"]}}) + + +def _probes(command: str) -> list[str]: + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + return [e["pattern"] for e in events if e["kind"] == "probe"] + + +@pytest.mark.parametrize("command", [ + "cat <<'EOF' > script.sh\nnproc\nEOF", + "cat <<-EOF > script.sh\n\tnproc\n\tEOF", + "echo '$(nproc)'", + "echo '`nproc`'", + "echo \\$(nproc)", + "echo '|' nproc", + "sysctl -a '|' grep machdep.cpu", + "grep -e /proc/cpuinfo README", + "grep /proc/cpuinfo README", + "rg -e /proc/cpuinfo README", + "sed -n /proc/cpuinfo README", + "python -c 'print(\"os.cpu_count()\")'", + "python -c '# os.cpu_count()\nprint(1)'", + "python -c 'x = os.cpu_count'", +]) +def test_text_the_shell_does_not_run_gives_no_probe(command): + assert _probes(command) == [] + + +@pytest.mark.parametrize(("command", "found"), [ + ("make -j$(nproc --ignore=$(printf 1))", ["nproc"]), + ('make -j"$(nproc)"', ["nproc"]), + ("make -j`nproc`", ["nproc"]), + ("bash < Date: Sun, 15 Nov 2026 18:25:12 -0600 Subject: [PATCH 05/24] fix(repo): read choice and probe commands as the shell runs them - restore the `_package` definition: `pip install 'scipy >= 1.11'` raised NameError - `exec pip install x` gives its choice; later `&&` parts still give none - redirection operators and targets are no package or file operand; `< file` still reads - a string literal no longer hides or fakes an import on its line - `uv --quiet add x` gives its install choice - `sudo bash -c 'nproc'` and an unquoted heredoc's `$(nproc)` give their probe - `echo \| nproc` runs no pipeline - grep/rg/sed/awk long pattern options and `-f` no longer turn a file into a pattern - `exec true && pip install x` gives no choice; a quoted `'>'` is a word, not a redirection - a command after `false ||` gives its probe; only `true ||` and `: ||` skip it - `FOO=1 exec true && pip install x` gives no choice - a quoted `'1'` before a redirection is a word, not a descriptor Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/repos/choices.py | 66 +++++++++++++------ dokimi/dokimi/repos/probes.py | 104 ++++++++++++++++++++++-------- dokimi/tests/test_repo_choices.py | 20 ++++++ dokimi/tests/test_repo_probes.py | 25 +++++++ 4 files changed, 171 insertions(+), 44 deletions(-) diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 4d29e8a41..10444552a 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -22,7 +22,7 @@ import yaml -from .probes import command_stages, command_words +from .probes import command_stages, command_words, split_redirects from .shell_split import split_script ACTS = ("install", "import", "use", "remove") @@ -44,7 +44,7 @@ def validate_candidates(raw: Any, where: str = "--candidates") -> dict[str, dict[str, tuple[str, ...]]]: """A ``{id: {act: [names]}}`` mapping as ``{id: {act: (names,)}}``; ``ValueError`` names what is wrong.""" - if raw is None: + if raw is None or raw == {}: raise ValueError(f"{where}: the file names no candidate") if not isinstance(raw, Mapping): raise ValueError(f"{where}: expected a mapping of candidate to acts, got {type(raw).__name__}") @@ -114,23 +114,27 @@ def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str ``&&`` (rc 0 then means each ran), and not in a pipeline (the last stage's code). A ``;``, ``||`` or ``&`` part may have failed or never run, so its install gives no choice.""" parts = split_script(str(event.get("text", ""))) - if any(part.after not in ("", "&&") for part in parts): + if any(p.after not in ("", "&&") for p in parts[:-1]) or any(p.after not in ("", "&&", ";") for p in parts[-1:]): return [] found: dict[tuple[str, str], None] = {} for part, *_ in parts: + if part.startswith("("): # a subshell: its exit code is not the command's own + continue stages, _inner = command_stages(part) if len(stages) > 1: # a pipeline reports its last stage's exit code continue words = stages[0] if stages else [] - if words[:1] in (["exit"], ["exec"]): # exit or exec: no later part runs + views = list(command_words(words)) + if any(view[:1] == ["exit"] for view in views): # no later part runs + break + call = next(filter(None, map(_package_call, views)), None) + if call is not None: + act, packages = call + for cid, acts in candidates.items(): + if _wanted(acts, act) & set(packages): + found[(cid, act)] = None + if any(view[:1] == ["exec"] for view in views): # the shell is replaced by the call above: no later part runs break - call = next(filter(None, map(_package_call, command_words(words))), None) - if call is None: - continue - act, packages = call - for cid, acts in candidates.items(): - if _wanted(acts, act) & set(packages): - found[(cid, act)] = None return [_choice(event, cid, act, None) for cid, act in found] @@ -161,7 +165,7 @@ def _choice(event: dict[str, Any], cid: str, act: str, path: str | None) -> dict def _imports(name: str, text: str) -> bool: """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line; a statement may follow another on its line after a ``;`` or a ``:``, a ``\\`` continues a line, a comment is no code.""" - text = re.sub(r"\\\n[ \t]*", " ", re.sub(r"#[^\n]*", "", text)) + text = re.sub(r"\\\n[ \t]*", " ", _code(text)) if re.search(rf"(?:^|[;:])[ \t]*from\s+{re.escape(name)}(?!\w)", text, re.M): return True for line in re.finditer(r"(?:^|[;:])[ \t]*import\s+([^\n;]+)", text, re.M): @@ -172,6 +176,29 @@ def _imports(name: str, text: str) -> bool: return False +def _code(text: str) -> str: + """The text without its comments and the insides of its string literals (a quote left open runs to its line's end, + or to the end of a triple-quoted one: an edit may add only part of a file).""" + out: list[str] = [] + i, n = 0, len(text) + while i < n: + c = text[i] + if c == "#": + while i < n and text[i] != "\n": + i += 1 + elif c in "'\"": + quote = text[i:i + 3] if text[i:i + 3] == c * 3 else c + i += len(quote) + while i < n and not text.startswith(quote, i) and not (len(quote) == 1 and text[i] == "\n"): + i += 2 if text[i] == "\\" else 1 + i += len(quote) if text.startswith(quote, i) else 0 + out.append(" ") + else: + out.append(c) + i += 1 + return "".join(out) + + def _calls(name: str, text: str) -> bool: return re.search(rf"(? tuple[str, list[str]] | None: return None names: list[str] = [] skip = False - for word in rest[1:]: + for word in split_redirects(rest[1:])[0]: # ``> scipy`` names an output file, not a package if skip: skip = False elif word.startswith("-"): @@ -208,10 +235,13 @@ def _manager_rest(words: list[str]) -> list[str]: head = posixpath.basename(words[0].replace("\\", "/")).casefold().removesuffix(".exe") if _PYTHON.fullmatch(head) and words[1:3] == ["-m", "pip"]: return words[3:] - if head == "uv" and words[1:2] == ["pip"]: - return words[2:] - if head == "uv" and words[1:2] in (["add"], ["remove"]): # uv's project verbs: ``uv add`` installs a requirement - return words[1:] + if head == "uv": + rest = words[1:] + while rest and rest[0].startswith("-"): # ``uv --quiet add`` + rest = rest[1:] + if rest[:1] == ["pip"]: + return rest[1:] + return rest if rest[:1] in (["add"], ["remove"]) else [] # uv's project verbs: ``uv add`` installs a requirement if _PIP.fullmatch(head) or head in ("conda", "mamba", "poetry"): return words[1:] return [] @@ -219,5 +249,5 @@ def _manager_rest(words: list[str]) -> list[str]: def _package(word: str) -> str: """A requirement's name as the package manager compares it: no version, extras or URL; case and ``-_.`` folded.""" - name = _REQUIREMENT_END.split(word, maxsplit=1)[0] + name = _REQUIREMENT_END.split(word, maxsplit=1)[0].strip() return re.sub(r"[-_.]+", "-", name).casefold() diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index d00354ff3..794f7e10c 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -54,11 +54,16 @@ "nice": frozenset({"-n", "--adjustment"}), "exec": frozenset({"-a"}), "time": frozenset({"-f", "-o", "--format", "--output"})} # options that take the next word _PATTERN_READERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is a pattern or script -_READER_VALUES = {"head": {"-n", "-c"}, "tail": {"-n", "-c"}, "grep": {"-e", "-f", "-m", "-A", "-B", "-C"}, - "egrep": {"-e", "-f", "-m", "-A", "-B", "-C"}, "fgrep": {"-e", "-f", "-m", "-A", "-B", "-C"}, - "rg": {"-e", "-f", "-m", "-A", "-B", "-C", "-g", "-t", "-T"}, "sed": {"-e", "-f", "-l"}, - "awk": {"-F", "-v", "-f"}} # options that take the next word -_PATTERN_OPTIONS = frozenset({"-e", "--regexp"}) # the pattern or script comes from the option, not the first operand +_GREP_VALUES = {"-e", "-f", "-m", "-A", "-B", "-C", "--regexp", "--file", "--max-count", "--after-context", + "--before-context", "--context"} +_READER_VALUES = {"head": {"-n", "-c", "--lines", "--bytes"}, "tail": {"-n", "-c", "--lines", "--bytes"}, + "grep": _GREP_VALUES, "egrep": _GREP_VALUES, "fgrep": _GREP_VALUES, + "rg": _GREP_VALUES | {"-g", "-t", "-T", "--glob", "--type", "--type-not"}, + "sed": {"-e", "-f", "-l", "--expression", "--file", "--line-length"}, + "awk": {"-F", "-v", "-f", "--file", "--assign", "--field-separator"}} # options that take the next word +_PATTERN_OPTIONS = frozenset({"-e", "-f", "--regexp", "--file", "--expression"}) # the pattern, script or a file of it comes from the option, not the first operand +_REDIRECT = re.compile(r"[<>&]*[<>][<>&]*") +_OPERATOR = "\0" # marks a word that is a shell operator, as against a quoted ``'>'`` _PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _ASSIGN = re.compile(r"[A-Za-z_]\w*=") @@ -182,34 +187,60 @@ def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" - for part, _after, _in_subshell, _outer in split_script(_without_heredocs(script)): + never = False # the part after ``true ||`` never runs; one after any other command may + for part, after, _in_subshell, _outer in split_script(_without_heredocs(script)): + skipped, never = never, after == "||" and part.strip() in ("true", ":") + if skipped: + continue stages, inner = command_stages(part) for body in inner: yield from _pipelines(body) if stages: yield stages - for stage in stages: # ``bash -lc 'nproc'``: the script a shell runs is a script of its own - if len(stage) > 2 and _program(stage[0]) in _SHELLS and _SHELL_C.fullmatch(stage[1]): - yield from _pipelines(stage[2]) + for stage in stages: # ``sudo bash -lc 'nproc'``: the script a shell runs is a script of its own + for rest in command_words(stage): + if len(rest) > 2 and _program(rest[0]) in _SHELLS and _SHELL_C.fullmatch(rest[1]): + yield from _pipelines(rest[2]) def _without_heredocs(script: str) -> str: - """The script without the bodies of its heredocs, which are data (``cat < run.sh``) unless a shell reads them.""" + """The script without the bodies of its heredocs, which are data (``cat < run.sh``) unless a shell reads them; + the ``$( )`` and backtick substitutions of an unquoted body ran, so they stay.""" if "<<" not in script: return script - lines, out, end = script.split("\n"), [], None - for line in lines: + out: list[str] = [] + end: str | None = None + body: list[str] = [] + expands = False + for line in script.split("\n"): if end is not None: if line.strip() == end: - end = None + out.extend(_expansions("\n".join(body)) if expands else []) + end, body = None, [] + else: + body.append(line) continue out.append(line) m = _HEREDOC.search(line) if m and not _SHELL_LINE.match(line): - end = m.group(2) + end, expands = m.group(2), not m.group(1) return "\n".join(out) +def _expansions(body: str) -> list[str]: + """The ``$( )`` and backtick substitutions of an unquoted heredoc body, as text.""" + found: list[str] = [] + i = 0 + while i < len(body): + if body[i] == "\\": + i += 1 + elif body[i] in "$`" and (sub := _substitution(body, i)): + found.append(body[i:sub[1] + 1]) + i = sub[1] + i += 1 + return found + + def command_stages(part: str) -> tuple[list[list[str]], list[str]]: """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped), and the bodies of the ``$( )`` and backtick substitutions outside single quotes: they ran as commands of their own.""" @@ -217,6 +248,7 @@ def command_stages(part: str) -> tuple[list[list[str]], list[str]]: bodies: list[str] = [] word: list[str] = [] started = False # a quoted empty string is a word + bare = True # the word so far has no quote or escape in it quote, i = "", 0 while i < len(part): c = part[i] @@ -224,11 +256,11 @@ def command_stages(part: str) -> tuple[list[list[str]], list[str]]: end = part.find("'", i + 1) end = len(part) if end < 0 else end word.append(part[i + 1:end]) - started = True + started, bare = True, False i = end - elif c == "\\" and part[i + 1:i + 2] in ("$", "`", '"'): - word.append(part[i + 1]) # an escaped ``$(`` is text; any other backslash is a path separator - started = True + elif c == "\\" and (part[i + 1:i + 2] in ("$", "`", '"') or (not quote and part[i + 1:i + 2] in tuple("|;&<>()"))): + word.append(part[i + 1]) # an escaped ``$(`` or ``|`` is text; any other backslash is a path separator + started, bare = True, False i += 1 elif c in "$`" and (sub := _substitution(part, i)): bodies.append(sub[0]) @@ -236,13 +268,16 @@ def command_stages(part: str) -> tuple[list[list[str]], list[str]]: elif c == quote: quote = "" elif c == '"' and not quote: - quote, started = c, True + quote, started, bare = c, True, False elif quote or c not in " \t\n" + "".join(_PUNCT): word.append(c) started = True else: + if c in "<>" and bare and "".join(word).isdigit(): # ``2>file``: the 2 names a descriptor, it is no word + word.clear() + started = False _end_word(stages, word, started) - started = False + started, bare = False, True i = _operator(part, i, stages) i += 1 _end_word(stages, word, started) @@ -280,10 +315,27 @@ def _operator(part: str, i: int, stages: list[list[str]]) -> int: if "|" in run: stages.append([]) elif run and run not in _SEPARATORS and not set(run) <= set("()"): - stages[-1].append(run) + stages[-1].append(_OPERATOR + run) return i +def split_redirects(words: Sequence[str]) -> tuple[list[str], list[str]]: + """The words without their redirections (an operator and its target), and the targets of ``<`` redirections: the + files the command reads.""" + plain: list[str] = [] + reads: list[str] = [] + i = 0 + while i < len(words): + if words[i].startswith(_OPERATOR) and _REDIRECT.fullmatch(words[i][1:]): + if words[i] == _OPERATOR + "<" and i + 1 < len(words): + reads.append(words[i + 1]) + i += 2 + continue + plain.append(words[i]) + i += 1 + return plain, reads + + def _views(words: list[str]) -> Iterator[Stage]: """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" for rest in command_words(words): @@ -328,13 +380,13 @@ def _operands(program: str, args: tuple[str, ...]) -> list[str]: """The files a reading command names: not its options, their values, or the pattern ``grep`` and ``sed`` search for.""" values = _READER_VALUES.get(program, ()) needs_pattern = program in _PATTERN_READERS - files: list[str] = [] + plain, files = split_redirects(args) i = 0 - while i < len(args): - word = args[i] - if word in _PATTERN_OPTIONS and needs_pattern: - needs_pattern = False + while i < len(plain): + word = plain[i] if word.startswith("-") and len(word) > 1: + if needs_pattern and (word.split("=", 1)[0] if word.startswith("--") else word[:2]) in _PATTERN_OPTIONS: + needs_pattern = False i += 2 if word in values else 1 continue if needs_pattern: diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 04847215f..b4fe4b05c 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -200,6 +200,13 @@ def test_a_denied_install_gives_no_choice(): "pip install --index-url scikit-learn-intelex numpy", "conda install --name scikit-learn-intelex numpy", "exit 0 && pip install scikit-learn-intelex", + "pip install numpy > scikit-learn-intelex", + "pip install numpy 2>scikit-learn-intelex", + "pip uninstall numpy > scikit-learn-intelex", + "exec pip install numpy && pip install scikit-learn-intelex", + "exec true && pip install scikit-learn-intelex", + "FOO=1 exec true && pip install scikit-learn-intelex", + "echo \\| pip install scikit-learn-intelex", ]) def test_a_command_that_installs_nothing_the_candidate_names_gives_no_choice(command): assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [] @@ -214,6 +221,10 @@ def test_a_command_that_installs_nothing_the_candidate_names_gives_no_choice(com "python.exe -m pip install scikit-learn-intelex", "pip install --target lib scikit-learn-intelex", "echo '|' && pip install scikit-learn-intelex", + "exec pip install scikit-learn-intelex", + "uv --quiet add scikit-learn-intelex", + "pip install numpy 2>&1 > /dev/null scikit-learn-intelex", + "pip install 'scikit-learn-intelex >= 1.0'", ]) def test_a_wrapped_or_option_laden_install_still_gives_its_choice(command): assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [("sklearn", "install", None)] @@ -228,6 +239,10 @@ def test_a_wrapped_or_option_laden_install_still_gives_its_choice(command): ("from sklearnexx import y\n", False), ("import sklearnex.patches\n", True), ("import sklearnexx\n", False), + ('s = "#"; import sklearnex\n', True), + ('"""\nimport sklearnex\n"""\n', False), + ("s = 'import sklearnex'\n", False), + ('s = "a\\"b"; import sklearnex\n', True), ]) def test_an_import_is_read_from_code_not_from_a_comment(text, found): event = {"t": 1.0, "kind": "edit", "path": "a.py", "rc": 0} @@ -253,3 +268,8 @@ def test_use_matches_a_dotted_name_only_at_a_name_boundary(): def test_a_bad_candidate_config_is_refused_with_what_is_wrong(raw, message): with pytest.raises(ValueError, match=message): validate_candidates(raw, "candidates") + + +def test_an_exec_of_a_removal_gives_its_choice(): + assert _choices(with_choices(_bash_rc("exec pip uninstall -y scikit-learn-intelex", rc=0), SKLEARN, [])) == [ + ("sklearn", "remove", None)] diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 8c43d064e..e008f5cc8 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -275,6 +275,16 @@ def _probes(command: str) -> list[str]: "python -c 'print(\"os.cpu_count()\")'", "python -c '# os.cpu_count()\nprint(1)'", "python -c 'x = os.cpu_count'", + "echo \\| nproc", + "cat README > /etc/os-release", + "cat README >> /etc/os-release", + "cat README 2>/etc/os-release", + "grep --regexp /proc/cpuinfo README", + "grep --regexp=/proc/cpuinfo README", + "grep -f patterns README /proc/cpuinfo.bak", + "cat <<'EOF'\n$(nproc)\nEOF", + "true || nproc", + ": || nproc", ]) def test_text_the_shell_does_not_run_gives_no_probe(command): assert _probes(command) == [] @@ -282,6 +292,21 @@ def test_text_the_shell_does_not_run_gives_no_probe(command): @pytest.mark.parametrize(("command", "found"), [ ("make -j$(nproc --ignore=$(printf 1))", ["nproc"]), + ("sudo -n bash -c 'nproc'", ["nproc"]), + ("time bash -lc 'nproc'", ["nproc"]), + ("cat < out.txt < /etc/os-release", ["/etc/os-release"]), + ("grep -f patterns /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep '>' /proc/cpuinfo", ["/proc/cpuinfo"]), + ("false || nproc", ["nproc"]), + ("grep '1'>/dev/null /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep 1>/dev/null cores /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep \"<\" /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep --file patterns /proc/cpuinfo", ["/proc/cpuinfo"]), + ("sed -f script.sed /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep -n cores /proc/cpuinfo 2>/dev/null", ["/proc/cpuinfo"]), ('make -j"$(nproc)"', ["nproc"]), ("make -j`nproc`", ["nproc"]), ("bash < Date: Sun, 15 Nov 2026 21:28:25 -0800 Subject: [PATCH 06/24] fix(repo): read exec, uv options, chained ||, escapes and redirect targets as the shell does Thread: PRRT_kwDOVBg7g863v6ZT Thread: PRRT_kwDOVBg7g863v6ZX Thread: PRRT_kwDOVBg7g863v6ZZ Thread: PRRT_kwDOVBg7g863v6Zd Thread: PRRT_kwDOVBg7g863v6Zi Thread: PRRT_kwDOVBg7g863v6Zl Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/choices.py | 9 +++++---- dokimi/dokimi/repos/probes.py | 18 ++++++++++++++---- dokimi/tests/test_repo_choices.py | 22 ++++++++++++++++++++++ dokimi/tests/test_repo_probes.py | 14 ++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 10444552a..9ee1093ea 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -33,6 +33,7 @@ _PYTHON = re.compile(r"python\d*(?:\.\d+)?") _PIP = re.compile(r"pip\d*(?:\.\d+)?") _REQUIREMENT_END = re.compile(r"[<>=!~\[;@]") # where a requirement's name ends: a version, extras, a URL +_UV_VALUE_OPTIONS = frozenset({"--directory", "--project", "--config-file", "--cache-dir", "--python", "-p", "--color"}) _INFO_FLAGS = frozenset({"-h", "--help", "-V", "--version"}) # the manager prints text and exits 0, changing nothing _VALUE_OPTIONS = frozenset({ "-r", "--requirement", "-c", "--constraint", "-e", "--editable", "-t", "--target", "-i", "--index-url", @@ -123,7 +124,7 @@ def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str stages, _inner = command_stages(part) if len(stages) > 1: # a pipeline reports its last stage's exit code continue - words = stages[0] if stages else [] + words = split_redirects(stages[0] if stages else [])[0] # ``exec >log`` has no program, ``> scipy`` no operand views = list(command_words(words)) if any(view[:1] == ["exit"] for view in views): # no later part runs break @@ -133,7 +134,7 @@ def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str for cid, acts in candidates.items(): if _wanted(acts, act) & set(packages): found[(cid, act)] = None - if any(view[:1] == ["exec"] for view in views): # the shell is replaced by the call above: no later part runs + if any(view[:1] == ["exec"] and len(view) > 1 for view in views): # exec with a program replaces the shell break return [_choice(event, cid, act, None) for cid, act in found] @@ -237,8 +238,8 @@ def _manager_rest(words: list[str]) -> list[str]: return words[3:] if head == "uv": rest = words[1:] - while rest and rest[0].startswith("-"): # ``uv --quiet add`` - rest = rest[1:] + while rest and rest[0].startswith("-"): # ``uv --quiet add``, ``uv --directory /tmp/p add`` + rest = rest[2:] if rest[0] in _UV_VALUE_OPTIONS else rest[1:] if rest[:1] == ["pip"]: return rest[1:] return rest if rest[:1] in (["add"], ["remove"]) else [] # uv's project verbs: ``uv add`` installs a requirement diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 794f7e10c..733e7bf6e 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -187,11 +187,13 @@ def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" - never = False # the part after ``true ||`` never runs; one after any other command may + ok = False # the list so far succeeded (its last part ran ``true``): a part after ``||`` never runs + before = "" # the operator before this part for part, after, _in_subshell, _outer in split_script(_without_heredocs(script)): - skipped, never = never, after == "||" and part.strip() in ("true", ":") + skipped, before = before == "||" and ok, after if skipped: continue + ok = part.strip() in ("true", ":") stages, inner = command_stages(part) for body in inner: yield from _pipelines(body) @@ -212,10 +214,12 @@ def _without_heredocs(script: str) -> str: end: str | None = None body: list[str] = [] expands = False + head = 0 # the line that opens the heredoc for line in script.split("\n"): if end is not None: if line.strip() == end: - out.extend(_expansions("\n".join(body)) if expands else []) + if expands: + out[head] += "".join(f" {text}" for text in _expansions("\n".join(body))) end, body = None, [] else: body.append(line) @@ -223,7 +227,7 @@ def _without_heredocs(script: str) -> str: out.append(line) m = _HEREDOC.search(line) if m and not _SHELL_LINE.match(line): - end, expands = m.group(2), not m.group(1) + end, expands, head = m.group(2), not m.group(1), len(out) - 1 return "\n".join(out) @@ -258,12 +262,18 @@ def command_stages(part: str) -> tuple[list[list[str]], list[str]]: word.append(part[i + 1:end]) started, bare = True, False i = end + elif c == "\\" and not quote and part[i + 1:i + 2] == "\\": # ``\\|`` is a backslash, then a pipe + word.append("\\\\") + started, bare = True, False + i += 1 elif c == "\\" and (part[i + 1:i + 2] in ("$", "`", '"') or (not quote and part[i + 1:i + 2] in tuple("|;&<>()"))): word.append(part[i + 1]) # an escaped ``$(`` or ``|`` is text; any other backslash is a path separator started, bare = True, False i += 1 elif c in "$`" and (sub := _substitution(part, i)): bodies.append(sub[0]) + word.append("\1") # the word's text is the substitution's output: a placeholder keeps its place + started, bare = True, False i = sub[1] elif c == quote: quote = "" diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index b4fe4b05c..1d1afa361 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -273,3 +273,25 @@ def test_a_bad_candidate_config_is_refused_with_what_is_wrong(raw, message): def test_an_exec_of_a_removal_gives_its_choice(): assert _choices(with_choices(_bash_rc("exec pip uninstall -y scikit-learn-intelex", rc=0), SKLEARN, [])) == [ ("sklearn", "remove", None)] + + +INSTALL_SCIPY = {"scipy": {"install": ["scipy"]}} + + +@pytest.mark.parametrize("command", [ + "exec >install.log && pip install scipy", # a commandless exec only redirects: the shell goes on + "exec && pip install scipy", + "uv --directory /tmp/project add scipy", # uv's global options take a value: it is not the project's verb + "uv --project=/tmp/project --quiet add scipy", + "pip install >$(printf /tmp/install.log) scipy", # a substitution is the redirection's target, not the package +]) +def test_a_command_the_shell_runs_before_or_around_its_redirects_gives_its_install(command): + assert _choices(_run(INSTALL_SCIPY, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("scipy", "install", None)] + + +@pytest.mark.parametrize("command", [ + "pip install scipy 2>logs\\\\|cat", # ``\\`` is a backslash: the pipe after it runs cat, whose code is the result + "exec true && pip install scipy", # exec with a program replaces the shell: nothing after it runs +]) +def test_a_pipeline_or_an_exec_with_a_program_gives_no_install(command): + assert _choices(_run(INSTALL_SCIPY, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index e008f5cc8..0afdf664f 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -343,3 +343,17 @@ def test_a_python_probe_needs_a_call_on_a_name_boundary(): assert _probes("python -c 'x = myos.cpu_count()'") == [] assert _probes("python -c 'x = a.os.cpu_count()'") == [] assert _probes("python -c 'os.cpu_count()'") == ["py:os.cpu_count"] + + +@pytest.mark.parametrize("command, patterns", [ + ("true || false || nproc", []), # ``||`` after a success skips the rest of the chain + ("true || false && nproc", ["nproc"]), # ``&&`` after a skipped part runs its own command + ("false || nproc", ["nproc"]), + ("true || cat <$(printf /tmp/probe.log) /etc/os-release", ["/etc/os-release"]), # the file is the operand, not the target +]) +def test_the_shell_structure_decides_which_probe_runs(command, patterns): + assert _probes(command) == patterns From 5d216a5e624ef763395c11e6290584b862b479d8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:28:25 -0800 Subject: [PATCH 07/24] Address review: record threads already fixed on this head Thread: PRRT_kwDOVBg7g863nhD_: already fixed in 5705149 (pipe without spaces) Thread: PRRT_kwDOVBg7g863nhEB: already fixed in 5705149 (semicolon in a comment) Thread: PRRT_kwDOVBg7g863nhEC: already fixed in 5705149 (help and version requests) Thread: PRRT_kwDOVBg7g863nhEE: already fixed in 5705149 (option values are not packages) Thread: PRRT_kwDOVBg7g863nhEG: already fixed in 5705149 (heredoc body is data) Thread: PRRT_kwDOVBg7g863nhEI: already fixed in 5705149 (quoted substitutions) Thread: PRRT_kwDOVBg7g863nhEJ: already fixed in 5705149 (nested substitutions) Thread: PRRT_kwDOVBg7g863nhEM: already fixed in 5705149 (quoted pipe) Thread: PRRT_kwDOVBg7g863nhEN: already fixed in 5705149 (wrapper options) Thread: PRRT_kwDOVBg7g863nhEP: already fixed in 5705149 (grep pattern options) Thread: PRRT_kwDOVBg7g863nhES: already fixed in 5705149 (python -c strings) Thread: PRRT_kwDOVBg7g863o6RG: already fixed in 5705149 (contract names Copilot edit tools) Thread: PRRT_kwDOVBg7g863sWEA: already fixed in 5705149 (exec replaces the shell) Thread: PRRT_kwDOVBg7g863sWEF: already fixed in 5705149 (comment and string scanner) Thread: PRRT_kwDOVBg7g863sWEK: already fixed in 5705149 (redirect targets) Thread: PRRT_kwDOVBg7g863sWEN: already fixed in 5705149 (wrapped shell scripts) Thread: PRRT_kwDOVBg7g863sWEV: already fixed in 5705149 (unquoted heredoc expansions) Thread: PRRT_kwDOVBg7g863sWEd: already fixed in 5705149 (escaped operators) Thread: PRRT_kwDOVBg7g863sWEi: already fixed in 5705149 (output redirection is no read) Thread: PRRT_kwDOVBg7g863sWEo: already fixed in 5705149 (grep --regexp and -f) Signed-off-by: Nikolay Petrov From 525851767e541cfe04360ee891377ddb6d09667d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:40:59 -0800 Subject: [PATCH 08/24] Address review: count a choice only where its exit code is 0 A call with no result (rc None: a cut-off call, or a Copilot command, which writes no rc) and a failed one both take no choice. The fake harness gets a result flag for the events test, so its golden cell ids move with its bytes. Thread: PRRT_kwDOVBg7g863nhD6 Thread: IC_kwDOVBg7g88AAAABmzVvzA#1 Thread: IC_kwDOVBg7g88AAAABm4kUeA#1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/choices.py | 5 +++-- dokimi/tests/repo_eval_fixture.py | 6 +++++- dokimi/tests/test_batch_ledger_golden.py | 8 ++++---- dokimi/tests/test_repo_choices.py | 12 ++++++++++++ dokimi/tests/test_repo_run_events.py | 1 + 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 9ee1093ea..e97fc7317 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -101,8 +101,9 @@ def with_choices(events: list[dict[str, Any]], candidates: Mapping[str, Mapping[ def _ran(event: dict[str, Any]) -> bool: - """Whether the call took effect: a denied one or one that failed did not install, import, use or remove anything.""" - return not event.get("denied") and event.get("rc") in (None, 0) + """Whether the call took effect: a denied one, one that failed, or one with no result (rc None, a cut-off call + or a Copilot command, which writes no rc) did not install, import, use or remove anything.""" + return not event.get("denied") and event.get("rc") == 0 def choice_timeline(events: Sequence[dict[str, Any]]) -> list[tuple[float | None, str, str]]: diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 4925cfa20..72e29cc3c 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -125,6 +125,10 @@ lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} for i, (n, a) in enumerate(json.loads(os.environ["FAKE_TOOL_USES"]))]}}) + if os.environ.get("FAKE_TOOL_RESULTS"): # a success for each of them: a choice needs its result (rc 0) + lines.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": f"u{i}", "is_error": False, "content": "ok"} + for i in range(len(json.loads(os.environ["FAKE_TOOL_USES"])))]}}) result = {"type": "result", "subtype": "success", "result": "done: src/calc.py", "total_cost_usd": 0.01, "num_turns": 3, "is_error": False, "usage": {"input_tokens": 5, "output_tokens": 2, "cache_read_input_tokens": 100, @@ -147,7 +151,7 @@ "--max-budget-usd", "{budget}", "{tools}"], "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "all": []}, "config_dir_env": "FAKE_CONFIG_DIR", - "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_SKILLS", "FAKE_USAGE", "FAKE_PROCESS", "FAKE_TOOL_USES", "PATH"], + "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_SKILLS", "FAKE_USAGE", "FAKE_PROCESS", "FAKE_TOOL_RESULTS", "FAKE_TOOL_USES", "PATH"], "egress": ["api.example.com"], "trace": "claude-stream-json", "prompt_suffix": "Stay in the foreground.", "config_files": {"settings.json": '{"hook": "python3 {support_dir}/no_background.py"}'}, diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index ac45bf111..fdfe68411 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -37,22 +37,22 @@ # The id moves whenever the fixture binary changes (its digest is in the id), e.g. when it gains a tool_result. GOLDEN_SAMPLE_CELLS = { "fix_add__guidance-none__m-1__r1": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-none__m-1__r2": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r1": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r2": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 1d1afa361..02a0d1508 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -136,6 +136,18 @@ def test_a_failed_install_gives_no_choice(): def test_a_module_in_a_multi_module_import_line_is_an_import(): +def test_a_call_with_no_result_gives_no_choice(): + unresolved = [_call("a", "Bash", {"command": "pip install scikit-learn-intelex"}, "2026-01-01T00:00:05Z"), + _call("e", "Edit", {"file_path": f"{WS}/x.py", "old_string": "", "new_string": "import sklearnex"}, + "2026-01-01T00:00:06Z")] + assert _choices(_run(SKLEARN, unresolved)) == [] + + +@pytest.mark.parametrize("rc", [None, 1]) +def test_a_command_whose_exit_code_is_not_0_gives_no_choice(rc): + assert _choices(with_choices(_bash_rc("pip install scikit-learn-intelex", rc=rc), SKLEARN, [])) == [] + + events = _run(SKLEARN, _edit("a", "app.py", "import os, sklearnex as sk\n", "2026-01-01T00:00:05Z")) assert _choices(events) == [("sklearn", "import", "app.py")] diff --git a/dokimi/tests/test_repo_run_events.py b/dokimi/tests/test_repo_run_events.py index 14fdfd560..438b18537 100644 --- a/dokimi/tests/test_repo_run_events.py +++ b/dokimi/tests/test_repo_run_events.py @@ -37,6 +37,7 @@ def test_a_run_writes_the_probe_and_choice_rows_its_files_name(setup, monkeypatc probes.write_text("cpu: [nproc]\n") monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([ ["Bash", {"command": "nproc"}], ["Write", {"file_path": f"{WORKSPACE}/a.py", "content": "import scipy\n"}]])) + monkeypatch.setenv("FAKE_TOOL_RESULTS", "1") _run(setup, "--arm", "guidance:none", "--task", "fix_add", "--candidates", str(cands), "--probes", str(probes)) (events_file,) = (setup["out"] / "b1").glob("*/events.jsonl") kinds = [(e["kind"], e.get("pattern") or e.get("act")) for e in map(json.loads, events_file.read_text().splitlines())] From 7d9b70d18878cbc6d51bbe176183fcfeedbcac5b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:41:06 -0800 Subject: [PATCH 09/24] Address review: a resume with other --candidates or --probes is refused The execution id takes the digest of each given file's parsed content, so a resume with another file is another execution and check_batch_executions refuses it. An unset file adds no key, so the ids of runs without one stay. Thread: IC_kwDOVBg7g88AAAABmzVvzA#2 Thread: PRR_kwDOVBg7g88AAAABYEmcQg#1 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/batch_ledger.py | 11 ++++++++--- dokimi/dokimi/repos/runner.py | 3 ++- dokimi/tests/test_repo_run_events.py | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/batch_ledger.py b/dokimi/dokimi/repos/batch_ledger.py index 6699c163b..df35f700a 100644 --- a/dokimi/dokimi/repos/batch_ledger.py +++ b/dokimi/dokimi/repos/batch_ledger.py @@ -72,7 +72,7 @@ def harness_sha256(path: Path) -> str: def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native", harness_sha256: str | None = None, harness_version: str | None = None, companion_sha256: dict[str, str | None] | None = None, - overlay: str | None = None) -> str: + overlay: str | None = None, inputs: dict[str, Any] | None = None) -> str: """Short hash of how the agent is started: the profile's ``invoke`` template with its tool arguments, environment, config, prompt suffix, egress, bridge, trace, answer format and companions, and for a cell of the overlay arm the overlay it keeps (``overlay``; the other cells carry none, so their ids do not change). @@ -100,15 +100,20 @@ def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native" else {n: companion_sha256.get(n) for n in profile.bin_companions}) if overlay is not None: key["overlay"] = overlay + given = {n: v for n, v in sorted((inputs or {}).items()) if v is not None} # unset adds no key, as overlay does + if given: # the run's own files (--candidates, --probes): a resume with other ones is another execution + key["inputs"] = {n: hashlib.sha256(json.dumps(v, sort_keys=True, default=list).encode("utf-8")).hexdigest() + for n, v in given.items()} return hashlib.sha256(json.dumps(key, sort_keys=True).encode("utf-8")).hexdigest()[:12] def execution_record(profile: HarnessProfile, tools: str, delegation: str, harness_sha256: str, harness_version: str | None, companion_sha256: dict[str, str | None] | None = None, - overlay: str | None = None) -> dict[str, str]: + overlay: str | None = None, inputs: dict[str, Any] | None = None) -> dict[str, str]: """How a cell's agent was started, as ``meta.json`` and ``result.json`` record it.""" - return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256, overlay), + return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256, overlay, + inputs), "tools": tools, "delegation": profile.effective_delegation(tools, delegation)} diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 08762161c..8863cd078 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -115,8 +115,9 @@ def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, st companions = {n: harness_sha256(beside / n) if (beside / n).is_file() else None for n in opts.profile.bin_companions} overlay = opts.profile.overlay if arm is not None and parse_arm(arm).base == OVERLAY_ARM else None + inputs = {"candidates": opts.candidates, "probes": opts.probes} return batch_ledger.execution_record(opts.profile, opts.tools, opts.delegation, harness_sha256(opts.harness_bin), - opts.harness_version, companions, overlay) + opts.harness_version, companions, overlay, inputs) def check_batch_executions(batch_dir: Path, options: list[AgentOptions], replacing: set[str] | None = None) -> None: diff --git a/dokimi/tests/test_repo_run_events.py b/dokimi/tests/test_repo_run_events.py index 438b18537..d42d0b425 100644 --- a/dokimi/tests/test_repo_run_events.py +++ b/dokimi/tests/test_repo_run_events.py @@ -42,3 +42,20 @@ def test_a_run_writes_the_probe_and_choice_rows_its_files_name(setup, monkeypatc (events_file,) = (setup["out"] / "b1").glob("*/events.jsonl") kinds = [(e["kind"], e.get("pattern") or e.get("act")) for e in map(json.loads, events_file.read_text().splitlines())] assert ("probe", "nproc") in kinds and ("choice", "import") in kinds + + +@pytest.mark.parametrize("flag,other", [("--candidates", "sp: {import: [sklearnex]}\n"), ("--probes", "cpu: [lscpu]\n")]) +def test_a_resume_with_other_candidates_or_probes_is_refused(setup, tmp_path, capsys, flag, other): + cands, probes, changed = tmp_path / "cands.yaml", tmp_path / "probes.yaml", tmp_path / "changed.yaml" + cands.write_text("sp: {import: [scipy]}\n") + probes.write_text("cpu: [nproc]\n") + changed.write_text(other) + + def args(c, p): + return ["--arm", "guidance:none", "--task", "fix_add", "--candidates", str(c), "--probes", str(p)] + + _run(setup, *args(cands, probes)) + _run_again = (args(changed, probes) if flag == "--candidates" else args(cands, changed)) + with pytest.raises(SystemExit): + _run(setup, *_run_again) + assert "holds cells of another execution" in capsys.readouterr().err From a77dda9a3f98472049c1acd1e3d03a12cf0dfec6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:41:06 -0800 Subject: [PATCH 10/24] Address review: pin the spellings and the python -c rule that mutants passed A bracketed extra on the package name and a -c that follows a script name were not pinned by any test: dropping either left the suite green. Thread: PRR_kwDOVBg7g88AAAABYEmcQg#2 Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_choices.py | 3 ++- dokimi/tests/test_repo_probes.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 02a0d1508..80ff33e42 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -88,6 +88,7 @@ def test_a_write_counts_its_whole_content(): "pip install 'scikit-learn-intelex==2024.7'", "uv pip install scikit_learn_intelex", "python3 -m pip install --upgrade scikit-learn-intelex", + "pip install scikit-learn-intelex[extra]", ]) def test_a_package_manager_install_names_the_package_whatever_the_spelling(command): assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] @@ -135,7 +136,6 @@ def test_a_failed_install_gives_no_choice(): assert _choices(_run(SKLEARN, failed)) == [] -def test_a_module_in_a_multi_module_import_line_is_an_import(): def test_a_call_with_no_result_gives_no_choice(): unresolved = [_call("a", "Bash", {"command": "pip install scikit-learn-intelex"}, "2026-01-01T00:00:05Z"), _call("e", "Edit", {"file_path": f"{WS}/x.py", "old_string": "", "new_string": "import sklearnex"}, @@ -148,6 +148,7 @@ def test_a_command_whose_exit_code_is_not_0_gives_no_choice(rc): assert _choices(with_choices(_bash_rc("pip install scikit-learn-intelex", rc=rc), SKLEARN, [])) == [] +def test_a_module_in_a_multi_module_import_line_is_an_import(): events = _run(SKLEARN, _edit("a", "app.py", "import os, sklearnex as sk\n", "2026-01-01T00:00:05Z")) assert _choices(events) == [("sklearn", "import", "app.py")] diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 0afdf664f..3aa792a25 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -285,6 +285,7 @@ def _probes(command: str) -> list[str]: "cat <<'EOF'\n$(nproc)\nEOF", "true || nproc", ": || nproc", + "python tool.py -c 'import os; os.cpu_count()'", ]) def test_text_the_shell_does_not_run_gives_no_probe(command): assert _probes(command) == [] From 8bc8b6c9da0bd4392b4156806abd2f8b3f020da6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:41:14 -0800 Subject: [PATCH 11/24] Address review: keep a heredoc's substitutions before its operator They were appended to the opening line, so a comment after the operator hid them: cat < --- dokimi/dokimi/repos/probes.py | 9 +++++---- dokimi/tests/test_repo_probes.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 733e7bf6e..6db3106c6 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -214,12 +214,13 @@ def _without_heredocs(script: str) -> str: end: str | None = None body: list[str] = [] expands = False - head = 0 # the line that opens the heredoc + head, at = 0, 0 # the line that opens the heredoc, and where its operator starts on it for line in script.split("\n"): if end is not None: if line.strip() == end: - if expands: - out[head] += "".join(f" {text}" for text in _expansions("\n".join(body))) + if expands: # before the operator: a comment after it on the opening line would hide them + found = "".join(f"{text} " for text in _expansions("\n".join(body))) + out[head] = out[head][:at] + found + out[head][at:] end, body = None, [] else: body.append(line) @@ -227,7 +228,7 @@ def _without_heredocs(script: str) -> str: out.append(line) m = _HEREDOC.search(line) if m and not _SHELL_LINE.match(line): - end, expands, head = m.group(2), not m.group(1), len(out) - 1 + end, expands, head, at = m.group(2), not m.group(1), len(out) - 1, m.start() return "\n".join(out) diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 3aa792a25..3989e1dea 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -296,6 +296,8 @@ def test_text_the_shell_does_not_run_gives_no_probe(command): ("sudo -n bash -c 'nproc'", ["nproc"]), ("time bash -lc 'nproc'", ["nproc"]), ("cat < cfg # note\n$(nproc)\nEOF", ["nproc"]), ("cat < out.txt < /etc/os-release", ["/etc/os-release"]), From 4abdcb28ee43c8ed804b4add9b95ab89ea71d36f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:07:17 -0800 Subject: [PATCH 12/24] fix(repo): a backslash pair inside double quotes is one backslash Inside "..." the second backslash of `\\` was read as an escape of the closing quote, so a pipe after it was lost (probe) or a pipeline read as one stage (choice). Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 2 +- dokimi/tests/test_repo_choices.py | 1 + dokimi/tests/test_repo_probes.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 6db3106c6..efa25be76 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -263,7 +263,7 @@ def command_stages(part: str) -> tuple[list[list[str]], list[str]]: word.append(part[i + 1:end]) started, bare = True, False i = end - elif c == "\\" and not quote and part[i + 1:i + 2] == "\\": # ``\\|`` is a backslash, then a pipe + elif c == "\\" and part[i + 1:i + 2] == "\\": # ``\\|`` is a backslash, then a pipe, in double quotes too word.append("\\\\") started, bare = True, False i += 1 diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 80ff33e42..11ca16721 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -304,6 +304,7 @@ def test_a_command_the_shell_runs_before_or_around_its_redirects_gives_its_insta @pytest.mark.parametrize("command", [ "pip install scipy 2>logs\\\\|cat", # ``\\`` is a backslash: the pipe after it runs cat, whose code is the result + 'pip install scipy "--log=C:\\\\" | tee l', # inside double quotes too: the quote closes, and the pipe is real "exec true && pip install scipy", # exec with a program replaces the shell: nothing after it runs ]) def test_a_pipeline_or_an_exec_with_a_program_gives_no_install(command): diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index 3989e1dea..a9416b3bb 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -355,6 +355,7 @@ def test_a_python_probe_needs_a_call_on_a_name_boundary(): ("true || cat <$(printf /tmp/probe.log) /etc/os-release", ["/etc/os-release"]), # the file is the operand, not the target ]) From b2e56e250a0b928d3d909759f88bed656bf0d711 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:07:17 -0800 Subject: [PATCH 13/24] docs(repo): the contract names the choice exit-code rule and the candidate and probe digests The execution id hashes the parsed --candidates and --probes files, so a resume with other ones is refused; the _ran docstring says a Copilot call with no exit code or success flag is rc None. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 6 ++++-- dokimi/dokimi/repos/choices.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index faeee4b1d..42b1a3d87 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -559,6 +559,7 @@ reads the upstream base (`base_sha`) and the workspace start commit events carry no added text (Copilot's `edit`, `create`, `str_replace_editor`, `write` and `apply_patch`, Codex's `file_change`) gives install and remove events only: no import or use choice there does not mean the agent did not choose the candidate. + A choice needs the call's own exit code 0: a failed or denied call, or one with no result, gives none. An install or removal gives a choice only when its command joins every part to the next with `&&` and has no pipe: the harness reports one exit code per command, so a `;`, `||` or `|` part could have failed or never run. A `--dry-run` installs nothing and gives none. An install in a `bash -c` wrapper, or whose first word carries a @@ -2330,8 +2331,9 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric `id` hashing the profile's name, `invoke`, tool arguments, `env_set`, `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress`, `bridge`, `trace` (unless `claude-stream-json`), `answer_format`, the SHA-256 of the harness executable (`--bin`), - the SHA-256 of each `bin_companions` file and harness version. - A rerun whose `id` differs (another `--tools`, a `--delegation` that changes the tool arguments, profile, + the SHA-256 of each `bin_companions` file and harness version. A `repo run` with `--candidates` or `--probes` adds the + digest of each parsed file (the same mapping, not its bytes), so another file with other content is a different `id`. + A rerun whose `id` differs (another `--tools`, another `--candidates` or `--probes` file, a `--delegation` that changes the tool arguments, profile, harness binary or harness version, or an edited `invoke`, `env_set`, `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress`, `bridge`, `trace` or `answer_format`) is refused before any agent starts, and `--force` or another `--batch` runs it. `--force` replaces the cells of the run's plan only: if the batch dir holds finished cells outside that plan diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index e97fc7317..39531e8b8 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -101,8 +101,8 @@ def with_choices(events: list[dict[str, Any]], candidates: Mapping[str, Mapping[ def _ran(event: dict[str, Any]) -> bool: - """Whether the call took effect: a denied one, one that failed, or one with no result (rc None, a cut-off call - or a Copilot command, which writes no rc) did not install, import, use or remove anything.""" + """Whether the call took effect: a denied one, one that failed, or one with no result (rc None: a cut-off call, + or a Copilot command with neither an exit code nor a success flag) did not install, import, use or remove anything.""" return not event.get("denied") and event.get("rc") == 0 From 949755343e3735259ea6cf7334a83ada714a55cb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 22:12:21 -0800 Subject: [PATCH 14/24] Address review: name the review and comments whose numbered rows answer them Thread: PRR_kwDOVBg7g88AAAABYEmcQg Thread: IC_kwDOVBg7g88AAAABmzVvzA Thread: IC_kwDOVBg7g88AAAABm4kUeA Signed-off-by: Nikolay Petrov From fb286a2cdc9b14d9902ecaff9a97928653d7994e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:00:26 -0800 Subject: [PATCH 15/24] fix(repo): an exec without a program, conda's download and pip's --report give no false choice exec -l has no program and does not end the shell, so the install after it counts. conda's --download-only fetches packages without linking them, and pip's --report takes a file name as its value, so the next word is not a package. Thread: PRRT_kwDOVBg7g863zaJY Thread: PRRT_kwDOVBg7g863zaJf Thread: PRRT_kwDOVBg7g863zaJk Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/choices.py | 7 ++++--- dokimi/tests/test_repo_choices.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 39531e8b8..57a366f68 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -40,7 +40,8 @@ "--extra-index-url", "-f", "--find-links", "--prefix", "--root", "--python", "--name", "-n", "-p", "--platform", "--python-version", "--abi", "--implementation", "--src", "--cache-dir", "--log", "--proxy", "--retries", "--timeout", "--cert", "--client-cert", "--trusted-host", "--config-settings", "-C", "--channel", "--group", - "--with"}) # options that take the next word, which is not a package + "--with", "--report", "--upgrade-strategy", "--progress-bar", "--keyring-provider", "--no-binary", "--only-binary", + "--global-option", "--install-option", "--exists-action", "--use-feature"}) # options that take the next word, which is not a package def validate_candidates(raw: Any, where: str = "--candidates") -> dict[str, dict[str, tuple[str, ...]]]: @@ -135,7 +136,7 @@ def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str for cid, acts in candidates.items(): if _wanted(acts, act) & set(packages): found[(cid, act)] = None - if any(view[:1] == ["exec"] and len(view) > 1 for view in views): # exec with a program replaces the shell + if any(view[:1] == ["exec"] for view in views[:-1]): # exec with a program replaces the shell; ``exec -l`` has none break return [_choice(event, cid, act, None) for cid, act in found] @@ -207,7 +208,7 @@ def _calls(name: str, text: str) -> bool: def _package_call(words: list[str]) -> tuple[str, list[str]] | None: """The act and the package names of a package manager's install, add, uninstall or remove call.""" - if "--dry-run" in words or _INFO_FLAGS & set(words): # a dry run installs nothing, nor does a help request + if {"--dry-run", "--download-only"} & set(words) or _INFO_FLAGS & set(words): # no install: a dry run, a download, a help request return None rest = _manager_rest(words) while rest and rest[0].startswith("-"): # ``pip --quiet install``: the options before the verb diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index 11ca16721..a75ed19a8 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -309,3 +309,32 @@ def test_a_command_the_shell_runs_before_or_around_its_redirects_gives_its_insta ]) def test_a_pipeline_or_an_exec_with_a_program_gives_no_install(command): assert _choices(_run(INSTALL_SCIPY, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def _choice_of(text: str, rc: object = 0) -> list[tuple]: + events = [{"t": 0.0, "kind": "command", "text": text, "rc": rc}] + return _choices(with_choices(events, SCIPY_INSTALL, [])) + + +SCIPY_INSTALL = {"scipy": {"install": ["scipy"]}} + + +@pytest.mark.parametrize("text", [ + "sudo --help pip install scipy", # the wrapper's --help runs no install + "conda install --download-only scipy", # downloads only: nothing is linked into the environment + "pip install --report scipy numpy", # --report takes the next word as its file +]) +def test_an_install_that_installs_nothing_gives_no_choice(text): + assert _choice_of(text) == [] + + +def test_exec_without_a_program_does_not_end_the_shell_for_the_install_after_it(): + assert _choice_of("exec -l && pip install scipy") == [("scipy", "install", None)] + + +def test_exec_with_a_program_still_gives_its_install(): + assert _choice_of("exec pip install scipy") == [("scipy", "install", None)] + + +def test_a_missing_exit_code_gives_no_choice(): + assert _choice_of("pip install scipy", rc=None) == [] From 26a68732168b3abecbe7483395af018517341ba4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:00:31 -0800 Subject: [PATCH 16/24] fix(repo): probes match only the commands the shell runs A denied command runs nothing. command -v and a wrapper's --help or --version run no program, and env runs the program after its NAME=value words, so no probe is of env there. A part after a failed && or a succeeded || never runs, nor does anything after an exit or an exec with a program. A heredoc ends only at its exact delimiter, and only an unquoted << opens one. $(( )) is arithmetic, not a substitution. A redirection's target is no operand of a command pattern. Thread: PRRT_kwDOVBg7g863zU5_ Thread: PRRT_kwDOVBg7g863zU6O Thread: PRRT_kwDOVBg7g863zU6U Thread: PRRT_kwDOVBg7g863zaJq Thread: PRRT_kwDOVBg7g863zaJs Thread: PRRT_kwDOVBg7g863zaJu Thread: PRRT_kwDOVBg7g863zaJy Thread: PRR_kwDOVBg7g88AAAABYJsZAA#2 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 52 ++++++++++++++++++++++++-------- dokimi/tests/test_repo_probes.py | 37 +++++++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index efa25be76..7b35464b8 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -49,10 +49,12 @@ _READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", "strings", "wc", "rg"}) -_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice"}) # each runs the command after it +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice", "env"}) # each runs the command after it +_INFO_OPTIONS = frozenset({"--help", "--version", "-V"}) # a wrapper given one prints text and runs nothing _WRAPPER_VALUES = {"sudo": frozenset({"-u", "-g", "-h", "-p", "-r", "-t", "-C", "-D", "-R", "-T", "-U"}), "nice": frozenset({"-n", "--adjustment"}), "exec": frozenset({"-a"}), - "time": frozenset({"-f", "-o", "--format", "--output"})} # options that take the next word + "time": frozenset({"-f", "-o", "--format", "--output"}), + "env": frozenset({"-u", "-C", "--unset", "--chdir"})} # options that take the next word _PATTERN_READERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is a pattern or script _GREP_VALUES = {"-e", "-f", "-m", "-A", "-B", "-C", "--regexp", "--file", "--max-count", "--after-context", "--before-context", "--context"} @@ -134,7 +136,8 @@ def with_probes(events: list[dict[str, Any]], probes: Sequence[Probe]) -> list[d for event in events: out.append(event) if event.get("kind") == "command": - out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) + if not event.get("denied"): # a denied command ran nothing + out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) return out @@ -187,13 +190,13 @@ def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" - ok = False # the list so far succeeded (its last part ran ``true``): a part after ``||`` never runs + status: bool | None = None # the exit status of the last part that ran; None when it is not known before = "" # the operator before this part for part, after, _in_subshell, _outer in split_script(_without_heredocs(script)): - skipped, before = before == "||" and ok, after - if skipped: + skipped, before = (before == "||" and status is True) or (before == "&&" and status is False), after + if skipped: # a part after ``||`` that succeeded, or after ``&&`` that failed, never runs continue - ok = part.strip() in ("true", ":") + status = {"true": True, ":": True, "false": False}.get(part.strip()) stages, inner = command_stages(part) for body in inner: yield from _pipelines(body) @@ -203,6 +206,14 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: for rest in command_words(stage): if len(rest) > 2 and _program(rest[0]) in _SHELLS and _SHELL_C.fullmatch(rest[1]): yield from _pipelines(rest[2]) + if any(_ends_shell(stage) for stage in stages): # what follows an ``exit`` or a replacing ``exec`` never runs + return + + +def _ends_shell(stage: list[str]) -> bool: + """Whether a simple command ends the shell: ``exit``, or ``exec`` with a program (``exec -l`` has none).""" + views = list(command_words(split_redirects(stage)[0])) + return any(_program(view[0]) == "exit" for view in views) or any(_program(view[0]) == "exec" for view in views[:-1]) def _without_heredocs(script: str) -> str: @@ -215,9 +226,10 @@ def _without_heredocs(script: str) -> str: body: list[str] = [] expands = False head, at = 0, 0 # the line that opens the heredoc, and where its operator starts on it + strip_tabs = False # ``<<-``: the terminator may be indented with tabs, and only those for line in script.split("\n"): if end is not None: - if line.strip() == end: + if (line.lstrip("\t") if strip_tabs else line) == end: if expands: # before the operator: a comment after it on the opening line would hide them found = "".join(f"{text} " for text in _expansions("\n".join(body))) out[head] = out[head][:at] + found + out[head][at:] @@ -226,12 +238,22 @@ def _without_heredocs(script: str) -> str: body.append(line) continue out.append(line) - m = _HEREDOC.search(line) + m = _opener(line) if m and not _SHELL_LINE.match(line): end, expands, head, at = m.group(2), not m.group(1), len(out) - 1, m.start() + strip_tabs = m.group(0).startswith("<<-") return "\n".join(out) +def _opener(line: str) -> re.Match[str] | None: + """The first heredoc operator of a line that is not inside quotes (``echo "< list[str]: """The ``$( )`` and backtick substitutions of an unquoted heredoc body, as text.""" found: list[str] = [] @@ -305,7 +327,7 @@ def _substitution(part: str, i: int) -> tuple[str, int] | None: """The body of the ``$( )`` or backtick substitution at ``i`` and the index of its last character; ``None`` where ``$`` or a backtick starts none.""" if part[i] == "$": - if part[i + 1:i + 2] != "(": + if part[i + 1:i + 2] != "(" or part[i + 2:i + 3] == "(": # ``$((`` is arithmetic: it runs no command return None end = substitution_end(part, i + 2) return part[i + 2:end - 1], end - 1 @@ -349,7 +371,11 @@ def split_redirects(words: Sequence[str]) -> tuple[list[str], list[str]]: def _views(words: list[str]) -> Iterator[Stage]: """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" - for rest in command_words(words): + views = list(command_words(words)) + for k, rest in enumerate(views): + later = [v for v in views[k + 1:] if not v[0].startswith(_OPERATOR)] # a redirection is no command + if _program(rest[0]) == "env" and later: # ``env FOO=1 nproc`` runs ``nproc``, not a probe of ``env`` + continue yield _head(rest[0], rest[1:]) @@ -368,6 +394,8 @@ def command_words(words: list[str]) -> Iterator[list[str]]: return i += 1 while i < len(words) and words[i].startswith("-") and len(words[i]) > 1: + if words[i] in _INFO_OPTIONS or (wrapper == "command" and words[i] == "-v"): # looks up or prints: runs nothing + return i += 2 if words[i] in _WRAPPER_VALUES.get(wrapper, ()) else 1 @@ -440,5 +468,5 @@ def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: def _stage_runs(want: Stage, views: list[Stage]) -> bool: """Whether one of the views is the wanted program with every wanted word among its words, any case.""" program, words = want - return any(prog == program and all(w.casefold() in {a.casefold() for a in args} for w in words) + return any(prog == program and all(w.casefold() in {a.casefold() for a in split_redirects(args)[0]} for w in words) for prog, args in views) diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index a9416b3bb..e7fbbe5cd 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -361,3 +361,40 @@ def test_a_python_probe_needs_a_call_on_a_name_boundary(): ]) def test_the_shell_structure_decides_which_probe_runs(command, patterns): assert _probes(command) == patterns + + +def _probed(text: str, **event: object) -> list[tuple[str, str]]: + from dokimi.repos.probes import probes_for, with_probes + + events = [{"t": 0.0, "kind": "command", "text": text, "rc": 0, **event}] + return [(e["target"], e["pattern"]) for e in with_probes(events, probes_for(None)) if e["kind"] == "probe"] + + +@pytest.mark.parametrize("text, event", [ + ("nproc", {"denied": True}), # a denied command ran nothing + ("true || echo skipped || nproc", {}), + ("command -v nproc", {}), # ``command -v`` only looks the program up + ("sudo --help nproc", {}), # ``--help`` prints text and runs nothing + ("false && nproc", {}), + ("exit 0; nproc", {}), + ("exec true && nproc", {}), # exec replaces the shell: nothing after it runs + ("cat < list", {}), # a redirection target is no pattern word +]) +def test_a_command_that_runs_nothing_probed_gives_no_probe(text, event): + assert not [p for p in _probed(text, **event) if p[0] == "cpu" or p[1] in ("pip list",)] + + +@pytest.mark.parametrize("text", ["exec -l && nproc", "env FOO=1 nproc", "echo \"< f") + assert ("env", "printenv") in _probed("printenv") From f24bb1eb7cd5fd8db4a634ad90bd4695e9e0b392 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:00:31 -0800 Subject: [PATCH 17/24] fix(repo): a Copilot shell command's rc is its exit code only The success flag of a Copilot result is no exit code, so it no longer gives a shell command's rc. Every CLI's choice needs rc 0 from the code the command returned; a read still counts a result reported as failed. Refs #807. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/cli_traces.py | 6 +++--- dokimi/tests/test_repo_cli_traces.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index e59477d4f..0814576c5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -771,9 +771,9 @@ def _copilot_result(out: Transcript, call: _Derived, data: dict[str, Any]) -> No continue shell = data.get("shellExecution") or {} code = shell.get("exitCode") if isinstance(shell, dict) else None - event["rc"] = code if isinstance(code, int) else (None if data.get("success") is None - else 0 if data.get("success") else 1) - failed = _failed_read(str(event["text"]), event["rc"]) + event["rc"] = code if isinstance(code, int) else None # the code the command returned, or none: no success fallback + failed = _failed_read(str(event["text"]), event["rc"] if event["rc"] is not None or data.get("success") is not False + else 1) # a read still counts a result reported as failed, as before result = data.get("result") or {} output = result.get("content") if isinstance(result, dict) else None if isinstance(output, str) and output: diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fe9d5c381..45913be7e 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -713,10 +713,14 @@ def test_copilot_json_to_events(): assert t.reads() == {"AGENTS.md", "sub/AGENTS.md"} -@pytest.mark.parametrize(("done", "rc"), [({"success": True}, 0), ({"success": False}, 1), ({}, None)], - ids=["success", "failure", "unknown"]) -def test_copilot_command_exit_code_falls_back_to_success(done, rc): - """Without `shellExecution.exitCode` the rc comes from `success`; with neither it stays unknown.""" +@pytest.mark.parametrize(("done", "rc"), [({"shellExecution": {"exitCode": 0}}, 0), + ({"shellExecution": {"exitCode": 1}, "success": True}, 1), + ({"success": True}, None), ({"success": False}, None), ({}, None)], + ids=["exit-0", "exit-code-wins-over-success", "success-without-code", "failure-without-code", + "unknown"]) +def test_copilot_command_rc_is_its_exit_code_only(done, rc): + """A shell command's rc is `shellExecution.exitCode`, the code the command returned; a result with only `success` + gives no rc, so no choice rests on it (the same rule for every CLI).""" raw = _jsonl( {"type": "tool.execution_start", "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "make"}}}, From f9130103b39b093c15a6e38e5692de39e6bb17e6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:00:39 -0800 Subject: [PATCH 18/24] Address review: already fixed on the PR head A `||` chain whose first part is true skips the rest, and a call with no exit code gives no choice: both are pinned by regressions on the head (949755343). Thread: PRRT_kwDOVBg7g863zU6I: already fixed in 949755343 Thread: PRR_kwDOVBg7g88AAAABYJsZAA#1: already fixed in 949755343 Signed-off-by: Nikolay Petrov From d93f7cb7fd350717171ef441d4560cdeb7ebc933 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:03:44 -0800 Subject: [PATCH 19/24] fix(repo): an exit or exec that may not run ends no scan A conditional exit (cd x || exit 1) does not end the scan, nor does one in a subshell. The Copilot shell rc rule is written into the contract, which gives null for a result with only success. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 4 +++- dokimi/dokimi/repos/choices.py | 4 ++-- dokimi/dokimi/repos/probes.py | 7 ++++--- dokimi/tests/test_repo_probes.py | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 42b1a3d87..98739e4a1 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -522,7 +522,9 @@ reads the upstream base (`base_sha`) and the workspace start commit line, and `output_truncated` is `true`; build errors are usually at the end. - `rc` is what the harness reports. Where it reports only success or failure (Claude Code), `rc` is `0` or `1`; a Claude Code `Skill` tool event carries it - too (`1`: the skill was not loaded). It is `null` when no result came back, + too (`1`: the skill was not loaded). A Copilot shell command takes its `rc` from + `shellExecution.exitCode` only, so a result that carries only `success` gives `null` + (no choice rests on it). It is `null` when no result came back, for example when the run stopped at its turn or budget limit; a grader must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 57a366f68..2499bdb88 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -102,8 +102,8 @@ def with_choices(events: list[dict[str, Any]], candidates: Mapping[str, Mapping[ def _ran(event: dict[str, Any]) -> bool: - """Whether the call took effect: a denied one, one that failed, or one with no result (rc None: a cut-off call, - or a Copilot command with neither an exit code nor a success flag) did not install, import, use or remove anything.""" + """Whether the call took effect: a denied one, one that failed, or one with no exit code (rc None: a cut-off call, + or a shell command whose result carries none) did not install, import, use or remove anything.""" return not event.get("denied") and event.get("rc") == 0 diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 7b35464b8..2570b38e7 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -192,7 +192,8 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" status: bool | None = None # the exit status of the last part that ran; None when it is not known before = "" # the operator before this part - for part, after, _in_subshell, _outer in split_script(_without_heredocs(script)): + for part, after, in_subshell, _outer in split_script(_without_heredocs(script)): + conditional = before in ("||", "&&") # the part runs only if the one before it did, or did not, succeed skipped, before = (before == "||" and status is True) or (before == "&&" and status is False), after if skipped: # a part after ``||`` that succeeded, or after ``&&`` that failed, never runs continue @@ -206,8 +207,8 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: for rest in command_words(stage): if len(rest) > 2 and _program(rest[0]) in _SHELLS and _SHELL_C.fullmatch(rest[1]): yield from _pipelines(rest[2]) - if any(_ends_shell(stage) for stage in stages): # what follows an ``exit`` or a replacing ``exec`` never runs - return + if not conditional and not in_subshell and any(_ends_shell(stage) for stage in stages): + return # what follows an ``exit`` or a replacing ``exec`` that surely runs never runs def _ends_shell(stage: list[str]) -> bool: diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index e7fbbe5cd..cb1363d59 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -387,7 +387,7 @@ def test_a_command_that_runs_nothing_probed_gives_no_probe(text, event): @pytest.mark.parametrize("text", ["exec -l && nproc", "env FOO=1 nproc", "echo \"< Date: Sun, 15 Nov 2026 23:06:21 -0800 Subject: [PATCH 20/24] fix(repo): an exit inside a group or a block ends no scan A { ...; exit 1; } or a multi-line if may not run its exit, so the scan goes on past it. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/probes.py | 7 +++++-- dokimi/tests/test_repo_probes.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py index 2570b38e7..b611327a6 100644 --- a/dokimi/dokimi/repos/probes.py +++ b/dokimi/dokimi/repos/probes.py @@ -77,6 +77,7 @@ _SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script _LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command _SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command +_GROUPED = re.compile(r"[{}()]|\b(?:if|then|else|elif|fi|do|done|while|until|case|esac)\b") # a group or a block Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words @@ -192,7 +193,9 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" status: bool | None = None # the exit status of the last part that ran; None when it is not known before = "" # the operator before this part - for part, after, in_subshell, _outer in split_script(_without_heredocs(script)): + text = _without_heredocs(script) + flat = not _GROUPED.search(text) # a group or a block may hold an exit that does not run: then nothing ends the scan + for part, after, in_subshell, _outer in split_script(text): conditional = before in ("||", "&&") # the part runs only if the one before it did, or did not, succeed skipped, before = (before == "||" and status is True) or (before == "&&" and status is False), after if skipped: # a part after ``||`` that succeeded, or after ``&&`` that failed, never runs @@ -207,7 +210,7 @@ def _pipelines(script: str) -> Iterator[list[list[str]]]: for rest in command_words(stage): if len(rest) > 2 and _program(rest[0]) in _SHELLS and _SHELL_C.fullmatch(rest[1]): yield from _pipelines(rest[2]) - if not conditional and not in_subshell and any(_ends_shell(stage) for stage in stages): + if flat and not conditional and not in_subshell and any(_ends_shell(stage) for stage in stages): return # what follows an ``exit`` or a replacing ``exec`` that surely runs never runs diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index cb1363d59..cf7a3204e 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -387,7 +387,8 @@ def test_a_command_that_runs_nothing_probed_gives_no_probe(text, event): @pytest.mark.parametrize("text", ["exec -l && nproc", "env FOO=1 nproc", "echo \"< Date: Sun, 15 Nov 2026 23:49:34 -0800 Subject: [PATCH 21/24] fix(repo): a conda channel prefix is not part of the package name conda-forge::scipy gives no install choice for scipy: the channel prefix stayed in the name. The name is now the part after the last '::', then the version, extras and URL are cut as before. Thread: PRRT_kwDOVBg7g8630I1c Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/choices.py | 4 ++-- dokimi/tests/test_repo_choices.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py index 2499bdb88..acf36f4c6 100644 --- a/dokimi/dokimi/repos/choices.py +++ b/dokimi/dokimi/repos/choices.py @@ -251,6 +251,6 @@ def _manager_rest(words: list[str]) -> list[str]: def _package(word: str) -> str: - """A requirement's name as the package manager compares it: no version, extras or URL; case and ``-_.`` folded.""" - name = _REQUIREMENT_END.split(word, maxsplit=1)[0].strip() + """A requirement's name as the package manager compares it: no channel, version, extras or URL; case and ``-_.`` folded.""" + name = _REQUIREMENT_END.split(word.rpartition("::")[2], maxsplit=1)[0].strip() return re.sub(r"[-_.]+", "-", name).casefold() diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py index a75ed19a8..f6aaabc0a 100644 --- a/dokimi/tests/test_repo_choices.py +++ b/dokimi/tests/test_repo_choices.py @@ -336,5 +336,13 @@ def test_exec_with_a_program_still_gives_its_install(): assert _choice_of("exec pip install scipy") == [("scipy", "install", None)] +@pytest.mark.parametrize("text", [ + "conda install -y conda-forge::scipy", # the channel prefix is not part of the package's name + "conda install -y conda-forge::scipy>=1.11", +]) +def test_a_conda_channel_prefix_still_names_the_package(text): + assert _choice_of(text) == [("scipy", "install", None)] + + def test_a_missing_exit_code_gives_no_choice(): assert _choice_of("pip install scipy", rc=None) == [] From 56791a5b1a34fd2ae22ebd605972bc3420abc2b8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:49:34 -0800 Subject: [PATCH 22/24] docs(repo): a Copilot shell rc change moves failed_commands and first_useful The contract and a changelog fragment say that failed_commands and first_useful read the Copilot shell rc, so a success-only result is now null for them. Refs #807 Signed-off-by: Nikolay Petrov --- .../20261115_120000_napetrov_copilot-shell-rc.md | 11 +++++++++++ dokimi/docs/repo-evaluation-contract.md | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md diff --git a/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md b/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md new file mode 100644 index 000000000..45b4c83b3 --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md @@ -0,0 +1,11 @@ + + +### Changed + +- **`repo run`: a Copilot shell command's `rc` is its `shellExecution.exitCode` only.** A result that carries only + `success` no longer gets `rc` `0` or `1` from it, so it is `null`. `failed_commands` and `first_useful` read this `rc`, + so for such a command they now count neither a failure nor a useful result where they counted one before. Results + that carry `shellExecution.exitCode` are unchanged. Refs #807. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 98739e4a1..14b9c441b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -524,7 +524,9 @@ reads the upstream base (`base_sha`) and the workspace start commit (Claude Code), `rc` is `0` or `1`; a Claude Code `Skill` tool event carries it too (`1`: the skill was not loaded). A Copilot shell command takes its `rc` from `shellExecution.exitCode` only, so a result that carries only `success` gives `null` - (no choice rests on it). It is `null` when no result came back, + (no choice rests on it). The counters read this `rc` too: for a Copilot shell + command that carries only `success`, `failed_commands` and `first_useful` now see `null` + where they saw `0` or `1` before, so such a command is neither a failed nor a useful one. It is `null` when no result came back, for example when the run stopped at its turn or budget limit; a grader must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's From 052e3749994107ca6554596c62f51b6c2e83ff25 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:53:13 -0800 Subject: [PATCH 23/24] test(repo): a <<- heredoc ends at a tab-indented terminator only Pins the strip_tabs branch of the heredoc body scan, and reword the Copilot rc paragraph of the contract to state the rule. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 5 ++--- dokimi/tests/test_repo_probes.py | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 14b9c441b..13506edc7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -524,9 +524,8 @@ reads the upstream base (`base_sha`) and the workspace start commit (Claude Code), `rc` is `0` or `1`; a Claude Code `Skill` tool event carries it too (`1`: the skill was not loaded). A Copilot shell command takes its `rc` from `shellExecution.exitCode` only, so a result that carries only `success` gives `null` - (no choice rests on it). The counters read this `rc` too: for a Copilot shell - command that carries only `success`, `failed_commands` and `first_useful` now see `null` - where they saw `0` or `1` before, so such a command is neither a failed nor a useful one. It is `null` when no result came back, + (no choice rests on it). `failed_commands` and `first_useful` read this `rc` too, so + such a command counts as neither failed nor useful. It is `null` when no result came back, for example when the run stopped at its turn or budget limit; a grader must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py index cf7a3204e..d90b32c23 100644 --- a/dokimi/tests/test_repo_probes.py +++ b/dokimi/tests/test_repo_probes.py @@ -358,6 +358,8 @@ def test_a_python_probe_needs_a_call_on_a_name_boundary(): ("echo \"a\\\\\" | nproc", ["nproc"]), # inside double quotes too: the closing quote is found, the pipe is real ("echo \\| nproc", []), ("cat >$(printf /tmp/probe.log) /etc/os-release", ["/etc/os-release"]), # the file is the operand, not the target + ("cat <<-EOF\n\tEOF\nnproc", ["nproc"]), # ``<<-`` takes a tab-indented terminator: the body ends there + ("cat <<-EOF\n EOF\nnproc\nEOF", []), # a space does not indent a ``<<-`` terminator: nproc is data ]) def test_the_shell_structure_decides_which_probe_runs(command, patterns): assert _probes(command) == patterns From 2661870ca56d4f33cf43ee2092bfb2f498567445 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 00:27:07 -0800 Subject: [PATCH 24/24] Address review: the review body's items are answered in their threads Thread: PRR_kwDOVBg7g88AAAABYKG9xw: already fixed in 052e374999 Signed-off-by: Nikolay Petrov