diff --git a/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md new file mode 100644 index 000000000..1903e829a --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_probe-events.md @@ -0,0 +1,14 @@ + + +### Added + +- **`repo run`: a `probe` event for a command that asks the host for its facts.** A command + such as `nproc`, `lscpu`, `nvidia-smi`, `pip list`, `printenv` or a `python -c` one-liner that + calls `os.cpu_count()` gets a `probe` event after its `command` event, with its `target` + (`cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`) and the `pattern` it matched. The + patterns match the program the shell runs, not argument text (`grep nproc README` is no probe). + The defaults ship in `dokimi/repos/probes.py`; a manifest's `probes` key or `repo run --probes + FILE` replaces a target's list, and `[]` turns it off. Existing event kinds are unchanged. diff --git a/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md new file mode 100644 index 000000000..79b4ea225 --- /dev/null +++ b/dokimi/changelog.d/20261114_120100_napetrov_choice-events.md @@ -0,0 +1,13 @@ + + +### Added + +- **`repo run --candidates FILE`: a `choice` event for an install, import, use or removal of a candidate.** A file + maps each candidate id to package names (`install`, `remove`), modules an added `import` line names (`import`) and + dotted names an added line mentions (`use`). A `choice` event follows the `command` or `edit` event that does it, with + its `candidate`, `act` and, for an edit, its `path`. Only added text counts: an edit's new side, a write's content. + Copilot's edit tools and Codex's `file_change` carry no added text, so they give install and remove events only. Existing event kinds are + unchanged. diff --git a/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md b/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md new file mode 100644 index 000000000..45b4c83b3 --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_copilot-shell-rc.md @@ -0,0 +1,11 @@ + + +### Changed + +- **`repo run`: a Copilot shell command's `rc` is its `shellExecution.exitCode` only.** A result that carries only + `success` no longer gets `rc` `0` or `1` from it, so it is `null`. `failed_commands` and `first_useful` read this `rc`, + so for such a command they now count neither a failure nor a useful result where they counted one before. Results + that carry `shellExecution.exitCode` are unchanged. Refs #807. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..8ba74e20f 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1380,6 +1380,7 @@ {"key": "--cache", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--cache-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--cache-root", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--candidates", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--delegation", "action": "_StoreAction", "choices": ["'native'", "'none'"], "const": null, "default": "native", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--docker", "action": "_StoreAction", "choices": null, "const": null, "default": "docker", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--force", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, @@ -1394,6 +1395,7 @@ {"key": "--max-turns", "action": "_StoreAction", "choices": null, "const": null, "default": 200, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--model", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-runs", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--probes", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index ccf72154b..13506edc7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -244,6 +244,9 @@ build_systems: [bazel, make] # selects L0 claim-check adapters process_metrics: # optional repository counters, result.json `process.counters` (§4.1) bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any + +probes: # optional: target -> patterns of `probe` events (§4.1) + cpu: [nproc, "py:os.cpu_count"] # a listed target replaces its defaults; [] turns it off ``` Rules: @@ -519,7 +522,10 @@ reads the upstream base (`base_sha`) and the workspace start commit line, and `output_truncated` is `true`; build errors are usually at the end. - `rc` is what the harness reports. Where it reports only success or failure (Claude Code), `rc` is `0` or `1`; a Claude Code `Skill` tool event carries it - too (`1`: the skill was not loaded). It is `null` when no result came back, + too (`1`: the skill was not loaded). A Copilot shell command takes its `rc` from + `shellExecution.exitCode` only, so a result that carries only `success` gives `null` + (no choice rests on it). `failed_commands` and `first_useful` read this `rc` too, so + such a command counts as neither failed nor useful. It is `null` when no result came back, for example when the run stopped at its turn or budget limit; a grader must not count `null` as a failure. - `agent` is present on an event a subagent made: the id of the parent's @@ -527,6 +533,41 @@ reads the upstream base (`base_sha`) and the workspace start commit - A `read` event may have no `path`: a refused Claude Code Bash read of a relative path after an earlier call moved the shell, whose file is not known (`{"kind": "read", "denied": true}`). +- A `probe` event follows the `command` event of a command that asks the host + for its facts, one per pattern it matches (a command may give several): + `{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}`. + `target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`; + `pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the + command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`, + `/proc/cpuinfo`, `nproc`, `sysctl -a | grep machdep.cpu`, `Get-CimInstance Win32_Processor`, + `py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`, + `py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`; + `memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`, + `conda list`; `env` `env`, `printenv`. A pattern is matched on the program the shell or + PowerShell runs, with the words it names in any order and case (`grep nproc README` runs + `grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a + path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A + `$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched: + a Python heredoc, a script the agent runs from a file, and a pattern with a `|` that is + not a pipeline of the same commands. + A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE` + (YAML or JSON, the same mapping) replaces them again; nothing else changes the events, + and the `process_metrics` counters do not count probes. +- A `choice` event follows the `command` or `edit` event of a call that installs, imports, uses or removes a + candidate the run names: `{"t": 5.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": null}` + (`repo run --candidates FILE`, YAML or JSON, `dokimi/repos/choices.py`). The file maps a candidate id to + `install` and `remove` (package names a package manager is given; a removal matches both lists), `import` + (modules an added `import` or `from` line names) and `use` (dotted names an added line mentions, textually: a comment counts too). Only added text + counts: the new side of an `Edit`, the content of a `Write`, never what the edit replaced. A harness whose edit + events carry no added text (Copilot's `edit`, `create`, `str_replace_editor`, `write` and `apply_patch`, Codex's + `file_change`) gives install and remove events only: no import or use choice there does not mean the agent did not + choose the candidate. + A choice needs the call's own exit code 0: a failed or denied call, or one with no result, gives none. + An install or removal gives a choice only when its command joins every part to the next with `&&` and has no pipe: + the harness reports one exit code per command, so a `;`, `||` or `|` part could have failed or never run. A + `--dry-run` installs nothing and gives none. An install in a `bash -c` wrapper, or whose first word carries a + parenthesis (`(pip install X)`), gives none. + Choice events are written to `events.jsonl`; the counters and `first_useful` do not read them. `result.json` reports the agent's tokens (`input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`. @@ -2293,8 +2334,9 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric `id` hashing the profile's name, `invoke`, tool arguments, `env_set`, `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress`, `bridge`, `trace` (unless `claude-stream-json`), `answer_format`, the SHA-256 of the harness executable (`--bin`), - the SHA-256 of each `bin_companions` file and harness version. - A rerun whose `id` differs (another `--tools`, a `--delegation` that changes the tool arguments, profile, + the SHA-256 of each `bin_companions` file and harness version. A `repo run` with `--candidates` or `--probes` adds the + digest of each parsed file (the same mapping, not its bytes), so another file with other content is a different `id`. + A rerun whose `id` differs (another `--tools`, another `--candidates` or `--probes` file, a `--delegation` that changes the tool arguments, profile, harness binary or harness version, or an edited `invoke`, `env_set`, `env_allow`, `config_dir_env`, `config_files`, `prompt_suffix`, `egress`, `bridge`, `trace` or `answer_format`) is refused before any agent starts, and `--force` or another `--batch` runs it. `--force` replaces the cells of the run's plan only: if the batch dir holds finished cells outside that plan diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 7c3a04035..b5ffe0ea3 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -320,6 +320,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.bases import check_mcp_configs from dokimi.repos.container import docker_network from dokimi.repos.harness_profile import load_profile + from dokimi.repos.choices import load_candidate_file + from dokimi.repos.probes import load_probe_file from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) from dokimi.repos.selfcheck import ANCESTOR_NAMES, isolation_probe @@ -332,6 +334,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock try: parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now profile = load_profile(args.harness) + probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start + candidates = load_candidate_file(args.candidates) if args.candidates else None profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now harness_bin = _harness_bin(args.bin or profile.bin) arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)), @@ -345,7 +349,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, - max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] + max_turns=args.max_turns, budget_usd=args.budget, probes=probes, + candidates=candidates) for m in args.model] for task in tasks: # refuse before anything runs, not cell by cell task_caps(task, options[0]) agent_network(options[0], ctx, task) @@ -548,6 +553,12 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str help="Seconds per agent run unless task.json sets timeout_s (default 2700)") run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns", help="Unless task.json sets max_turns (default 200)") + run_p.add_argument("--probes", default=None, metavar="FILE", + help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the " + "framework's list; contract §4.1)") + run_p.add_argument("--candidates", default=None, metavar="FILE", + help="YAML or JSON: candidate id -> install/import/use/remove names; adds choice events " + "(contract §4.1)") run_p.add_argument("--budget", type=_usd, default=2.0, help="USD cap per agent run unless task.json sets budget_usd (default 2.0)") run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", diff --git a/dokimi/dokimi/repos/batch_ledger.py b/dokimi/dokimi/repos/batch_ledger.py index 6699c163b..df35f700a 100644 --- a/dokimi/dokimi/repos/batch_ledger.py +++ b/dokimi/dokimi/repos/batch_ledger.py @@ -72,7 +72,7 @@ def harness_sha256(path: Path) -> str: def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native", harness_sha256: str | None = None, harness_version: str | None = None, companion_sha256: dict[str, str | None] | None = None, - overlay: str | None = None) -> str: + overlay: str | None = None, inputs: dict[str, Any] | None = None) -> str: """Short hash of how the agent is started: the profile's ``invoke`` template with its tool arguments, environment, config, prompt suffix, egress, bridge, trace, answer format and companions, and for a cell of the overlay arm the overlay it keeps (``overlay``; the other cells carry none, so their ids do not change). @@ -100,15 +100,20 @@ def execution_id(profile: HarnessProfile, tools: str, delegation: str = "native" else {n: companion_sha256.get(n) for n in profile.bin_companions}) if overlay is not None: key["overlay"] = overlay + given = {n: v for n, v in sorted((inputs or {}).items()) if v is not None} # unset adds no key, as overlay does + if given: # the run's own files (--candidates, --probes): a resume with other ones is another execution + key["inputs"] = {n: hashlib.sha256(json.dumps(v, sort_keys=True, default=list).encode("utf-8")).hexdigest() + for n, v in given.items()} return hashlib.sha256(json.dumps(key, sort_keys=True).encode("utf-8")).hexdigest()[:12] def execution_record(profile: HarnessProfile, tools: str, delegation: str, harness_sha256: str, harness_version: str | None, companion_sha256: dict[str, str | None] | None = None, - overlay: str | None = None) -> dict[str, str]: + overlay: str | None = None, inputs: dict[str, Any] | None = None) -> dict[str, str]: """How a cell's agent was started, as ``meta.json`` and ``result.json`` record it.""" - return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256, overlay), + return {"id": execution_id(profile, tools, delegation, harness_sha256, harness_version, companion_sha256, overlay, + inputs), "tools": tools, "delegation": profile.effective_delegation(tools, delegation)} diff --git a/dokimi/dokimi/repos/choices.py b/dokimi/dokimi/repos/choices.py new file mode 100644 index 000000000..acf36f4c6 --- /dev/null +++ b/dokimi/dokimi/repos/choices.py @@ -0,0 +1,256 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Choice events: an agent installs, imports, uses or removes a candidate the caller names (contract §4.1). + + {"t": 4.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": null} + +A candidate is ``{id: {act: [names]}}``. ``install`` and ``remove`` name packages a package manager is given (``pip +install``, ``uv pip uninstall``, ``conda remove``); a removal matches the names of both lists. ``import`` names modules +an added ``import``/``from`` line names; +``use`` names dotted names an added line mentions (a comment too: the match is textual). Only added text counts: the new side of an edit, never what it +replaced. The events go to ``events.jsonl`` beside the trace's own events; the counters do not read them. +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import yaml + +from .probes import command_stages, command_words, split_redirects +from .shell_split import split_script + +ACTS = ("install", "import", "use", "remove") +_PACKAGE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_INSTALL_VERBS = frozenset({"install", "add"}) +_REMOVE_VERBS = frozenset({"uninstall", "remove"}) +_PYTHON = re.compile(r"python\d*(?:\.\d+)?") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_REQUIREMENT_END = re.compile(r"[<>=!~\[;@]") # where a requirement's name ends: a version, extras, a URL +_UV_VALUE_OPTIONS = frozenset({"--directory", "--project", "--config-file", "--cache-dir", "--python", "-p", "--color"}) +_INFO_FLAGS = frozenset({"-h", "--help", "-V", "--version"}) # the manager prints text and exits 0, changing nothing +_VALUE_OPTIONS = frozenset({ + "-r", "--requirement", "-c", "--constraint", "-e", "--editable", "-t", "--target", "-i", "--index-url", + "--extra-index-url", "-f", "--find-links", "--prefix", "--root", "--python", "--name", "-n", "-p", "--platform", + "--python-version", "--abi", "--implementation", "--src", "--cache-dir", "--log", "--proxy", "--retries", + "--timeout", "--cert", "--client-cert", "--trusted-host", "--config-settings", "-C", "--channel", "--group", + "--with", "--report", "--upgrade-strategy", "--progress-bar", "--keyring-provider", "--no-binary", "--only-binary", + "--global-option", "--install-option", "--exists-action", "--use-feature"}) # options that take the next word, which is not a package + + +def validate_candidates(raw: Any, where: str = "--candidates") -> dict[str, dict[str, tuple[str, ...]]]: + """A ``{id: {act: [names]}}`` mapping as ``{id: {act: (names,)}}``; ``ValueError`` names what is wrong.""" + if raw is None or raw == {}: + raise ValueError(f"{where}: the file names no candidate") + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of candidate to acts, got {type(raw).__name__}") + config: dict[str, dict[str, tuple[str, ...]]] = {} + for cid, acts in raw.items(): + if not isinstance(cid, str) or not cid.strip(): + raise ValueError(f"{where}: a candidate id must be a non-empty string") + if not isinstance(acts, Mapping) or not acts: + raise ValueError(f"{where}: {cid} must map some of {', '.join(ACTS)} to lists of names") + config[cid] = {act: _names(where, cid, act, names) for act, names in acts.items()} + return config + + +def _names(where: str, cid: str, act: str, names: Any) -> tuple[str, ...]: + if act not in ACTS: + raise ValueError(f"{where}: {cid}: unknown act {act!r} (known: {', '.join(ACTS)})") + if not isinstance(names, list) or not all(isinstance(n, str) and n.strip() for n in names): + raise ValueError(f"{where}: {cid}.{act} must be a list of non-empty strings") + shape = _PACKAGE if act in ("install", "remove") else _DOTTED + bad = [n for n in names if not shape.fullmatch(n.strip())] + if bad: + kind = "a package name" if act in ("install", "remove") else "a dotted Python name" + raise ValueError(f"{where}: {cid}.{act} takes {kind}: {bad[0]!r}") + return tuple(n.strip() for n in names) + + +def load_candidate_file(path: str) -> dict[str, dict[str, tuple[str, ...]]]: + """A run's ``--candidates`` file (YAML, or JSON, which is YAML) through :func:`validate_candidates`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"--candidates {path}: {exc}") from exc + return validate_candidates(raw, f"--candidates {path}") + + +def with_choices(events: list[dict[str, Any]], candidates: Mapping[str, Mapping[str, Sequence[str]]], + added: Sequence[tuple[dict[str, Any], str]]) -> list[dict[str, Any]]: + """``events`` with the ``choice`` events each command or edit gives, after it. ``added`` pairs an edit event with + the text it adds (``Transcript.added_text``); an edit without one gives no import or use.""" + if not candidates: + return events + texts = {id(event): text for event, text in added} + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if not _ran(event): + continue + if event.get("kind") == "command": + out.extend(_command_choices(event, candidates)) + elif event.get("kind") == "edit" and id(event) in texts: + out.extend(_text_choices(event, texts[id(event)], candidates)) + return out + + +def _ran(event: dict[str, Any]) -> bool: + """Whether the call took effect: a denied one, one that failed, or one with no exit code (rc None: a cut-off call, + or a shell command whose result carries none) did not install, import, use or remove anything.""" + return not event.get("denied") and event.get("rc") == 0 + + +def choice_timeline(events: Sequence[dict[str, Any]]) -> list[tuple[float | None, str, str]]: + """The ``(t, candidate, act)`` of each choice event, in the order the trace gives them.""" + return [(e.get("t"), str(e["candidate"]), str(e["act"])) for e in events if e.get("kind") == "choice"] + + +def _command_choices(event: dict[str, Any], candidates: Mapping[str, Mapping[str, Sequence[str]]]) -> list[dict]: + """An install or removal counts only where the command's exit code is its own: every part joined to the next by + ``&&`` (rc 0 then means each ran), and not in a pipeline (the last stage's code). A ``;``, ``||`` or ``&`` part may + have failed or never run, so its install gives no choice.""" + parts = split_script(str(event.get("text", ""))) + if any(p.after not in ("", "&&") for p in parts[:-1]) or any(p.after not in ("", "&&", ";") for p in parts[-1:]): + return [] + found: dict[tuple[str, str], None] = {} + for part, *_ in parts: + if part.startswith("("): # a subshell: its exit code is not the command's own + continue + stages, _inner = command_stages(part) + if len(stages) > 1: # a pipeline reports its last stage's exit code + continue + words = split_redirects(stages[0] if stages else [])[0] # ``exec >log`` has no program, ``> scipy`` no operand + views = list(command_words(words)) + if any(view[:1] == ["exit"] for view in views): # no later part runs + break + call = next(filter(None, map(_package_call, views)), None) + if call is not None: + act, packages = call + for cid, acts in candidates.items(): + if _wanted(acts, act) & set(packages): + found[(cid, act)] = None + if any(view[:1] == ["exec"] for view in views[:-1]): # exec with a program replaces the shell; ``exec -l`` has none + break + return [_choice(event, cid, act, None) for cid, act in found] + + +def _text_choices(event: dict[str, Any], text: str, + candidates: Mapping[str, Mapping[str, Sequence[str]]]) -> list[dict]: + out: list[dict] = [] + for cid, acts in candidates.items(): + if any(_imports(name, text) for name in acts.get("import", ())): + out.append(_choice(event, cid, "import", event.get("path"))) + if any(_calls(name, text) for name in acts.get("use", ())): + out.append(_choice(event, cid, "use", event.get("path"))) + return out + + +def _wanted(acts: Mapping[str, Sequence[str]], act: str) -> set[str]: + """The normalised package names an install or a removal of the candidate names; a removal also takes ``install``.""" + names = [*acts.get("install", ()), *acts.get("remove", ())] if act == "remove" else list(acts.get("install", ())) + return {_package(n) for n in names} + + +def _choice(event: dict[str, Any], cid: str, act: str, path: str | None) -> dict[str, Any]: + out: dict[str, Any] = {"t": event.get("t"), "kind": "choice", "candidate": cid, "act": act, "path": path} + if "agent" in event: # a subagent's choice names its agent, as its probe and command events do + out["agent"] = event["agent"] + return out + + +def _imports(name: str, text: str) -> bool: + """``from name import ...``, or ``import name`` among the comma-separated modules of an ``import`` line; a + statement may follow another on its line after a ``;`` or a ``:``, a ``\\`` continues a line, a comment is no code.""" + text = re.sub(r"\\\n[ \t]*", " ", _code(text)) + if re.search(rf"(?:^|[;:])[ \t]*from\s+{re.escape(name)}(?!\w)", text, re.M): + return True + for line in re.finditer(r"(?:^|[;:])[ \t]*import\s+([^\n;]+)", text, re.M): + for part in line.group(1).split(","): + module = re.split(r"\s+as\s+", part.strip())[0] + if module == name or module.startswith(name + "."): + return True + return False + + +def _code(text: str) -> str: + """The text without its comments and the insides of its string literals (a quote left open runs to its line's end, + or to the end of a triple-quoted one: an edit may add only part of a file).""" + out: list[str] = [] + i, n = 0, len(text) + while i < n: + c = text[i] + if c == "#": + while i < n and text[i] != "\n": + i += 1 + elif c in "'\"": + quote = text[i:i + 3] if text[i:i + 3] == c * 3 else c + i += len(quote) + while i < n and not text.startswith(quote, i) and not (len(quote) == 1 and text[i] == "\n"): + i += 2 if text[i] == "\\" else 1 + i += len(quote) if text.startswith(quote, i) else 0 + out.append(" ") + else: + out.append(c) + i += 1 + return "".join(out) + + +def _calls(name: str, text: str) -> bool: + return re.search(rf"(? tuple[str, list[str]] | None: + """The act and the package names of a package manager's install, add, uninstall or remove call.""" + if {"--dry-run", "--download-only"} & set(words) or _INFO_FLAGS & set(words): # no install: a dry run, a download, a help request + return None + rest = _manager_rest(words) + while rest and rest[0].startswith("-"): # ``pip --quiet install``: the options before the verb + rest = rest[2:] if rest[0] in _VALUE_OPTIONS else rest[1:] + if not rest: + return None + verb = rest[0].casefold() + act = "install" if verb in _INSTALL_VERBS else "remove" if verb in _REMOVE_VERBS else None + if act is None: + return None + names: list[str] = [] + skip = False + for word in split_redirects(rest[1:])[0]: # ``> scipy`` names an output file, not a package + if skip: + skip = False + elif word.startswith("-"): + skip = word in _VALUE_OPTIONS + elif _package(word): + names.append(_package(word)) + return act, names + + +def _manager_rest(words: list[str]) -> list[str]: + """The words after the package manager's program (``pip``, ``python -m pip``, ``uv pip``, ``conda``...).""" + if not words: + return [] + head = posixpath.basename(words[0].replace("\\", "/")).casefold().removesuffix(".exe") + if _PYTHON.fullmatch(head) and words[1:3] == ["-m", "pip"]: + return words[3:] + if head == "uv": + rest = words[1:] + while rest and rest[0].startswith("-"): # ``uv --quiet add``, ``uv --directory /tmp/p add`` + rest = rest[2:] if rest[0] in _UV_VALUE_OPTIONS else rest[1:] + if rest[:1] == ["pip"]: + return rest[1:] + return rest if rest[:1] in (["add"], ["remove"]) else [] # uv's project verbs: ``uv add`` installs a requirement + if _PIP.fullmatch(head) or head in ("conda", "mamba", "poetry"): + return words[1:] + return [] + + +def _package(word: str) -> str: + """A requirement's name as the package manager compares it: no channel, version, extras or URL; case and ``-_.`` folded.""" + name = _REQUIREMENT_END.split(word.rpartition("::")[2], maxsplit=1)[0].strip() + return re.sub(r"[-_.]+", "-", name).casefold() diff --git a/dokimi/dokimi/repos/cli_traces.py b/dokimi/dokimi/repos/cli_traces.py index e59477d4f..0814576c5 100644 --- a/dokimi/dokimi/repos/cli_traces.py +++ b/dokimi/dokimi/repos/cli_traces.py @@ -771,9 +771,9 @@ def _copilot_result(out: Transcript, call: _Derived, data: dict[str, Any]) -> No continue shell = data.get("shellExecution") or {} code = shell.get("exitCode") if isinstance(shell, dict) else None - event["rc"] = code if isinstance(code, int) else (None if data.get("success") is None - else 0 if data.get("success") else 1) - failed = _failed_read(str(event["text"]), event["rc"]) + event["rc"] = code if isinstance(code, int) else None # the code the command returned, or none: no success fallback + failed = _failed_read(str(event["text"]), event["rc"] if event["rc"] is not None or data.get("success") is not False + else 1) # a read still counts a result reported as failed, as before result = data.get("result") or {} output = result.get("content") if isinstance(result, dict) else None if isinstance(output, str) and output: diff --git a/dokimi/dokimi/repos/events.py b/dokimi/dokimi/repos/events.py index 956171d08..d336bd16f 100644 --- a/dokimi/dokimi/repos/events.py +++ b/dokimi/dokimi/repos/events.py @@ -10,6 +10,11 @@ {"t": 5.0, "kind": "edit", "path": "src/foo.cpp"} {"t": 20.2, "kind": "tool", "name": "WebFetch", "denied": true} {"t": 2.0, "kind": "tool", "name": "Skill", "skill": "lint-fix"} + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per +pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1). A ``choice`` event follows the ``command`` or +``edit`` event of an install, import, use or removal of a named candidate (:mod:`dokimi.repos.choices`). ``t`` is seconds since the first timestamped trace line, or ``null`` when the format has no timestamps. Paths inside the workspace are made relative to it. @@ -235,6 +240,15 @@ def _deny_read(read: dict[str, Any], unknown: bool) -> None: del read["path"] +def _added_text(args: Any) -> str: + """The text an edit tool adds: the new side of each replacement, or the whole file of a write (choices.py).""" + if not isinstance(args, dict): + return "" + if isinstance(args.get("edits"), list): # MultiEdit + return "\n".join(str(e.get("new_string", "")) for e in args["edits"] if isinstance(e, dict)) + return str(args.get("new_string") or args.get("content") or args.get("new_source") or "") + + def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: str, stream: _Stream) -> None: """The event of one tool_use block; a Bash call's derived reads come before it when the trace ends (:func:`_walk_bash`).""" event = _tool_event(block, t, stream.workdir) @@ -244,6 +258,8 @@ def _tool_use(out: Transcript, block: dict[str, Any], t: float | None, agent: st stream.bash[id(event)] = _BashCall(str(event["text"]), event) if block.get("name") in _AGENT_TOOLS: out.subagent_calls = (out.subagent_calls or 0) + 1 + if block.get("name") in _EDIT_TOOLS: + out.added_text.append((event, _added_text(block.get("input")))) out.events.append(event) if block.get("id"): stream.by_id[block["id"]] = event diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 9dc481491..884bdd504 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -35,6 +35,7 @@ from .graders.hidden_tests import HIDDEN_DIR, hidden_files from .host_spec import HostSpecError, parse_host from .process_metrics import Counter, parse_counter +from .probes import validate_config from .source import RepoTree, SourceError from .useful_spec import UsefulSpecError, parse_useful @@ -137,6 +138,7 @@ class RepoManifest: explore_tasks: str | None = None build_systems: tuple[str, ...] = () process_metrics: dict[str, Counter] = field(default_factory=dict) + probes: dict[str, tuple[str, ...]] = field(default_factory=dict) # target -> patterns (repos/probes.py) @property def runnable(self) -> bool: @@ -175,6 +177,7 @@ def to_dict(self) -> dict[str, Any]: "build_systems": list(self.build_systems), "process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field} for name, c in self.process_metrics.items()}, + **({"probes": {t: list(p) for t, p in self.probes.items()}} if self.probes else {}), } @@ -291,6 +294,7 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: if duplicates: errors.append(f"image/caches: duplicate cache name(s) {duplicates}") counters = _parse_counters(raw.get("process_metrics") or {}, errors) + probes = _parse_probes(raw.get("probes"), errors) if errors: raise ManifestError(path, errors) @@ -312,9 +316,19 @@ def parse_manifest(raw: Any, path: str = "") -> RepoManifest: explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None, build_systems=tuple(raw.get("build_systems") or ()), process_metrics=counters, + probes=probes, ) +def _parse_probes(raw: Any, errors: list[str]) -> dict[str, tuple[str, ...]]: + """The manifest's ``probes`` (repos/probes.py); a bad one is appended to ``errors``.""" + try: + return validate_config(raw, "probes") + except ValueError as exc: + errors.append(str(exc)) + return {} + + def _test_entry_problems(key: str, found: set[str]) -> list[str]: """``{tests}`` belongs to ``entry_points.test`` alone, which must name it and ``{out}``, its report.""" if key != _TEST_ENTRY_POINT: diff --git a/dokimi/dokimi/repos/probes.py b/dokimi/dokimi/repos/probes.py new file mode 100644 index 000000000..b611327a6 --- /dev/null +++ b/dokimi/dokimi/repos/probes.py @@ -0,0 +1,476 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events: a command that asks the host for its facts gets a ``probe`` event beside its ``command`` event. + + {"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0} + +One event per pattern the command matches, right after the command event, so one command may give both. A +pattern is matched on the commands the shell runs (the words of each simple command, its pipeline stages, the +code of ``python -c`` and of ``$( )`` substitutions), never on the text of an argument: ``grep nproc README`` +runs ``grep``, not ``nproc``. The pattern syntax, in a manifest's ``probes`` or a run's ``--probes`` file: + +- ``nproc``, ``pip list``, ``Get-CimInstance Win32_Processor``: a program, with the words that follow it (in + any order, any case); ``sysctl -a | grep machdep.cpu``: consecutive pipeline stages; +- ``/proc/cpuinfo``: a file a reading command names (``cat``, ``grep``, ``head``, ...); +- ``py:os.cpu_count``: a call of that name in the code of a ``python -c`` one-liner. + +A target listed in a manifest replaces that target's defaults, and a run's ``--probes`` replaces them again; +an empty list turns the target off. See ``docs/repo-evaluation-contract.md`` §2 and §4.1. +""" + +from __future__ import annotations + +import ast +import posixpath +import re +import shlex +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from .shell_split import split_script, substitution_end + +TARGETS = ("cpu", "gpu", "os", "memory", "packages", "env", "network") + +DEFAULT_PROBES: dict[str, tuple[str, ...]] = { + "cpu": ("lscpu", "/proc/cpuinfo", "nproc", "sysctl -a | grep machdep.cpu", "Get-CimInstance Win32_Processor", + "py:os.cpu_count", "py:platform.processor"), + "gpu": ("nvidia-smi", "rocm-smi", "py:torch.cuda.is_available"), + "os": ("uname", "/etc/os-release", "sw_vers", "systeminfo"), + "memory": ("free", "/proc/meminfo"), + "packages": ("pip list", "pip show", "pip freeze", "conda list"), + "env": ("env", "printenv"), + "network": (), +} + +_READERS = frozenset({"cat", "head", "tail", "less", "more", "nl", "sed", "grep", "egrep", "fgrep", "awk", "tac", + "strings", "wc", "rg"}) +_WRAPPERS = frozenset({"time", "nohup", "sudo", "command", "exec", "builtin", "nice", "env"}) # each runs the command after it +_INFO_OPTIONS = frozenset({"--help", "--version", "-V"}) # a wrapper given one prints text and runs nothing +_WRAPPER_VALUES = {"sudo": frozenset({"-u", "-g", "-h", "-p", "-r", "-t", "-C", "-D", "-R", "-T", "-U"}), + "nice": frozenset({"-n", "--adjustment"}), "exec": frozenset({"-a"}), + "time": frozenset({"-f", "-o", "--format", "--output"}), + "env": frozenset({"-u", "-C", "--unset", "--chdir"})} # options that take the next word +_PATTERN_READERS = frozenset({"grep", "egrep", "fgrep", "rg", "sed", "awk"}) # the first operand is a pattern or script +_GREP_VALUES = {"-e", "-f", "-m", "-A", "-B", "-C", "--regexp", "--file", "--max-count", "--after-context", + "--before-context", "--context"} +_READER_VALUES = {"head": {"-n", "-c", "--lines", "--bytes"}, "tail": {"-n", "-c", "--lines", "--bytes"}, + "grep": _GREP_VALUES, "egrep": _GREP_VALUES, "fgrep": _GREP_VALUES, + "rg": _GREP_VALUES | {"-g", "-t", "-T", "--glob", "--type", "--type-not"}, + "sed": {"-e", "-f", "-l", "--expression", "--file", "--line-length"}, + "awk": {"-F", "-v", "-f", "--file", "--assign", "--field-separator"}} # options that take the next word +_PATTERN_OPTIONS = frozenset({"-e", "-f", "--regexp", "--file", "--expression"}) # the pattern, script or a file of it comes from the option, not the first operand +_REDIRECT = re.compile(r"[<>&]*[<>][<>&]*") +_OPERATOR = "\0" # marks a word that is a shell operator, as against a quoted ``'>'`` +_PYTHON = re.compile(r"python\d*(?:\.\d+)?|py|pypy\d*") +_PIP = re.compile(r"pip\d*(?:\.\d+)?") +_ASSIGN = re.compile(r"[A-Za-z_]\w*=") +_DOTTED = re.compile(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*") +_HEREDOC = re.compile(r"<<-?[ \t]*(['\"]?)([A-Za-z_]\w*)\1") +_SHELL_LINE = re.compile(r"\s*(?:(?:sudo|exec|command|time)\s+)*(?:bash|sh|zsh|dash)\b") # a heredoc fed to a shell runs +_PUNCT = frozenset("();<>&|") +_SHELLS = frozenset({"bash", "sh", "zsh", "dash"}) +_SHELL_C = re.compile(r"-[a-z]*c[a-z]*") # ``-c``, ``-lc``: the next word is a script +_LEX_SKIP = frozenset({"(", ")", "{", "}"}) # grouping a PowerShell or shell expression puts around a command +_SEPARATORS = frozenset({"|", "||", "&&", ";", "&"}) # ``|`` splits stages; the rest end a command +_GROUPED = re.compile(r"[{}()]|\b(?:if|then|else|elif|fi|do|done|while|until|case|esac)\b") # a group or a block + +Stage = tuple[str, tuple[str, ...]] # a simple command's program (normalised) and its words + + +@dataclass(frozen=True) +class Probe: + """One compiled pattern. ``kind``: ``command`` (``stages``, consecutive pipeline stages), ``path`` (``text``, the + file's path) or ``python`` (``call``, the dotted name a call in the code of a ``-c`` one-liner must have).""" + + target: str + text: str + kind: str + stages: tuple[Stage, ...] = () + call: str | None = None + + +def validate_config(raw: Any, where: str) -> dict[str, tuple[str, ...]]: + """A ``probes`` mapping (a manifest's or a run file's) as target -> patterns; ``ValueError`` names what is wrong.""" + if raw is None: + return {} + if not isinstance(raw, Mapping): + raise ValueError(f"{where}: expected a mapping of target to patterns, got {type(raw).__name__}") + config: dict[str, tuple[str, ...]] = {} + for target, patterns in raw.items(): + if target not in TARGETS: + raise ValueError(f"{where}: unknown target {target!r} (known: {', '.join(TARGETS)})") + if not isinstance(patterns, list) or not all(isinstance(p, str) and p.strip() for p in patterns): + raise ValueError(f"{where}: {target} must be a list of non-empty pattern strings") + for pattern in patterns: + _compile(target, pattern.strip()) # a pattern that cannot be read is refused here, not per cell + config[target] = tuple(p.strip() for p in patterns) + return config + + +def load_probe_file(path: str) -> dict[str, tuple[str, ...]]: + """A run's ``--probes`` file (YAML, or JSON, which is YAML) through :func:`validate_config`.""" + try: + raw = yaml.safe_load(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"--probes {path}: {exc}") from exc + return validate_config(raw, f"--probes {path}") + + +def probes_for(*layers: Mapping[str, Sequence[str]] | None) -> tuple[Probe, ...]: + """The compiled patterns of the defaults with each layer applied in turn: a target a layer names is replaced.""" + config: dict[str, Sequence[str]] = dict(DEFAULT_PROBES) + for layer in layers: + config.update(layer or {}) + return tuple(_compile(target, text.strip()) for target in TARGETS for text in config.get(target, ())) + + +def with_probes(events: list[dict[str, Any]], probes: Sequence[Probe]) -> list[dict[str, Any]]: + """``events`` with a ``probe`` event after each ``command`` event whose command matches a pattern.""" + if not probes: + return events + out: list[dict[str, Any]] = [] + for event in events: + out.append(event) + if event.get("kind") == "command": + if not event.get("denied"): # a denied command ran nothing + out.extend(_probe_event(event, probe) for probe in _matching(str(event.get("text", "")), probes)) + return out + + +def _probe_event(command: dict[str, Any], probe: Probe) -> dict[str, Any]: + event: dict[str, Any] = {"t": command.get("t"), "kind": "probe", "target": probe.target, "pattern": probe.text, + "text": command.get("text", ""), "rc": command.get("rc")} + if "agent" in command: + event["agent"] = command["agent"] + return event + + +def _compile(target: str, text: str) -> Probe: + if target not in TARGETS: + raise ValueError(f"unknown probe target {target!r}") + if text.startswith("py:"): + name = text[3:].strip() + if not _DOTTED.fullmatch(name): + raise ValueError(f"{text!r}: py: takes a dotted Python name such as os.cpu_count") + return Probe(target, text, "python", call=name) + if text.startswith("/"): + if any(c.isspace() for c in text): + raise ValueError(f"{text!r}: a path pattern is one word with no spaces") + return Probe(target, text, "path") + return Probe(target, text, "command", stages=tuple(_stage(part, text) for part in text.split("|"))) + + +def _stage(part: str, text: str) -> Stage: + if any(c in part for c in ";&<>"): + raise ValueError(f"{text!r}: separate commands and redirections are not part of a pattern; use '|' for stages") + try: + words = [w for w in shlex.split(part) if w not in _LEX_SKIP] + except ValueError as exc: + raise ValueError(f"{text!r}: {exc}") from exc + if not words: + raise ValueError(f"{text!r}: a pattern names a program") + return _program(words[0]), tuple(words[1:]) + + +def _program(word: str) -> str: + """A program's name as the shell or PowerShell finds it: no directory, no ``.exe``, no case.""" + name = posixpath.basename(word.replace("\\", "/")).casefold() + return name[:-4] if name.endswith(".exe") else name + + +def _matching(command: str, probes: Sequence[Probe]) -> list[Probe]: + """The probes the command runs, each once, in the order they are configured.""" + pipelines = [[list(_views(words)) for words in pipe] for pipe in _pipelines(command)] + return [p for p in dict.fromkeys(probes) if any(_hits(p, pipe) for pipe in pipelines)] + + +def _pipelines(script: str) -> Iterator[list[list[str]]]: + """Each pipeline of the script as its stages' words; the commands of its ``$( )`` and backtick substitutions too.""" + status: bool | None = None # the exit status of the last part that ran; None when it is not known + before = "" # the operator before this part + text = _without_heredocs(script) + flat = not _GROUPED.search(text) # a group or a block may hold an exit that does not run: then nothing ends the scan + for part, after, in_subshell, _outer in split_script(text): + conditional = before in ("||", "&&") # the part runs only if the one before it did, or did not, succeed + skipped, before = (before == "||" and status is True) or (before == "&&" and status is False), after + if skipped: # a part after ``||`` that succeeded, or after ``&&`` that failed, never runs + continue + status = {"true": True, ":": True, "false": False}.get(part.strip()) + stages, inner = command_stages(part) + for body in inner: + yield from _pipelines(body) + if stages: + yield stages + for stage in stages: # ``sudo bash -lc 'nproc'``: the script a shell runs is a script of its own + for rest in command_words(stage): + if len(rest) > 2 and _program(rest[0]) in _SHELLS and _SHELL_C.fullmatch(rest[1]): + yield from _pipelines(rest[2]) + if flat and not conditional and not in_subshell and any(_ends_shell(stage) for stage in stages): + return # what follows an ``exit`` or a replacing ``exec`` that surely runs never runs + + +def _ends_shell(stage: list[str]) -> bool: + """Whether a simple command ends the shell: ``exit``, or ``exec`` with a program (``exec -l`` has none).""" + views = list(command_words(split_redirects(stage)[0])) + return any(_program(view[0]) == "exit" for view in views) or any(_program(view[0]) == "exec" for view in views[:-1]) + + +def _without_heredocs(script: str) -> str: + """The script without the bodies of its heredocs, which are data (``cat < run.sh``) unless a shell reads them; + the ``$( )`` and backtick substitutions of an unquoted body ran, so they stay.""" + if "<<" not in script: + return script + out: list[str] = [] + end: str | None = None + body: list[str] = [] + expands = False + head, at = 0, 0 # the line that opens the heredoc, and where its operator starts on it + strip_tabs = False # ``<<-``: the terminator may be indented with tabs, and only those + for line in script.split("\n"): + if end is not None: + if (line.lstrip("\t") if strip_tabs else line) == end: + if expands: # before the operator: a comment after it on the opening line would hide them + found = "".join(f"{text} " for text in _expansions("\n".join(body))) + out[head] = out[head][:at] + found + out[head][at:] + end, body = None, [] + else: + body.append(line) + continue + out.append(line) + m = _opener(line) + if m and not _SHELL_LINE.match(line): + end, expands, head, at = m.group(2), not m.group(1), len(out) - 1, m.start() + strip_tabs = m.group(0).startswith("<<-") + return "\n".join(out) + + +def _opener(line: str) -> re.Match[str] | None: + """The first heredoc operator of a line that is not inside quotes (``echo "< list[str]: + """The ``$( )`` and backtick substitutions of an unquoted heredoc body, as text.""" + found: list[str] = [] + i = 0 + while i < len(body): + if body[i] == "\\": + i += 1 + elif body[i] in "$`" and (sub := _substitution(body, i)): + found.append(body[i:sub[1] + 1]) + i = sub[1] + i += 1 + return found + + +def command_stages(part: str) -> tuple[list[list[str]], list[str]]: + """A part's simple commands, cut at its unquoted pipes, each as its words (quotes removed, grouping dropped), and + the bodies of the ``$( )`` and backtick substitutions outside single quotes: they ran as commands of their own.""" + stages: list[list[str]] = [[]] + bodies: list[str] = [] + word: list[str] = [] + started = False # a quoted empty string is a word + bare = True # the word so far has no quote or escape in it + quote, i = "", 0 + while i < len(part): + c = part[i] + if c == "'" and not quote: # nothing inside single quotes is special + end = part.find("'", i + 1) + end = len(part) if end < 0 else end + word.append(part[i + 1:end]) + started, bare = True, False + i = end + elif c == "\\" and part[i + 1:i + 2] == "\\": # ``\\|`` is a backslash, then a pipe, in double quotes too + word.append("\\\\") + started, bare = True, False + i += 1 + elif c == "\\" and (part[i + 1:i + 2] in ("$", "`", '"') or (not quote and part[i + 1:i + 2] in tuple("|;&<>()"))): + word.append(part[i + 1]) # an escaped ``$(`` or ``|`` is text; any other backslash is a path separator + started, bare = True, False + i += 1 + elif c in "$`" and (sub := _substitution(part, i)): + bodies.append(sub[0]) + word.append("\1") # the word's text is the substitution's output: a placeholder keeps its place + started, bare = True, False + i = sub[1] + elif c == quote: + quote = "" + elif c == '"' and not quote: + quote, started, bare = c, True, False + elif quote or c not in " \t\n" + "".join(_PUNCT): + word.append(c) + started = True + else: + if c in "<>" and bare and "".join(word).isdigit(): # ``2>file``: the 2 names a descriptor, it is no word + word.clear() + started = False + _end_word(stages, word, started) + started, bare = False, True + i = _operator(part, i, stages) + i += 1 + _end_word(stages, word, started) + return [stage for stage in stages if stage], bodies + + +def _end_word(stages: list[list[str]], word: list[str], started: bool) -> None: + if started and "".join(word) not in _LEX_SKIP: + stages[-1].append("".join(word)) + word.clear() + + +def _substitution(part: str, i: int) -> tuple[str, int] | None: + """The body of the ``$( )`` or backtick substitution at ``i`` and the index of its last character; ``None`` + where ``$`` or a backtick starts none.""" + if part[i] == "$": + if part[i + 1:i + 2] != "(" or part[i + 2:i + 3] == "(": # ``$((`` is arithmetic: it runs no command + return None + end = substitution_end(part, i + 2) + return part[i + 2:end - 1], end - 1 + end = part.find("`", i + 1) + end = len(part) if end < 0 else end + return part[i + 1:end], end + + +def _operator(part: str, i: int, stages: list[list[str]]) -> int: + """Take the run of operator characters at ``i`` (a blank is none): a pipe starts a stage, a list separator or a + grouping parenthesis is dropped, a redirection stays a word; the index of the run's last character.""" + run = part[i].strip() + if not run: + return i + while part[i + 1:i + 2] in _PUNCT: + i += 1 + run += part[i] + if "|" in run: + stages.append([]) + elif run and run not in _SEPARATORS and not set(run) <= set("()"): + stages[-1].append(_OPERATOR + run) + return i + + +def split_redirects(words: Sequence[str]) -> tuple[list[str], list[str]]: + """The words without their redirections (an operator and its target), and the targets of ``<`` redirections: the + files the command reads.""" + plain: list[str] = [] + reads: list[str] = [] + i = 0 + while i < len(words): + if words[i].startswith(_OPERATOR) and _REDIRECT.fullmatch(words[i][1:]): + if words[i] == _OPERATOR + "<" and i + 1 < len(words): + reads.append(words[i + 1]) + i += 2 + continue + plain.append(words[i]) + i += 1 + return plain, reads + + +def _views(words: list[str]) -> Iterator[Stage]: + """The command a simple command runs: its own program, then the command each wrapper (``time``, ``sudo``...) runs.""" + views = list(command_words(words)) + for k, rest in enumerate(views): + later = [v for v in views[k + 1:] if not v[0].startswith(_OPERATOR)] # a redirection is no command + if _program(rest[0]) == "env" and later: # ``env FOO=1 nproc`` runs ``nproc``, not a probe of ``env`` + continue + yield _head(rest[0], rest[1:]) + + +def command_words(words: list[str]) -> Iterator[list[str]]: + """The words of the command a simple command runs, from its program on, then those of the command each wrapper + (``time``, ``sudo -n``, ``nice -n 5``...) runs: its options are not the program.""" + i = 0 + while i < len(words): + while i < len(words) and _ASSIGN.match(words[i]): # NAME=value words set variables for the command after them + i += 1 + if i >= len(words): + return + yield words[i:] + wrapper = _program(words[i]) + if wrapper not in _WRAPPERS: + return + i += 1 + while i < len(words) and words[i].startswith("-") and len(words[i]) > 1: + if words[i] in _INFO_OPTIONS or (wrapper == "command" and words[i] == "-v"): # looks up or prints: runs nothing + return + i += 2 if words[i] in _WRAPPER_VALUES.get(wrapper, ()) else 1 + + +def _head(word: str, args: list[str]) -> Stage: + """A program and its words; ``python -m pip list`` is the program ``pip``.""" + program = _program(word) + if _PYTHON.fullmatch(program) and args[:1] == ["-m"] and len(args) > 1 and _PIP.fullmatch(args[1]): + return "pip", tuple(args[2:]) + return ("pip" if _PIP.fullmatch(program) else program), tuple(args) + + +def _hits(probe: Probe, pipeline: list[list[Stage]]) -> bool: + if probe.kind == "command": + return _in_order(probe.stages, pipeline) + if probe.kind == "path": + return any(prog in _READERS and probe.text in _operands(prog, args) for views in pipeline for prog, args in views) + return any(_PYTHON.fullmatch(prog) and _calls(args, probe) for views in pipeline for prog, args in views) + + +def _operands(program: str, args: tuple[str, ...]) -> list[str]: + """The files a reading command names: not its options, their values, or the pattern ``grep`` and ``sed`` search for.""" + values = _READER_VALUES.get(program, ()) + needs_pattern = program in _PATTERN_READERS + plain, files = split_redirects(args) + i = 0 + while i < len(plain): + word = plain[i] + if word.startswith("-") and len(word) > 1: + if needs_pattern and (word.split("=", 1)[0] if word.startswith("--") else word[:2]) in _PATTERN_OPTIONS: + needs_pattern = False + i += 2 if word in values else 1 + continue + if needs_pattern: + needs_pattern = False + else: + files.append(word) + i += 1 + return files + + +def _calls(args: tuple[str, ...], probe: Probe) -> bool: + """Whether the ``-c`` code of a Python command calls the probe's name, as code (not in a string or a comment).""" + if "-c" not in args or probe.call is None: + return False + at = args.index("-c") + if any(not w.startswith("-") for w in args[:at]): # after a script name, ``-c`` is the script's own + return False + try: + tree = ast.parse(args[at + 1]) if at + 1 < len(args) else None + except (SyntaxError, ValueError): + return False + return tree is not None and any(_dotted(node.func) == probe.call for node in ast.walk(tree) if isinstance(node, ast.Call)) + + +def _dotted(node: ast.expr) -> str: + """``os.cpu_count`` for the expression ``os.cpu_count``; ``""`` for anything that is not a dotted name.""" + parts: list[str] = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + return ".".join([node.id, *reversed(parts)]) if isinstance(node, ast.Name) else "" + + +def _in_order(want: tuple[Stage, ...], pipeline: list[list[Stage]]) -> bool: + """Whether the pattern's stages are consecutive stages of the pipeline (one stage: any stage of it).""" + n = len(want) + return any(all(_stage_runs(want[j], pipeline[i + j]) for j in range(n)) for i in range(len(pipeline) - n + 1)) + + +def _stage_runs(want: Stage, views: list[Stage]) -> bool: + """Whether one of the views is the wanted program with every wanted word among its words, any case.""" + program, words = want + return any(prog == program and all(w.casefold() in {a.casefold() for a in split_redirects(args)[0]} for w in words) + for prog, args in views) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a90455bfc..8863cd078 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -71,6 +71,8 @@ ) from .bases import MCP_CONFIG, NOT_UTF8, mcp_config_error from .events import convert +from .choices import with_choices +from .probes import probes_for, with_probes from .uptake import arm_uptake from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry @@ -99,6 +101,8 @@ class AgentOptions: timeout_s: float = 2700 max_turns: int = 200 budget_usd: float = 2.0 + probes: dict[str, tuple[str, ...]] | None = None # the run's --probes, over the manifest's (repos/probes.py) + candidates: dict[str, dict[str, tuple[str, ...]]] | None = None # the run's --candidates (repos/choices.py) def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, str]: @@ -111,8 +115,9 @@ def execution_record(opts: AgentOptions, arm: str | None = None) -> dict[str, st companions = {n: harness_sha256(beside / n) if (beside / n).is_file() else None for n in opts.profile.bin_companions} overlay = opts.profile.overlay if arm is not None and parse_arm(arm).base == OVERLAY_ARM else None + inputs = {"candidates": opts.candidates, "probes": opts.probes} return batch_ledger.execution_record(opts.profile, opts.tools, opts.delegation, harness_sha256(opts.harness_bin), - opts.harness_version, companions, overlay) + opts.harness_version, companions, overlay, inputs) def check_batch_executions(batch_dir: Path, options: list[AgentOptions], replacing: set[str] | None = None) -> None: @@ -260,7 +265,10 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, try: raw = (rd / "agent.stdout").read_text(encoding="utf-8", errors="replace") transcript = convert(opts.profile.trace, raw, WORKSPACE) - (rd / "events.jsonl").write_text(transcript.events_jsonl(), encoding="utf-8") + # Probe events go to events.jsonl only: the counters, first_useful and skill usage score the trace's events + probed = with_probes(transcript.events, probes_for(ctx.manifest.probes, opts.probes)) + shown = with_choices(probed, opts.candidates or {}, transcript.added_text) # choice events: choices.py + (rd / "events.jsonl").write_text(transcript.events_jsonl(shown), encoding="utf-8") failed = transcript.is_error or not result.ok # a crashed review is missing, not clean answer = convert_answer(opts.profile.answer_format, transcript.answer, WORKSPACE, failed=failed, messages=transcript.agent_messages) diff --git a/dokimi/dokimi/repos/transcript.py b/dokimi/dokimi/repos/transcript.py index 7d9408625..f74251d94 100644 --- a/dokimi/dokimi/repos/transcript.py +++ b/dokimi/dokimi/repos/transcript.py @@ -46,6 +46,7 @@ class Transcript: dropped_lines: int = 0 # trace lines that did not parse or were of no known shape: their events are missing init_model: str | None = None # the model the harness says it starts with (``system``/``init``) answered_by: set[str] = field(default_factory=set) # ``message.model`` of the main thread's replies + added_text: list[tuple[dict[str, Any], str]] = field(default_factory=list) # edit event, the text it adds (choices.py) def model_resolved(self) -> str | list[str] | None: """The model that answered: one name, a sorted list when several did, ``None`` when the trace cannot tell. @@ -59,8 +60,8 @@ def reads(self) -> set[str]: """The paths the agent was shown (:func:`shown_read`); a refused or failed read stays in the events only.""" return {e["path"] for e in self.events if shown_read(e) and e.get("path")} - def events_jsonl(self) -> str: - return "".join(json.dumps(e, sort_keys=True) + "\n" for e in self.events) + def events_jsonl(self, events: list[dict[str, Any]] | None = None) -> str: + return "".join(json.dumps(e, sort_keys=True) + "\n" for e in (self.events if events is None else events)) def shown_read(event: dict[str, Any]) -> bool: diff --git a/dokimi/dokimi/schemas/repo_manifest.v1.json b/dokimi/dokimi/schemas/repo_manifest.v1.json index be85eb5ae..11608ffb0 100644 --- a/dokimi/dokimi/schemas/repo_manifest.v1.json +++ b/dokimi/dokimi/schemas/repo_manifest.v1.json @@ -76,6 +76,12 @@ "uniqueItems": true, "items": {"enum": ["bazel", "make", "cmake", "pytest", "pre-commit"]} }, + "probes": { + "description": "Target -> command, file or python patterns whose commands get a probe event (docs/repo-evaluation-contract.md §2, §4.1). A listed target replaces its defaults; [] turns it off.", + "type": "object", + "propertyNames": {"enum": ["cpu", "gpu", "os", "memory", "packages", "env", "network"]}, + "additionalProperties": {"type": "array", "items": {"type": "string", "minLength": 1}} + }, "process_metrics": { "type": "object", "propertyNames": {"pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..52c5282b9 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -35,6 +35,7 @@ --cache-root --calls --candidate +--candidates # repo run's candidate file (repos/choices.py): no registry concept yet; a F1b migration names it --catalog --checklist --claude # catalog trigger-probe (#217) adds it: backend choice has no registry concept yet diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 4925cfa20..72e29cc3c 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -125,6 +125,10 @@ lines.append({"type": "assistant", "message": {"content": [ {"type": "tool_use", "id": f"u{i}", "name": n, "input": a} for i, (n, a) in enumerate(json.loads(os.environ["FAKE_TOOL_USES"]))]}}) + if os.environ.get("FAKE_TOOL_RESULTS"): # a success for each of them: a choice needs its result (rc 0) + lines.append({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": f"u{i}", "is_error": False, "content": "ok"} + for i in range(len(json.loads(os.environ["FAKE_TOOL_USES"])))]}}) result = {"type": "result", "subtype": "success", "result": "done: src/calc.py", "total_cost_usd": 0.01, "num_turns": 3, "is_error": False, "usage": {"input_tokens": 5, "output_tokens": 2, "cache_read_input_tokens": 100, @@ -147,7 +151,7 @@ "--max-budget-usd", "{budget}", "{tools}"], "tools": {"none": ["--tools", ""], "read": ["--tools", "Read"], "all": []}, "config_dir_env": "FAKE_CONFIG_DIR", - "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_SKILLS", "FAKE_USAGE", "FAKE_PROCESS", "FAKE_TOOL_USES", "PATH"], + "env_allow": ["FAKE_ACTION", "FAKE_ANSWER_MODEL", "FAKE_MARKER", "FAKE_SKILLS", "FAKE_USAGE", "FAKE_PROCESS", "FAKE_TOOL_RESULTS", "FAKE_TOOL_USES", "PATH"], "egress": ["api.example.com"], "trace": "claude-stream-json", "prompt_suffix": "Stay in the foreground.", "config_files": {"settings.json": '{"hook": "python3 {support_dir}/no_background.py"}'}, diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index ac45bf111..fdfe68411 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -37,22 +37,22 @@ # The id moves whenever the fixture binary changes (its digest is in the id), e.g. when it gains a tool_result. GOLDEN_SAMPLE_CELLS = { "fix_add__guidance-none__m-1__r1": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-none__m-1__r2": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r1": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, "fix_add__guidance-raw__m-1__r2": { - "id": "ac034e664259", + "id": "d4546308cce5", "tools": "all", "delegation": "native", }, diff --git a/dokimi/tests/test_repo_choices.py b/dokimi/tests/test_repo_choices.py new file mode 100644 index 000000000..f6aaabc0a --- /dev/null +++ b/dokimi/tests/test_repo_choices.py @@ -0,0 +1,348 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Choice events (repos/choices.py, contract §4.1): an install, import, use or removal of a named candidate.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.choices import choice_timeline, validate_candidates, with_choices +from dokimi.repos.events import convert + +WS = "/ws" +SKLEARN = {"sklearn": {"install": ["scikit-learn-intelex"], "import": ["sklearnex"], "use": ["patch_sklearn"]}} +SCIPY = {"lstsq": {"use": ["scipy.linalg.lstsq"]}} + + +def _call(tool_id: str, name: str, args: dict, t: str) -> dict: + return {"type": "assistant", "timestamp": t, + "message": {"content": [{"type": "tool_use", "id": tool_id, "name": name, "input": args}]}} + + +def _result(tool_id: str, t: str) -> dict: + return {"type": "user", "timestamp": t, + "message": {"content": [{"type": "tool_result", "tool_use_id": tool_id, "is_error": False, + "content": ""}]}} + + +def _bash(tool_id: str, command: str, t: str) -> list[dict]: + return [_call(tool_id, "Bash", {"command": command}, t), _result(tool_id, t)] + + +def _edit(tool_id: str, path: str, new: str, t: str, old: str = "", name: str = "Edit") -> list[dict]: + args = {"file_path": f"{WS}/{path}", "new_string": new, "old_string": old} if name == "Edit" else { + "file_path": f"{WS}/{path}", "content": new} + return [_call(tool_id, name, args, t), _result(tool_id, t)] + + +def _run(candidates: dict, *records: list[dict] | dict) -> list[dict]: + flat = [r for rec in records for r in (rec if isinstance(rec, list) else [rec])] + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *flat)) + out = convert("claude-stream-json", raw, WS) + return with_choices(out.events, candidates, out.added_text) + + +def _choices(events: list[dict]) -> list[tuple]: + return [(e["candidate"], e["act"], e["path"]) for e in events if e["kind"] == "choice"] + + +def test_install_then_import_then_use_give_one_choice_each_in_order(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _edit("b", "app.py", "from sklearnex import patch_sklearn\npatch_sklearn()\n", + "2026-01-01T00:00:09Z")) + assert [(e["kind"], e.get("act")) for e in events] == [ + ("command", None), ("choice", "install"), ("edit", None), ("choice", "import"), ("choice", "use")] + assert events[1] == {"t": 5.0, "kind": "choice", "candidate": "sklearn", "act": "install", "path": None} + assert events[3]["path"] == "app.py" and events[4]["path"] == "app.py" + + +def test_install_then_remove_are_two_choice_events(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _bash("b", "pip uninstall scikit-learn-intelex", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("sklearn", "install", None), ("sklearn", "remove", None)] + + +def test_a_use_without_an_install_is_a_use_and_no_install(): + events = _run(SCIPY, _edit("a", "solve.py", "x = scipy.linalg.lstsq(A, b)\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("lstsq", "use", "solve.py")] + + +def test_only_the_added_side_of_an_edit_counts(): + events = _run(SKLEARN, _edit("a", "app.py", "import numpy\n", "2026-01-01T00:00:05Z", + old="from sklearnex import patch_sklearn\n")) + assert _choices(events) == [] + + +def test_a_write_counts_its_whole_content(): + events = _run(SKLEARN, _edit("a", "app.py", "import sklearnex\n", "2026-01-01T00:00:05Z", name="Write")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + +@pytest.mark.parametrize("command", [ + "pip install 'scikit-learn-intelex==2024.7'", + "uv pip install scikit_learn_intelex", + "python3 -m pip install --upgrade scikit-learn-intelex", + "pip install scikit-learn-intelex[extra]", +]) +def test_a_package_manager_install_names_the_package_whatever_the_spelling(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + "grep 'pip install scikit-learn-intelex' README", + "echo pip install scikit-learn-intelex", + "pip install numpy", +]) +def test_a_command_that_only_mentions_the_package_is_no_choice(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def test_no_candidates_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "pip install scikit-learn-intelex", "rc": 0}] + assert with_choices(events, {}, []) == events + + +def test_the_timeline_lists_each_choice_in_trace_order(): + events = _run(SKLEARN, + _bash("a", "pip install scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _edit("b", "app.py", "patch_sklearn()\n", "2026-01-01T00:00:09Z")) + assert choice_timeline(events) == [(5.0, "sklearn", "install"), (9.0, "sklearn", "use")] + + +def test_a_name_listed_only_for_removal_is_no_install(): + events = _run({"x": {"remove": ["foo"]}}, _bash("a", "pip install foo", "2026-01-01T00:00:05Z"), + _bash("b", "pip uninstall foo", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("x", "remove", None)] + + +def test_a_subagents_choice_names_its_agent(): + record = {"type": "assistant", "timestamp": "2026-01-01T00:00:05Z", "parent_tool_use_id": "task-1", + "message": {"content": [{"type": "tool_use", "id": "a", "name": "Bash", + "input": {"command": "pip install scikit-learn-intelex"}}]}} + events = _run(SKLEARN, record, _result("a", "2026-01-01T00:00:06Z")) + assert [e for e in events if e["kind"] == "choice"][0]["agent"] == "task-1" + + +def test_a_failed_install_gives_no_choice(): + failed = [_call("a", "Bash", {"command": "pip install scikit-learn-intelex"}, "2026-01-01T00:00:05Z"), + {"type": "user", "timestamp": "2026-01-01T00:00:06Z", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "a", "is_error": True, "content": "no such package"}]}}] + assert _choices(_run(SKLEARN, failed)) == [] + + +def test_a_call_with_no_result_gives_no_choice(): + unresolved = [_call("a", "Bash", {"command": "pip install scikit-learn-intelex"}, "2026-01-01T00:00:05Z"), + _call("e", "Edit", {"file_path": f"{WS}/x.py", "old_string": "", "new_string": "import sklearnex"}, + "2026-01-01T00:00:06Z")] + assert _choices(_run(SKLEARN, unresolved)) == [] + + +@pytest.mark.parametrize("rc", [None, 1]) +def test_a_command_whose_exit_code_is_not_0_gives_no_choice(rc): + assert _choices(with_choices(_bash_rc("pip install scikit-learn-intelex", rc=rc), SKLEARN, [])) == [] + + +def test_a_module_in_a_multi_module_import_line_is_an_import(): + events = _run(SKLEARN, _edit("a", "app.py", "import os, sklearnex as sk\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + +@pytest.mark.parametrize("command", ["pip --quiet install scikit-learn-intelex", "pip3.11 install scikit-learn-intelex"]) +def test_a_pip_option_or_a_versioned_pip_still_names_the_install(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +def test_uvs_project_verbs_add_and_remove_name_the_package(): + events = _run(SKLEARN, _bash("a", "uv add scikit-learn-intelex", "2026-01-01T00:00:05Z"), + _bash("b", "uv remove scikit-learn-intelex", "2026-01-01T00:00:09Z")) + assert _choices(events) == [("sklearn", "install", None), ("sklearn", "remove", None)] + + +def test_an_install_in_an_and_chain_names_the_install(): + assert _choices(_run(SKLEARN, _bash("a", "cd /ws && pip install scikit-learn-intelex", + "2026-01-01T00:00:05Z"))) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + # the exit code is the last part's: the install may have failed, or never run + "python -c 'import sklearnex' || pip install scikit-learn-intelex", + "pip install scikit-learn-intelex; echo done", + "pip install scikit-learn-intelex 2>&1 | tail -3", + "pip install scikit-learn-intelex &", + "pip install --dry-run scikit-learn-intelex", +]) +def test_an_install_whose_exit_code_is_not_its_own_gives_no_choice(command): + assert _choices(_run(SKLEARN, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def test_an_import_after_a_semicolon_on_the_same_line_is_an_import(): + events = _run(SKLEARN, _edit("a", "app.py", "import os; from sklearnex import patch_sklearn\n", + "2026-01-01T00:00:05Z")) + assert ("sklearn", "import", "app.py") in _choices(events) + events = _run(SKLEARN, _edit("a", "app.py", "import os; import sklearnex\n", "2026-01-01T00:00:05Z")) + assert _choices(events) == [("sklearn", "import", "app.py")] + + +def _bash_rc(command: str, **extra) -> list[dict]: + return [{"t": 1.0, "kind": "command", "text": command, **extra}] + + +def test_a_denied_install_gives_no_choice(): + events = with_choices(_bash_rc("pip install scikit-learn-intelex", rc=0, denied=True), SKLEARN, []) + assert _choices(events) == [] + assert _choices(with_choices(_bash_rc("pip install scikit-learn-intelex", rc=0), SKLEARN, [])) == [ + ("sklearn", "install", None)] + + +@pytest.mark.parametrize("command", [ + "pip install scikit-learn-intelex |cat", + "pip install scikit-learn-intelex|tail -3", + "pip install scikit-learn-intelex --help", + "pip install -h scikit-learn-intelex", + "pip --version install scikit-learn-intelex", + "pip uninstall scikit-learn-intelex --help", + "pip install --target scikit-learn-intelex numpy", + "pip install -r scikit-learn-intelex numpy", + "pip install -c scikit-learn-intelex numpy", + "pip install --index-url scikit-learn-intelex numpy", + "conda install --name scikit-learn-intelex numpy", + "exit 0 && pip install scikit-learn-intelex", + "pip install numpy > scikit-learn-intelex", + "pip install numpy 2>scikit-learn-intelex", + "pip uninstall numpy > scikit-learn-intelex", + "exec pip install numpy && pip install scikit-learn-intelex", + "exec true && pip install scikit-learn-intelex", + "FOO=1 exec true && pip install scikit-learn-intelex", + "echo \\| pip install scikit-learn-intelex", +]) +def test_a_command_that_installs_nothing_the_candidate_names_gives_no_choice(command): + assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [] + + +@pytest.mark.parametrize("command", [ + "sudo pip install scikit-learn-intelex", + "time pip install scikit-learn-intelex", + "sudo -n pip install scikit-learn-intelex", + "nice -n 5 python -m pip install scikit-learn-intelex", + "pip.exe install scikit-learn-intelex", + "python.exe -m pip install scikit-learn-intelex", + "pip install --target lib scikit-learn-intelex", + "echo '|' && pip install scikit-learn-intelex", + "exec pip install scikit-learn-intelex", + "uv --quiet add scikit-learn-intelex", + "pip install numpy 2>&1 > /dev/null scikit-learn-intelex", + "pip install 'scikit-learn-intelex >= 1.0'", +]) +def test_a_wrapped_or_option_laden_install_still_gives_its_choice(command): + assert _choices(with_choices(_bash_rc(command, rc=0), SKLEARN, [])) == [("sklearn", "install", None)] + + +@pytest.mark.parametrize(("text", "found"), [ + ("# example; import sklearnex\n", False), + ("# example; from sklearnex import patch_sklearn\n", False), + ("import os, \\\n sklearnex\n", True), + ("if True: import sklearnex\n", True), + ("from sklearnex.svm import SVC\n", True), + ("from sklearnexx import y\n", False), + ("import sklearnex.patches\n", True), + ("import sklearnexx\n", False), + ('s = "#"; import sklearnex\n', True), + ('"""\nimport sklearnex\n"""\n', False), + ("s = 'import sklearnex'\n", False), + ('s = "a\\"b"; import sklearnex\n', True), +]) +def test_an_import_is_read_from_code_not_from_a_comment(text, found): + event = {"t": 1.0, "kind": "edit", "path": "a.py", "rc": 0} + out = with_choices([event], SKLEARN, [(event, text)]) + assert (("sklearn", "import", "a.py") in _choices(out)) is found + + +def test_use_matches_a_dotted_name_only_at_a_name_boundary(): + event = {"t": 1.0, "kind": "edit", "path": "a.py", "rc": 0} + for text, found in (("x = scipy.linalg.lstsq(a)\n", True), ("x = myscipy.linalg.lstsq(a)\n", False), + ("x = a.scipy.linalg.lstsq(a)\n", False), ("x = scipy.linalg.lstsqx(a)\n", False)): + assert bool(_choices(with_choices([event], SCIPY, [(event, text)]))) is found, text + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"sk": {"fit": ["x"]}}, "unknown act 'fit'"), + ({"sk": {"import": ["sklearn learn"]}}, "takes a dotted Python name"), + ({"sk": {"install": ["bad name;rm"]}}, "takes a package name"), + ({"sk": {"install": "scikit-learn"}}, "must be a list of non-empty strings"), + ({"sk": []}, "must map some of"), + (["sk"], "expected a mapping"), +]) +def test_a_bad_candidate_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_candidates(raw, "candidates") + + +def test_an_exec_of_a_removal_gives_its_choice(): + assert _choices(with_choices(_bash_rc("exec pip uninstall -y scikit-learn-intelex", rc=0), SKLEARN, [])) == [ + ("sklearn", "remove", None)] + + +INSTALL_SCIPY = {"scipy": {"install": ["scipy"]}} + + +@pytest.mark.parametrize("command", [ + "exec >install.log && pip install scipy", # a commandless exec only redirects: the shell goes on + "exec && pip install scipy", + "uv --directory /tmp/project add scipy", # uv's global options take a value: it is not the project's verb + "uv --project=/tmp/project --quiet add scipy", + "pip install >$(printf /tmp/install.log) scipy", # a substitution is the redirection's target, not the package +]) +def test_a_command_the_shell_runs_before_or_around_its_redirects_gives_its_install(command): + assert _choices(_run(INSTALL_SCIPY, _bash("a", command, "2026-01-01T00:00:05Z"))) == [("scipy", "install", None)] + + +@pytest.mark.parametrize("command", [ + "pip install scipy 2>logs\\\\|cat", # ``\\`` is a backslash: the pipe after it runs cat, whose code is the result + 'pip install scipy "--log=C:\\\\" | tee l', # inside double quotes too: the quote closes, and the pipe is real + "exec true && pip install scipy", # exec with a program replaces the shell: nothing after it runs +]) +def test_a_pipeline_or_an_exec_with_a_program_gives_no_install(command): + assert _choices(_run(INSTALL_SCIPY, _bash("a", command, "2026-01-01T00:00:05Z"))) == [] + + +def _choice_of(text: str, rc: object = 0) -> list[tuple]: + events = [{"t": 0.0, "kind": "command", "text": text, "rc": rc}] + return _choices(with_choices(events, SCIPY_INSTALL, [])) + + +SCIPY_INSTALL = {"scipy": {"install": ["scipy"]}} + + +@pytest.mark.parametrize("text", [ + "sudo --help pip install scipy", # the wrapper's --help runs no install + "conda install --download-only scipy", # downloads only: nothing is linked into the environment + "pip install --report scipy numpy", # --report takes the next word as its file +]) +def test_an_install_that_installs_nothing_gives_no_choice(text): + assert _choice_of(text) == [] + + +def test_exec_without_a_program_does_not_end_the_shell_for_the_install_after_it(): + assert _choice_of("exec -l && pip install scipy") == [("scipy", "install", None)] + + +def test_exec_with_a_program_still_gives_its_install(): + assert _choice_of("exec pip install scipy") == [("scipy", "install", None)] + + +@pytest.mark.parametrize("text", [ + "conda install -y conda-forge::scipy", # the channel prefix is not part of the package's name + "conda install -y conda-forge::scipy>=1.11", +]) +def test_a_conda_channel_prefix_still_names_the_package(text): + assert _choice_of(text) == [("scipy", "install", None)] + + +def test_a_missing_exit_code_gives_no_choice(): + assert _choice_of("pip install scipy", rc=None) == [] diff --git a/dokimi/tests/test_repo_cli_traces.py b/dokimi/tests/test_repo_cli_traces.py index fe9d5c381..45913be7e 100644 --- a/dokimi/tests/test_repo_cli_traces.py +++ b/dokimi/tests/test_repo_cli_traces.py @@ -713,10 +713,14 @@ def test_copilot_json_to_events(): assert t.reads() == {"AGENTS.md", "sub/AGENTS.md"} -@pytest.mark.parametrize(("done", "rc"), [({"success": True}, 0), ({"success": False}, 1), ({}, None)], - ids=["success", "failure", "unknown"]) -def test_copilot_command_exit_code_falls_back_to_success(done, rc): - """Without `shellExecution.exitCode` the rc comes from `success`; with neither it stays unknown.""" +@pytest.mark.parametrize(("done", "rc"), [({"shellExecution": {"exitCode": 0}}, 0), + ({"shellExecution": {"exitCode": 1}, "success": True}, 1), + ({"success": True}, None), ({"success": False}, None), ({}, None)], + ids=["exit-0", "exit-code-wins-over-success", "success-without-code", "failure-without-code", + "unknown"]) +def test_copilot_command_rc_is_its_exit_code_only(done, rc): + """A shell command's rc is `shellExecution.exitCode`, the code the command returned; a result with only `success` + gives no rc, so no choice rests on it (the same rule for every CLI).""" raw = _jsonl( {"type": "tool.execution_start", "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "make"}}}, diff --git a/dokimi/tests/test_repo_probes.py b/dokimi/tests/test_repo_probes.py new file mode 100644 index 000000000..d90b32c23 --- /dev/null +++ b/dokimi/tests/test_repo_probes.py @@ -0,0 +1,403 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Probe events (repos/probes.py, contract §4.1): a command that asks the host for its facts gets a ``probe`` event.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.repos.events import convert +from dokimi.repos.manifest import ManifestError, parse_manifest +from dokimi.repos.probes import DEFAULT_PROBES, load_probe_file, probes_for, validate_config, with_probes + +WS = "/ws" + + +def _claude_call(tool_id: str, command: str, t: str, name: str = "Bash", agent: str | None = None) -> dict: + rec = {"type": "assistant", "timestamp": t, "message": {"content": [ + {"type": "tool_use", "id": tool_id, "name": name, "input": {"command": command}}]}} + return {**rec, "parent_tool_use_id": agent} if agent else rec + + +def _claude_result(tool_id: str, text: str = "ok", t: str = "2026-01-01T00:00:06Z") -> dict: + return {"type": "user", "timestamp": t, "message": {"content": [ + {"type": "tool_result", "tool_use_id": tool_id, "is_error": False, "content": text}]}} + + +def _claude(*lines: dict) -> list[dict]: + """The command and probe events of a Claude Code stream whose first line starts the clock.""" + first = {"type": "system", "subtype": "init", "timestamp": "2026-01-01T00:00:00Z"} + raw = "\n".join(json.dumps(x) for x in (first, *lines)) + return with_probes(convert("claude-stream-json", raw, WS).events, probes_for()) + + +def _commands(events: list[dict]) -> list[dict]: + return [e for e in events if e["kind"] in ("command", "probe")] + + +def _probe(target: str, pattern: str, text: str, t: float | None, rc: int | None) -> dict: + return {"t": t, "kind": "probe", "target": target, "pattern": pattern, "text": text, "rc": rc} + + +def test_nproc_then_a_threaded_solver_gives_the_probe_after_its_command(): + events = _claude( + _claude_call("a", "nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "8", "2026-01-01T00:00:05Z"), + _claude_call("b", "python solve.py --threads 8", "2026-01-01T00:00:09Z"), + _claude_result("b", "done", "2026-01-01T00:00:20Z"), + ) + assert events == [ + {"t": 5.0, "kind": "command", "text": "nproc", "rc": 0, "output": "8", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", 5.0, 0), + {"t": 9.0, "kind": "command", "text": "python solve.py --threads 8", "rc": 0, "output": "done", + "output_truncated": False}, + ] + + +def test_a_grep_of_the_word_is_no_probe(): + events = _claude(_claude_call("a", "grep nproc README", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_a_probe_is_matched_on_the_program_not_on_an_argument_of_another_one(): + events = _claude(_claude_call("a", "pip install nproc && echo lscpu", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["kind"] for e in events] == ["command"] + + +def test_an_agent_that_never_probes_gives_no_probe_event(): + events = _claude(_claude_call("a", "ls -la", "2026-01-01T00:00:05Z"), _claude_result("a"), + _claude_call("b", "git status", "2026-01-01T00:00:06Z"), _claude_result("b")) + assert all(e["kind"] != "probe" for e in events) + + +def test_one_command_gives_the_command_and_every_probe_it_runs(): + events = _claude(_claude_call("a", "grep -c processor /proc/cpuinfo | cat && nproc", "2026-01-01T00:00:05Z"), + _claude_result("a", "4")) + assert _commands(events)[1:] == [ + _probe("cpu", "/proc/cpuinfo", events[0]["text"], 5.0, 0), + _probe("cpu", "nproc", events[0]["text"], 5.0, 0), + ] + assert events[0]["kind"] == "command" + + +def test_a_probe_in_a_substitution_is_run_by_the_command(): + events = _claude(_claude_call("a", "make -j$(nproc)", "2026-01-01T00:00:05Z"), _claude_result("a")) + assert _commands(events)[1] == _probe("cpu", "nproc", "make -j$(nproc)", 5.0, 0) + + +@pytest.mark.parametrize(("command", "pattern"), [ + ("lscpu", "lscpu"), + ("time nproc", "nproc"), + ("uname -a", "uname"), + ("free -h", "free"), + ("nvidia-smi --query-gpu=name --format=csv", "nvidia-smi"), + ("/usr/bin/nvidia-smi", "nvidia-smi"), + ("head -n 3 /etc/os-release", "/etc/os-release"), + ("cat \"/proc/meminfo\"", "/proc/meminfo"), + ("pip list --format=json", "pip list"), + ("python -m pip freeze", "pip freeze"), + ("pip3 show numpy", "pip show"), + ("conda list -n base", "conda list"), + ("printenv PATH", "printenv"), + ("env", "env"), + ("sysctl -a | grep machdep.cpu", "sysctl -a | grep machdep.cpu"), +]) +def test_default_command_patterns_match_the_program_that_runs(command, pattern): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["pattern"] for e in events if e["kind"] == "probe"] == [pattern] + + +@pytest.mark.parametrize(("command", "target"), [ + ('python -c "import os; print(os.cpu_count())"', "cpu"), + ("python3 -c 'import torch; print(torch.cuda.is_available())'", "gpu"), + ('python -c "import platform; print(platform.processor())"', "cpu"), +]) +def test_a_python_one_liner_that_calls_a_probe_is_a_probe(command, target): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert [e["target"] for e in events if e["kind"] == "probe"] == [target] + + +@pytest.mark.parametrize("command", [ + 'echo "os.cpu_count()"', + "python -c 'print(1)' # os.cpu_count()", + 'python -c "import os; os.makedirs(\'x\')"', + "cat README | grep 'nproc'", + "git diff | grep nproc", +]) +def test_text_that_is_not_run_code_is_no_probe(command): + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + assert all(e["kind"] != "probe" for e in events) + + +def test_a_windows_powershell_probe_is_matched_case_blind_and_through_parentheses(): + events = _claude( + _claude_call("a", "Get-CimInstance -ClassName Win32_Processor | Select-Object NumberOfCores", + "2026-01-01T00:00:05Z", name="PowerShell"), + _claude_result("a", "8"), + _claude_call("b", "(get-ciminstance win32_processor).NumberOfCores", "2026-01-01T00:00:07Z", + name="PowerShell"), + _claude_result("b", "8", "2026-01-01T00:00:07Z"), + _claude_call("c", "Get-CimInstance Win32_ComputerSystem", "2026-01-01T00:00:08Z", name="PowerShell"), + _claude_result("c", "", "2026-01-01T00:00:08Z"), + ) + assert [(e["kind"], e.get("target")) for e in events] == [ + ("command", None), ("probe", "cpu"), ("command", None), ("probe", "cpu"), ("command", None)] + + +def test_a_subagents_probe_carries_its_agent(): + events = _claude(_claude_call("a", "nproc", "2026-01-01T00:00:05Z", agent="task-1"), _claude_result("a")) + assert events[1] == {**_probe("cpu", "nproc", "nproc", 5.0, 0), "agent": "task-1"} + + +def _codex(*items: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in items) + return with_probes(convert("codex-json", raw, WS).events, probes_for()) + + +def test_codex_unwraps_the_shell_before_it_matches_a_probe(): + events = _codex( + {"type": "item.completed", "item": {"type": "command_execution", "command": "/usr/bin/bash -lc 'nproc'", + "aggregated_output": "4\n", "exit_code": 0}}, + {"type": "item.completed", "item": {"type": "command_execution", + "command": "/usr/bin/bash -lc 'nvidia-smi'", + "aggregated_output": "", "exit_code": 127}}, + ) + assert events == [ + {"t": None, "kind": "command", "text": "nproc", "rc": 0, "output": "4\n", "output_truncated": False}, + _probe("cpu", "nproc", "nproc", None, 0), + {"t": None, "kind": "command", "text": "nvidia-smi", "rc": 127}, + _probe("gpu", "nvidia-smi", "nvidia-smi", None, 127), + ] + + +def _copilot(*records: dict) -> list[dict]: + raw = "\n".join(json.dumps(x) for x in records) + return with_probes(convert("copilot-json", raw, WS).events, probes_for()) + + +def test_copilot_bash_and_powershell_probes_give_probe_events(): + events = _copilot( + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:01Z", + "data": {"toolCallId": "a", "toolName": "bash", "arguments": {"command": "nvidia-smi"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "a", "success": True, + "shellExecution": {"exitCode": 0}, + "result": {"content": "GPU 0"}}}, + {"type": "tool.execution_start", "timestamp": "2026-01-01T00:00:02Z", + "data": {"toolCallId": "b", "toolName": "powershell", + "arguments": {"command": "Get-CimInstance Win32_Processor"}}}, + {"type": "tool.execution_complete", "data": {"toolCallId": "b", "success": True, + "shellExecution": {"exitCode": 0}, "result": {"content": "x"}}}, + ) + assert [(e["kind"], e.get("target"), e.get("pattern")) for e in events] == [ + ("command", None, None), ("probe", "gpu", "nvidia-smi"), + ("command", None, None), ("probe", "cpu", "Get-CimInstance Win32_Processor"), + ] + + +def test_a_manifest_replaces_a_target_and_an_empty_list_turns_it_off(): + probes = probes_for({"cpu": ["lscpu"], "env": []}) + assert [p.text for p in probes if p.target == "cpu"] == ["lscpu"] + assert all(p.target != "env" for p in probes) + assert {p.text for p in probes if p.target == "gpu"} == set(DEFAULT_PROBES["gpu"]) + + +def test_a_run_layer_replaces_the_manifest_layer(): + manifest = {"cpu": ["lscpu"]} + run = {"cpu": ["nproc"]} + assert [p.text for p in probes_for(manifest, run) if p.target == "cpu"] == ["nproc"] + + +def test_no_probes_leaves_the_events_as_they_are(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + assert with_probes(events, ()) == events + + +def test_probe_events_go_to_a_copy_so_the_trace_events_the_counters_read_are_unchanged(): + events = [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + out = with_probes(events, probes_for()) + assert [e["kind"] for e in out] == ["command", "probe"] + assert events == [{"t": 1.0, "kind": "command", "text": "nproc", "rc": 0}] + + +@pytest.mark.parametrize(("raw", "message"), [ + ({"gpus": ["nvidia-smi"]}, "unknown target 'gpus'"), + ({"cpu": "nproc"}, "cpu must be a list"), + ({"cpu": [" "]}, "cpu must be a list"), + ({"cpu": ["py:os.cpu_count()"]}, "dotted Python name"), + ({"cpu": ['nproc "']}, "No closing quotation"), + ({"cpu": ["|"]}, "names a program"), + ({"cpu": ["nproc;lscpu"]}, "separate commands"), + ({"cpu": ["/proc/cpu info"]}, "one word with no spaces"), + (["cpu"], "expected a mapping"), +]) +def test_a_bad_pattern_config_is_refused_with_what_is_wrong(raw, message): + with pytest.raises(ValueError, match=message): + validate_config(raw, "probes") + + +def test_a_run_probe_file_is_yaml_or_json_and_checked(tmp_path): + good = tmp_path / "probes.yaml" + good.write_text("cpu: [lscpu]\nnetwork: [curl]\n", encoding="utf-8") + assert load_probe_file(str(good)) == {"cpu": ("lscpu",), "network": ("curl",)} + bad = tmp_path / "bad.json" + bad.write_text('{"cpu": ["nproc"', encoding="utf-8") + with pytest.raises(ValueError, match="--probes .*bad.json"): + load_probe_file(str(bad)) + + +def test_a_manifest_probes_key_is_read_and_a_bad_one_is_a_manifest_error(): + base = {"schema": "repo_manifest.v1", "id": "x", "guidance": {"globs": ["AGENTS.md"]}} + assert parse_manifest({**base, "probes": {"memory": ["free"]}}).probes == {"memory": ("free",)} + with pytest.raises(ManifestError, match="probes: 'disk' is not one of"): + parse_manifest({**base, "probes": {"disk": ["df"]}}) + + +def _probes(command: str) -> list[str]: + events = _claude(_claude_call("a", command, "2026-01-01T00:00:05Z"), _claude_result("a")) + return [e["pattern"] for e in events if e["kind"] == "probe"] + + +@pytest.mark.parametrize("command", [ + "cat <<'EOF' > script.sh\nnproc\nEOF", + "cat <<-EOF > script.sh\n\tnproc\n\tEOF", + "echo '$(nproc)'", + "echo '`nproc`'", + "echo \\$(nproc)", + "echo '|' nproc", + "sysctl -a '|' grep machdep.cpu", + "grep -e /proc/cpuinfo README", + "grep /proc/cpuinfo README", + "rg -e /proc/cpuinfo README", + "sed -n /proc/cpuinfo README", + "python -c 'print(\"os.cpu_count()\")'", + "python -c '# os.cpu_count()\nprint(1)'", + "python -c 'x = os.cpu_count'", + "echo \\| nproc", + "cat README > /etc/os-release", + "cat README >> /etc/os-release", + "cat README 2>/etc/os-release", + "grep --regexp /proc/cpuinfo README", + "grep --regexp=/proc/cpuinfo README", + "grep -f patterns README /proc/cpuinfo.bak", + "cat <<'EOF'\n$(nproc)\nEOF", + "true || nproc", + ": || nproc", + "python tool.py -c 'import os; os.cpu_count()'", +]) +def test_text_the_shell_does_not_run_gives_no_probe(command): + assert _probes(command) == [] + + +@pytest.mark.parametrize(("command", "found"), [ + ("make -j$(nproc --ignore=$(printf 1))", ["nproc"]), + ("sudo -n bash -c 'nproc'", ["nproc"]), + ("time bash -lc 'nproc'", ["nproc"]), + ("cat < cfg # note\n$(nproc)\nEOF", ["nproc"]), + ("cat < out.txt < /etc/os-release", ["/etc/os-release"]), + ("grep -f patterns /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep '>' /proc/cpuinfo", ["/proc/cpuinfo"]), + ("false || nproc", ["nproc"]), + ("grep '1'>/dev/null /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep 1>/dev/null cores /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep \"<\" /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep --file patterns /proc/cpuinfo", ["/proc/cpuinfo"]), + ("sed -f script.sed /proc/cpuinfo", ["/proc/cpuinfo"]), + ("grep -n cores /proc/cpuinfo 2>/dev/null", ["/proc/cpuinfo"]), + ('make -j"$(nproc)"', ["nproc"]), + ("make -j`nproc`", ["nproc"]), + ("bash <$(printf /tmp/probe.log) /etc/os-release", ["/etc/os-release"]), # the file is the operand, not the target + ("cat <<-EOF\n\tEOF\nnproc", ["nproc"]), # ``<<-`` takes a tab-indented terminator: the body ends there + ("cat <<-EOF\n EOF\nnproc\nEOF", []), # a space does not indent a ``<<-`` terminator: nproc is data +]) +def test_the_shell_structure_decides_which_probe_runs(command, patterns): + assert _probes(command) == patterns + + +def _probed(text: str, **event: object) -> list[tuple[str, str]]: + from dokimi.repos.probes import probes_for, with_probes + + events = [{"t": 0.0, "kind": "command", "text": text, "rc": 0, **event}] + return [(e["target"], e["pattern"]) for e in with_probes(events, probes_for(None)) if e["kind"] == "probe"] + + +@pytest.mark.parametrize("text, event", [ + ("nproc", {"denied": True}), # a denied command ran nothing + ("true || echo skipped || nproc", {}), + ("command -v nproc", {}), # ``command -v`` only looks the program up + ("sudo --help nproc", {}), # ``--help`` prints text and runs nothing + ("false && nproc", {}), + ("exit 0; nproc", {}), + ("exec true && nproc", {}), # exec replaces the shell: nothing after it runs + ("cat < list", {}), # a redirection target is no pattern word +]) +def test_a_command_that_runs_nothing_probed_gives_no_probe(text, event): + assert not [p for p in _probed(text, **event) if p[0] == "cpu" or p[1] in ("pip list",)] + + +@pytest.mark.parametrize("text", ["exec -l && nproc", "env FOO=1 nproc", "echo \"< f") + assert ("env", "printenv") in _probed("printenv") diff --git a/dokimi/tests/test_repo_run_events.py b/dokimi/tests/test_repo_run_events.py new file mode 100644 index 000000000..d42d0b425 --- /dev/null +++ b/dokimi/tests/test_repo_run_events.py @@ -0,0 +1,61 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo run --candidates` and `--probes` reach a cell's ``events.jsonl`` (contract §4.1), with a fake runner and harness.""" + +from __future__ import annotations + +import json + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.container import WORKSPACE +from repo_eval_fixture import FakeRunner, fake_harness, make_repo + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + root, _ = make_repo(tmp_path) + exe, profile = fake_harness(tmp_path) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + monkeypatch.setenv("FAKE_ACTION", "fix") + return {"root": root, "exe": exe, "profile": profile, "out": tmp_path / "runs"} + + +def _run(s, *extra): + argv = ["repo", "run", f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(s["profile"]), "--bin", str(s["exe"]), "--model", "m-1", "--batch", "b1", "--out", str(s["out"]), + "--cache-root", str(s["out"] / "caches"), "--agent-network", "host", *extra] + args = build_parser().parse_args(argv) + args.func(args) + + +def test_a_run_writes_the_probe_and_choice_rows_its_files_name(setup, monkeypatch, tmp_path): + cands, probes = tmp_path / "cands.yaml", tmp_path / "probes.yaml" + cands.write_text("sp: {import: [scipy]}\n") + probes.write_text("cpu: [nproc]\n") + monkeypatch.setenv("FAKE_TOOL_USES", json.dumps([ + ["Bash", {"command": "nproc"}], ["Write", {"file_path": f"{WORKSPACE}/a.py", "content": "import scipy\n"}]])) + monkeypatch.setenv("FAKE_TOOL_RESULTS", "1") + _run(setup, "--arm", "guidance:none", "--task", "fix_add", "--candidates", str(cands), "--probes", str(probes)) + (events_file,) = (setup["out"] / "b1").glob("*/events.jsonl") + kinds = [(e["kind"], e.get("pattern") or e.get("act")) for e in map(json.loads, events_file.read_text().splitlines())] + assert ("probe", "nproc") in kinds and ("choice", "import") in kinds + + +@pytest.mark.parametrize("flag,other", [("--candidates", "sp: {import: [sklearnex]}\n"), ("--probes", "cpu: [lscpu]\n")]) +def test_a_resume_with_other_candidates_or_probes_is_refused(setup, tmp_path, capsys, flag, other): + cands, probes, changed = tmp_path / "cands.yaml", tmp_path / "probes.yaml", tmp_path / "changed.yaml" + cands.write_text("sp: {import: [scipy]}\n") + probes.write_text("cpu: [nproc]\n") + changed.write_text(other) + + def args(c, p): + return ["--arm", "guidance:none", "--task", "fix_add", "--candidates", str(c), "--probes", str(p)] + + _run(setup, *args(cands, probes)) + _run_again = (args(changed, probes) if flag == "--candidates" else args(cands, changed)) + with pytest.raises(SystemExit): + _run(setup, *_run_again) + assert "holds cells of another execution" in capsys.readouterr().err