Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
d3360cd
feat(repo): probe events for commands that ask the host for its facts
napetrov Nov 15, 2026
d7ee57a
Pin quoted text that is still read as a probe with xfail tests
antonwolfy Nov 15, 2026
4002160
Skip wrapper options before taking the program a wrapper runs
antonwolfy Nov 15, 2026
8a0885f
Leave a searcher's search expression out of the path probe match
antonwolfy Nov 15, 2026
57508d3
Match commands that follow a shell keyword such as then or do
antonwolfy Nov 15, 2026
2fe9ab0
Leave option values and redirection targets out of a reader's files
antonwolfy Nov 15, 2026
658e2ac
Read the probe forms that review found the matcher missing or inventing
napetrov Nov 16, 2026
13e3d9c
Merge intel/main into feat/trace-probe-events
napetrov Nov 16, 2026
a445b7f
Address review: preserve POSIX backslash-newline continuations
napetrov Nov 16, 2026
815dbc9
Address review: a quoted parenthesis does not end a substitution
napetrov Nov 16, 2026
6c007cf
Address review: a search option's value is no operand
napetrov Nov 16, 2026
e39de28
Merge branch 'main' of https://github.com/intel/dokimi into pr782-add…
napetrov Nov 16, 2026
7746c59
Address review: PowerShell assignment, python -Ic, here-string
napetrov Nov 16, 2026
9f90047
Merge branch 'main' of https://github.com/intel/dokimi into pr782-add…
napetrov Nov 16, 2026
a3672c1
Address review: a probe matcher that raises keeps the cell
napetrov Nov 17, 2026
77061ba
Address review: launchers, env and pip3 patterns match the probe they…
napetrov Nov 17, 2026
f72fa43
Merge branch 'main' of https://github.com/intel/dokimi into feat/trac…
napetrov Nov 17, 2026
39c94ed
Self-review: a launcher pattern matches the program it runs
napetrov Nov 17, 2026
af6d2a8
Address review: default cpu probe matches bare sysctl
napetrov Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions dokimi/changelog.d/20261114_120000_napetrov_probe-events.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Added

- **`repo run`: a `probe` event for a command that asks the host for its facts.** A command
such as `nproc`, `lscpu`, `nvidia-smi`, `pip list`, `printenv` or a `python -c` one-liner that
calls `os.cpu_count()` gets a `probe` event after its `command` event, with its `target`
(`cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`) and the `pattern` it matched. The
patterns match the program the shell runs, not argument text (`grep nproc README` is no probe).
The defaults ship in `dokimi/repos/probes.py`; a manifest's `probes` key or `repo run --probes
FILE` replaces a target's list, and `[]` turns it off. Existing event kinds are unchanged.
1 change: 1 addition & 0 deletions dokimi/docs/reference/cli-interface.json
Original file line number Diff line number Diff line change
Expand Up @@ -1395,6 +1395,7 @@
{"key": "--max-turns", "action": "_StoreAction", "choices": null, "const": null, "default": 200, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser.<locals>.positive_int#faf57db5ff9f"},
{"key": "--model", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null},
{"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-runs", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--probes", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null},
{"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser.<locals>.positive_int#faf57db5ff9f"},
{"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
Expand Down
26 changes: 26 additions & 0 deletions dokimi/docs/repo-evaluation-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,9 @@ build_systems: [bazel, make] # selects L0 claim-check adapters
process_metrics: # optional repository counters, result.json `process.counters` (§4.1)
bazel_cmds: '\bbazel(?:isk)?\b' # a plain regex searches the command text
icpx_missing: {regex: "icpx is not found", match: output} # match: text (default), output or any

probes: # optional: target -> patterns of `probe` events (§4.1)
cpu: [nproc, "py:os.cpu_count"] # a listed target replaces its defaults; [] turns it off
```

Rules:
Expand Down Expand Up @@ -530,6 +533,29 @@ The line format, its version and the promise that comes with it: [events-format.
- A `read` event may have no `path`: a refused Claude Code Bash read of a
relative path after an earlier call moved the shell, whose file is not
known (`{"kind": "read", "denied": true}`).
- A `probe` event follows the `command` event of a command that asks the host
for its facts, one per pattern it matches (a command may give several):
`{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}`.
`target` is one of `cpu`, `gpu`, `os`, `memory`, `packages`, `env`, `network`;
`pattern` is the configured pattern it matched; `text`, `rc` and `agent` are the
command's. The defaults (`dokimi/repos/probes.py`, `DEFAULT_PROBES`): `cpu` `lscpu`,
`/proc/cpuinfo`, `nproc`, `sysctl`, `Get-CimInstance Win32_Processor`,
`py:os.cpu_count`, `py:platform.processor`; `gpu` `nvidia-smi`, `rocm-smi`,
`py:torch.cuda.is_available`; `os` `uname`, `/etc/os-release`, `sw_vers`, `systeminfo`;
`memory` `free`, `/proc/meminfo`; `packages` `pip list`, `pip show`, `pip freeze`,
`conda list`; `env` `env`, `printenv`. A pattern is matched on the program the shell or
PowerShell runs, with the words it names in any order and case (`grep nproc README` runs
`grep`: no probe), on a file a reading command (`cat`, `head`, `grep`, ...) names for a
path pattern, and on the call in the code of a `python -c` one-liner for `py:NAME`. A
`$( )` substitution or a `bash -c` script counts as a command the agent ran. Not matched:
a Python heredoc, a script the agent runs from a file, a pattern with a `|` that is
not a pipeline of the same commands, and a launcher the wrapper lists do not know
(`docker exec`, `nix develop -c`). `env`, `timeout`, `xargs`, `conda run` and `uv run`
run the program after them; `env` with no program after its options is its own probe (`env -S` is not read). Each cell's `meta.json`
records the patterns per target (`probes`) that it ran with.
A manifest's `probes` (§2) replaces a target's defaults, and `repo run --probes FILE`
(YAML or JSON, the same mapping) replaces them again; nothing else changes the events,
and the `process_metrics` counters do not count probes.

`result.json` reports the agent's tokens (`input_tokens`, `output_tokens`,
`cache_read_tokens`, `cache_write_tokens`) over the same scope as `cost_usd`.
Expand Down
7 changes: 6 additions & 1 deletion dokimi/dokimi/commands/repo_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock
from dokimi.repos.bases import check_mcp_configs
from dokimi.repos.container import docker_network
from dokimi.repos.harness_profile import load_profile
from dokimi.repos.probes import load_probe_file
from dokimi.repos.runner import (
AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks)
from dokimi.repos.selfcheck import ANCESTOR_NAMES, isolation_probe
Expand All @@ -332,6 +333,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock
try:
parsed = [parse_arm(a) for a in arms] # before the image build: a typo or an unreadable patch fails now
profile = load_profile(args.harness)
probes = load_probe_file(args.probes) if args.probes else None # a bad file fails before the containers start
profile.tool_args(args.tools, args.delegation) # a profile that cannot turn subagents off: refuse now
harness_bin = _harness_bin(args.bin or profile.bin)
arms = pin_arms(parsed, source_git_dir(source, open_source(source, _cache_dir(args)), _cache_dir(args)),
Expand All @@ -345,7 +347,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock
validate_tasks(tasks) # refuse malformed host requirements before containers start
options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network,
tools=args.tools, delegation=args.delegation, timeout_s=args.timeout,
max_turns=args.max_turns, budget_usd=args.budget) for m in args.model]
max_turns=args.max_turns, budget_usd=args.budget, probes=probes) for m in args.model]
for task in tasks: # refuse before anything runs, not cell by cell
task_caps(task, options[0])
agent_network(options[0], ctx, task)
Expand Down Expand Up @@ -548,6 +550,9 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str
help="Seconds per agent run unless task.json sets timeout_s (default 2700)")
run_p.add_argument("--max-turns", type=positive_int, default=200, dest="max_turns",
help="Unless task.json sets max_turns (default 200)")
run_p.add_argument("--probes", default=None, metavar="FILE",
help="YAML or JSON: target -> probe patterns, over the manifest's `probes` (default: the "
"framework's list; contract §4.1)")
run_p.add_argument("--budget", type=_usd, default=2.0,
help="USD cap per agent run unless task.json sets budget_usd (default 2.0)")
run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd",
Expand Down
4 changes: 4 additions & 0 deletions dokimi/dokimi/repos/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
{"t": 5.0, "kind": "edit", "path": "src/foo.cpp"}
{"t": 20.2, "kind": "tool", "name": "WebFetch", "denied": true}
{"t": 2.0, "kind": "tool", "name": "Skill", "skill": "lint-fix"}
{"t": 4.0, "kind": "probe", "target": "cpu", "pattern": "nproc", "text": "nproc", "rc": 0}

A ``probe`` event follows the ``command`` event of a command that asks the host for its facts, one per
pattern it matches (:mod:`dokimi.repos.probes`, contract §4.1).

``t`` is seconds since the first timestamped trace line, or ``null`` when the
format has no timestamps. Paths inside the workspace are made relative to it.
Expand Down
14 changes: 14 additions & 0 deletions dokimi/dokimi/repos/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from .graders.hidden_tests import HIDDEN_DIR, hidden_files
from .host_spec import HostSpecError, parse_host
from .process_metrics import Counter, parse_counter
from .probes import validate_config
from .source import RepoTree, SourceError
from .useful_spec import UsefulSpecError, parse_useful

Expand Down Expand Up @@ -137,6 +138,7 @@
explore_tasks: str | None = None
build_systems: tuple[str, ...] = ()
process_metrics: dict[str, Counter] = field(default_factory=dict)
probes: dict[str, tuple[str, ...]] = field(default_factory=dict) # target -> patterns (repos/probes.py)

@property
def runnable(self) -> bool:
Expand Down Expand Up @@ -175,6 +177,7 @@
"build_systems": list(self.build_systems),
"process_metrics": {name: {"regex": c.pattern.pattern, "match": c.field}
for name, c in self.process_metrics.items()},
**({"probes": {t: list(p) for t, p in self.probes.items()}} if self.probes else {}),

Check warning on line 180 in dokimi/dokimi/repos/manifest.py

View workflow job for this annotation

GitHub Actions / Mutation of changed lines

mutant inconclusive: a candidate test file for it did not run
}


Expand Down Expand Up @@ -291,6 +294,7 @@
if duplicates:
errors.append(f"image/caches: duplicate cache name(s) {duplicates}")
counters = _parse_counters(raw.get("process_metrics") or {}, errors)
probes = _parse_probes(raw.get("probes"), errors)
if errors:
raise ManifestError(path, errors)

Expand All @@ -312,9 +316,19 @@
explore_tasks=_norm(raw["explore_tasks"]) if raw.get("explore_tasks") else None,
build_systems=tuple(raw.get("build_systems") or ()),
process_metrics=counters,
probes=probes,
)


def _parse_probes(raw: Any, errors: list[str]) -> dict[str, tuple[str, ...]]:
"""The manifest's ``probes`` (repos/probes.py); a bad one is appended to ``errors``."""
try:
return validate_config(raw, "probes")
except ValueError as exc:
errors.append(str(exc))
return {}


def _test_entry_problems(key: str, found: set[str]) -> list[str]:
"""``{tests}`` belongs to ``entry_points.test`` alone, which must name it and ``{out}``, its report."""
if key != _TEST_ENTRY_POINT:
Expand Down
Loading
Loading