Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
2ee29fd
feat(repo): fingerprint the host with collectors; --label-host runs a…
napetrov Nov 15, 2026
881673e
Say the Windows CPU model comes from platform.processor()
antonwolfy Nov 15, 2026
a22fd3a
fix(repo): host fingerprint report rows, path scrub, Windows CPU
vchamarthi Nov 15, 2026
f042010
fix(repo): scrub whole paths, keep legacy host rows
vchamarthi Nov 15, 2026
6d3945e
fix(repo): scrub a path that follows a colon
napetrov Nov 16, 2026
1203985
Merge branch 'main' of https://github.com/intel/dokimi into pr781-add…
napetrov Nov 16, 2026
b4fdefe
Address review: record items already fixed on this branch
napetrov Nov 16, 2026
7cfe034
Merge branch 'main' of https://github.com/intel/dokimi into pr781-add…
napetrov Nov 16, 2026
f8757b9
Address review: redact a bare POSIX path with spaces whole
napetrov Nov 16, 2026
ed0ec8e
Address review: the path comment lists the lookbehind's characters
napetrov Nov 16, 2026
30ae922
Merge branch 'main' of https://github.com/intel/dokimi into pr781-add…
napetrov Nov 16, 2026
106a99f
Merge branch 'main' of https://github.com/intel/dokimi into pr781-add…
napetrov Nov 16, 2026
ac2d4d7
fix(repo): redact file: URL and UNC paths in host-probe errors
napetrov Nov 16, 2026
9046541
Address review: count a started missing cell whose host has no row
napetrov Nov 17, 2026
a22850c
Address review: warn when pooled labelled cells missed their host
napetrov Nov 17, 2026
60a9c54
Address review: a tool that cannot be started keeps the other tools
napetrov Nov 17, 2026
9b265dc
Address review: scrub the probe-did-not-start error
napetrov Nov 17, 2026
7444d7c
Address review: the contract says what no-flag runs do now
napetrov Nov 17, 2026
779a950
Address review: move the unplaced step of summarize into a helper
napetrov Nov 17, 2026
da971f6
Merge branch 'main' of https://github.com/intel/dokimi into pr781-add…
napetrov Nov 17, 2026
face922
Address review: tool identity is presence only, failing collectors ar…
napetrov Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Added

- **The host probe fingerprints the machine with a registry of collectors** — `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`, each a flat dict under `collectors` (`collectors_version` 1), beside the container's unchanged keys. A failing collector reports `{"error": ...}` and the others still run; no host or user name, IP address, environment value or absolute path is recorded.
- **`repo run --label-host` runs a task the host misses instead of blocking it** — each cell records `host_match` (per requirement met, unmet or unknown) in `result.json` and `meta.json`, and `repo report` groups results by host fingerprint. Without the flag, `repo report` still splits rows by host fingerprint when a batch's cells ran on different hosts.
1 change: 1 addition & 0 deletions dokimi/docs/reference/cli-interface.json
Original file line number Diff line number Diff line change
Expand Up @@ -1389,6 +1389,7 @@
{"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null},
{"key": "--image", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--jobs/-j", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser.<locals>.positive_int#faf57db5ff9f"},
{"key": "--label-host", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null},
{"key": "--manifest", "action": "_StoreAction", "choices": null, "const": null, "default": ".agent-eval/repo-eval.yaml", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--manifest-file", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null},
{"key": "--max-batch-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.repo_run._usd#591d6e2f0474"},
Expand Down
25 changes: 24 additions & 1 deletion dokimi/docs/repo-evaluation-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ belongs to the repository's grader and is passed through unread, except `grader`
| `gate` | no | framework | what counts as success; default `["pass"]`. Each entry is a boolean metric name, which must be `true`, or `{"metric": <name>, "min": <number>}` for a numeric metric |
| `score` | for score-only tasks | framework | `{"metric": <name>, "max": <number>}`: the metric that is the result when `pass` is `null`, and the value the oracle must reach (§4.3) |
| `timeout_s`, `max_turns`, `budget_usd` | no | framework | per-task caps, overriding run defaults; each a finite number > 0 |
| `host` | no | framework | `{"min_cpus": <int>, "cpu_vendors": [<str>], "cpu_flags": [<str>]}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not |
| `host` | no | framework | `{"min_cpus": <int>, "cpu_vendors": [<str>], "cpu_flags": [<str>]}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), unless `repo run --label-host` runs it and records the miss, and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not |
| `egress` | no | framework | extra hosts (DNS names, optional `:port`; `*` stands for one whole label and the last two labels are literal, so `*.pythonhosted.org` passes and `*` or `*.org` does not) this task's agent and entry points reach through the proxy (§2), for example `["pypi.org", "files.pythonhosted.org"]` |
| `expected_skill` | no | framework | the name (directory name) of the skill the agent should use; a `skill-catalog:` arm must install it, or the arm is refused before anything runs. On a `guidance:as-skill` catalog arm the generated `<repo>-guidance` skill is refused too (its usage is `guidance_skill_loaded`). `result.json` `skills` records whether the agent used it and whether it used it first |
| `useful` | no | framework | the definition of the run's first useful result, reported as `first_useful` in `result.json` (§4.1): `{"command": "<regex>"}` for a task whose grade runs the target tests, or `{"edit": "<glob>"}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` |
Expand Down Expand Up @@ -700,6 +700,29 @@ apart, neither failed nor missing. If the probe itself fails, a task with
`host` requirements is not run either, and `repo run` exits 1 when one of its
cells is left unrun; a cell that already has a `result.json` keeps it.

The probe also fingerprints the host (`dokimi/repos/host_fingerprint.py`, `collectors_version` 1).
Its collectors are `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`; each is a
flat dict of JSON scalars under `collectors` in the probe's facts, beside the container's own keys
(`cpus`, `cpu_vendor`, ...), which are unchanged. A collector that fails reports `{"error": ...}`
and the others still run. Linux reads `/proc`, `/sys` and `/etc/os-release`; macOS reads `sysctl`;
Windows reads the kernel's memory status, takes the CPU model and vendor from the registry (not from
`platform.processor()`, which reads `PROCESSOR_IDENTIFIER`) and spawns only `git --version`, `nvidia-smi` and the other
tools' `--version` calls when they are on the path. `gpu` is NVIDIA only. `virt` reads a container
from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from DMI or the `hypervisor`
CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value,
interface name or absolute path; an error text is cut to 300 characters with its paths replaced.
`fingerprint_id` hashes the CPU and identity collectors but not `network`, so a bridge that appears
between runs does not split a batch. `tools` count by presence only (a version change or a timed-out
`--version` does not split a host), and a collector that still fails after one retry is left out.

`repo run --label-host` runs a task that the host misses, instead of marking it unsupported. Each
cell's `result.json` and `meta.json` hold `host_match`: the `fingerprint`, the report's `met`, and one
field per declared requirement with `met` true, false, or null where the probe could not tell. The
report's `met` is false if any field is, else null if any is undecided, else true. The requirements
are checked against the grading container's CPU view, as in §3, not against the host's collectors,
which only enter the fingerprint. `repo report` groups rows by `host_fingerprint` and notes a
labelled miss ("labelled, unmet: cpu_flags.avx512f"). Without the flag, cells are not run on a host that misses the requirements, but results still carry `host_fingerprint` and `repo report` still splits rows by it.

Both commands exit 2, before any build, on a source that does not parse (a local path that is not a
directory, an unpinned remote), on a pinned local path inside a repository that is not its root and on a `--cache-dir` that is not a usable directory; a
source that parses but cannot be opened (clone, fetch, revision) exits 1 with a `✗` line. A manifest that
Expand Down
6 changes: 5 additions & 1 deletion dokimi/dokimi/commands/repo_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock
if skill_pin is not None:
write_skill_pin(batch_dir, skill_pin)
summary = run_batch(ctx, tasks, arms, options, args.reps, batch_dir, jobs=args.jobs, force=args.force,
max_batch_usd=args.max_batch_usd, log=_progress)
max_batch_usd=args.max_batch_usd, log=_progress, label_host=args.label_host)
except _errors() as exc:
print(f"✗ {exc}", file=sys.stderr)
sys.exit(1)
Expand Down Expand Up @@ -554,6 +554,10 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str
help="Start no new cell once the batch has spent this much, earlier invocations included "
"(and the budgets of the cells in flight)")
run_p.add_argument("--force", action="store_true", help="Rerun cells that already have result.json")
run_p.add_argument("--label-host", action="store_true", dest="label_host",
help="Run a task whose task.json `host` requirements this host misses anyway, and record "
"per field what was met and what was not (host_match in result.json) instead of marking "
"the cells unsupported; results carry the host fingerprint, so `repo report` groups them by it")
run_p.set_defaults(func=cmd_repo_run)

report_p = repo_sub.add_parser("report", help="Per (task, arm, model) table over a batch directory")
Expand Down
5 changes: 5 additions & 0 deletions dokimi/dokimi/repos/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,11 @@ def _arm_warnings(cells: list[dict[str, Any]], baseline: str, text: str, models:
if uptake and uptake["cells"] and not uptake["used"]:
out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); "
"a delta measures the patch's text, not the tool")
# A labelled cell on a host that misses its task (or could not be checked) is pooled like any other: say so.
missed = sorted({c["task"] for arm in arms for c in arm if "host_match" in c and c["host_match"].get("met") is not True})
if missed:
out.append(f"{text} vs {baseline}: cells run with --label-host on a host that misses the task's requirements "
f"or could not be checked are pooled (tasks {', '.join(missed)}); not a measured pass")
return out


Expand Down
84 changes: 72 additions & 12 deletions dokimi/dokimi/repos/host.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,16 @@

from __future__ import annotations

import hashlib
import json
import math
import re
import uuid
from pathlib import Path
from typing import Any

from .container import ContainerError, ContainerSpec
from .grading import EvalContext
from .host_fingerprint import IDENTITY_COLLECTORS, Probe, collect, cpuinfo_field, scrub
from .host_spec import HOST_KEYS, KNOWN_VENDORS, HostRequirements, HostSpecError, parse_host # noqa: F401
from .useful_spec import parse_useful
from .workspace import Task
Expand Down Expand Up @@ -171,23 +172,82 @@ def host_facts(raw: dict[str, Any]) -> dict[str, Any]:
quota = cgroup_cpus(raw)
counts = [n for n in (affinity, quota) if n]
text = str(raw["cpuinfo"]) if isinstance(raw.get("cpuinfo"), str) else ""
flags = _field(text, "flags") or _field(text, "Features") # x86 | arm64
flags = cpuinfo_field(text, "flags") or cpuinfo_field(text, "Features") # x86 | arm64
return {"cpus": min(counts) if counts else None, "affinity_cpus": affinity, "cgroup_cpus": quota,
"cpu_vendor": _field(text, "vendor_id"), "cpu_model": _field(text, "model name"),
"cpu_vendor": cpuinfo_field(text, "vendor_id"), "cpu_model": cpuinfo_field(text, "model name"),
"cpu_flags": sorted(set(flags.split())) if flags else []}


def _field(text: str, name: str) -> str | None:
match = re.search(rf"^{re.escape(name)}\s*:\s*(.*)$", text, re.M)
return match.group(1).strip() if match else None
def probe_host(ctx: EvalContext, work_dir: Path, probe: Probe | None = None) -> dict[str, Any]:
"""Facts of a container like the grading one, plus the host fingerprint (:mod:`.host_fingerprint`).

The container's facts are the top-level keys, ``{"error": ...}`` in place of them when the probe
could not run. The spec carries what decides the CPU view: the evaluation's image and runner
(whose flags every container shares); no mounts, no network.
"""
return {**_container_facts(ctx, work_dir), **collect(probe)}


def fingerprint_id(facts: dict[str, Any]) -> str:
"""A short id of the host: the same machine gives the same id on every run (see ``IDENTITY_COLLECTORS``)."""
keys = ("cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags")
collectors = facts.get("collectors") or {}
identity = {**{k: facts.get(k) for k in keys},
**{name: _identity_value(name, collectors.get(name)) for name in IDENTITY_COLLECTORS}}
return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()[:12]


def _identity_value(name: str, value: Any) -> Any:
"""What a collector adds to the id: nothing when it failed (left out, as if absent), and for ``tools``
only whether each tool is present, so a version change or a timed-out ``--version`` does not split a host."""
if not isinstance(value, dict) or "error" in value:
return None
if name == "tools":
return {tool: version is not None for tool, version in value.items()}
return value


def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]:
"""Facts of a container like the grading one, or ``{"error": ...}`` when the probe could not run.
def match_report(task: Task, facts: dict[str, Any] | None) -> dict[str, Any]:
"""Per declared requirement, whether this host meets it, for a run that labels a miss instead of blocking it.

The spec carries what decides the CPU view: the evaluation's image and runner (whose flags
every container shares); no mounts, no network.
``met`` is ``True`` or ``False`` per field, ``None`` when the probe could not tell (the rule of :func:`unmet`);
the report's ``met`` is ``False`` if any field is, else ``None`` if any is undecided, else ``True``.
"""
facts = facts or {}
need = requirements(task)
fields = _match_fields(need, facts) if need is not None else []
verdicts = {f["met"] for f in fields}
met = False if False in verdicts else None if None in verdicts else True
return {"fingerprint": fingerprint_id(facts), "met": met, "fields": fields}


def match_summary(match: dict[str, Any]) -> str:
"""The fields a labelled run's host misses, or those it could not tell: ``cpu_flags.avx512f missing``."""
missed = [f"{f['field']} missing" for f in match["fields"] if f["met"] is False]
undecided = [f"{f['field']} unknown" for f in match["fields"] if f["met"] is None]
return ", ".join(missed + undecided) or "nothing unmet"


def _match_fields(need: HostRequirements, facts: dict[str, Any]) -> list[dict[str, Any]]:
"""One entry per declared requirement; undecided (``met`` None) where :func:`unmet` says the probe cannot tell."""
cpus, vendor = facts.get("cpus"), facts.get("cpu_vendor")
have = {f.lower() for f in facts.get("cpu_flags") or []} # docs write AVX512F, cpuinfo avx512f
decided = cpus is not None and bool(vendor or have) # no cpuinfo vendor or flags: a miss would be a guess
fields: list[dict[str, Any]] = []
if need.min_cpus > 1:
fields.append({"field": "min_cpus", "required": need.min_cpus, "actual": cpus,
"met": cpus >= need.min_cpus if cpus is not None else None})
if need.cpu_vendors:
ok = (vendor or "").lower() in {v.lower() for v in need.cpu_vendors} if decided else None
fields.append({"field": "cpu_vendors", "required": list(need.cpu_vendors), "actual": vendor, "met": ok})
for flag in need.cpu_flags:
ok = flag.lower() in have if decided else None
fields.append({"field": f"cpu_flags.{flag}", "required": flag, "actual": ok, "met": ok})
return fields


def _container_facts(ctx: EvalContext, work_dir: Path) -> dict[str, Any]:
"""The probe container's facts, or ``{"error": ...}`` when it could not run."""
work_dir.mkdir(parents=True, exist_ok=True)
out, err = work_dir / "host-probe.out", work_dir / "host-probe.err"
# HOME is the probe container's own /tmp, never a host path.
Expand All @@ -197,7 +257,7 @@ def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]:
try:
result = ctx.runner.run(spec, stdout=out, stderr=err, timeout_s=PROBE_TIMEOUT_S)
except ContainerError as exc:
return {"error": f"host probe did not start: {str(exc)[:300]}"}
return {"error": scrub(f"host probe did not start: {str(exc)[:300]}")}
text = out.read_text(encoding="utf-8", errors="replace") if out.is_file() else ""
try:
raw = json.loads(text.strip().splitlines()[-1]) if result.ok and text.strip() else None
Expand All @@ -206,7 +266,7 @@ def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]:
if not isinstance(raw, dict):
tail = (err.read_text(encoding="utf-8", errors="replace") if err.is_file() else "").strip()[-300:]
how = "timed out" if result.timed_out else f"exited {result.returncode}"
return {"error": f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}"}
return {"error": scrub(f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}")}
Comment thread
napetrov marked this conversation as resolved.
Resolved
return host_facts(raw)


Expand Down
Loading
Loading