From 2ee29fdc84044d7053223a44bcb719c1bdb87531 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 22:03:37 -0800 Subject: [PATCH 01/16] feat(repo): fingerprint the host with collectors; --label-host runs a missed host The host probe now records a fingerprint from a registry of collectors (os, cpu, memory, gpu, python, tools, network, virt) under `collectors`, beside the container's unchanged keys. A failing collector reports its error and the others still run. Nothing identifying (host or user name, address, env value, interface name, absolute path) is recorded. `repo run --label-host` runs a task the host misses instead of marking it unsupported, and records per-requirement `host_match` in result.json and meta.json. `repo report` groups rows by host fingerprint. Block mode is the default and unchanged. Signed-off-by: Nikolay Petrov --- ...00_napetrov_host-fingerprint-collectors.md | 4 + dokimi/docs/reference/cli-interface.json | 1 + dokimi/docs/repo-evaluation-contract.md | 23 +- dokimi/dokimi/commands/repo_run.py | 6 +- dokimi/dokimi/repos/host.py | 71 ++++- dokimi/dokimi/repos/host_fingerprint.py | 246 +++++++++++++++ dokimi/dokimi/repos/report.py | 45 ++- dokimi/dokimi/repos/runner.py | 40 ++- dokimi/tests/data/cli_flags_allowlist.txt | 1 + dokimi/tests/test_batch_ledger_golden.py | 1 + dokimi/tests/test_host_fingerprint.py | 293 ++++++++++++++++++ dokimi/tests/test_repo_host_label.py | 177 +++++++++++ 12 files changed, 875 insertions(+), 33 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md create mode 100644 dokimi/dokimi/repos/host_fingerprint.py create mode 100644 dokimi/tests/test_host_fingerprint.py create mode 100644 dokimi/tests/test_repo_host_label.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md new file mode 100644 index 000000000..1312d64eb --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md @@ -0,0 +1,4 @@ +### Added + +- **The host probe fingerprints the machine with a registry of collectors** — `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`, each a flat dict under `collectors` (`collectors_version` 1), beside the container's unchanged keys. A failing collector reports `{"error": ...}` and the others still run; no host or user name, IP address, environment value or absolute path is recorded. +- **`repo run --label-host` runs a task the host misses instead of blocking it** — each cell records `host_match` (per requirement met, unmet or unknown) in `result.json` and `meta.json`, and `repo report` groups results by host fingerprint. Without the flag nothing changes. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..fafb5f40e 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1388,6 +1388,7 @@ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--image", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--jobs/-j", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--label-host", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--manifest", "action": "_StoreAction", "choices": null, "const": null, "default": ".agent-eval/repo-eval.yaml", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--manifest-file", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--max-batch-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.repo_run._usd#591d6e2f0474"}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c1c0de52..4cd7d5248 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -329,7 +329,7 @@ belongs to the repository's grader and is passed through unread, except `grader` | `gate` | no | framework | what counts as success; default `["pass"]`. Each entry is a boolean metric name, which must be `true`, or `{"metric": , "min": }` for a numeric metric | | `score` | for score-only tasks | framework | `{"metric": , "max": }`: the metric that is the result when `pass` is `null`, and the value the oracle must reach (§4.3) | | `timeout_s`, `max_turns`, `budget_usd` | no | framework | per-task caps, overriding run defaults; each a finite number > 0 | -| `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | +| `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), unless `repo run --label-host` runs it and records the miss, and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | | `egress` | no | framework | extra hosts (DNS names, optional `:port`; `*` stands for one whole label and the last two labels are literal, so `*.pythonhosted.org` passes and `*` or `*.org` does not) this task's agent and entry points reach through the proxy (§2), for example `["pypi.org", "files.pythonhosted.org"]` | | `expected_skill` | no | framework | the name (directory name) of the skill the agent should use; a `skill-catalog:` arm must install it, or the arm is refused before anything runs. On a `guidance:as-skill` catalog arm the generated `-guidance` skill is refused too (its usage is `guidance_skill_loaded`). `result.json` `skills` records whether the agent used it and whether it used it first | | `useful` | no | framework | the definition of the run's first useful result, reported as `first_useful` in `result.json` (§4.1): `{"command": ""}` for a task whose grade runs the target tests, or `{"edit": ""}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` | @@ -697,6 +697,27 @@ apart, neither failed nor missing. If the probe itself fails, a task with `host` requirements is not run either, and `repo run` exits 1 when one of its cells is left unrun; a cell that already has a `result.json` keeps it. +The probe also fingerprints the host (`dokimi/repos/host_fingerprint.py`, `collectors_version` 1). +Its collectors are `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`; each is a +flat dict of JSON scalars under `collectors` in the probe's facts, beside the container's own keys +(`cpus`, `cpu_vendor`, ...), which are unchanged. A collector that fails reports `{"error": ...}` +and the others still run. Linux reads `/proc`, `/sys` and `/etc/os-release`; macOS reads `sysctl`; +Windows reads the kernel's memory status and spawns only `git --version`, `nvidia-smi` and the other +tools' `--version` calls when they are on the path. `gpu` is NVIDIA only. `virt` reads a container +from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from DMI or the `hypervisor` +CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value, +interface name or absolute path; an error text is cut to 300 characters with its paths replaced. +`fingerprint_id` hashes the CPU and identity collectors but not `network`, so a bridge that appears +between runs does not split a batch. + +`repo run --label-host` runs a task that the host misses, instead of marking it unsupported. Each +cell's `result.json` and `meta.json` hold `host_match`: the `fingerprint`, the report's `met`, and one +field per declared requirement with `met` true, false, or null where the probe could not tell. The +report's `met` is false if any field is, else null if any is undecided, else true. The requirements +are checked against the grading container's CPU view, as in §3, not against the host's collectors, +which only enter the fingerprint. `repo report` groups rows by `host_fingerprint` and notes a +labelled miss ("labelled, unmet: cpu_flags.avx512f"). Without the flag, nothing changes. + Both commands exit 2, before any build, on a source that does not parse (a local path that is not a directory, an unpinned remote) and on a pinned local path inside a repository that is not its root; a source that parses but cannot be opened (clone, fetch, revision) exits 1 with a `✗` line. A manifest that diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 3f9368856..5d6bccd78 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -361,7 +361,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock if skill_pin is not None: write_skill_pin(batch_dir, skill_pin) summary = run_batch(ctx, tasks, arms, options, args.reps, batch_dir, jobs=args.jobs, force=args.force, - max_batch_usd=args.max_batch_usd, log=_progress) + max_batch_usd=args.max_batch_usd, log=_progress, label_host=args.label_host) except _errors() as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) @@ -552,6 +552,10 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str run_p.add_argument("--max-batch-usd", type=_usd, default=None, dest="max_batch_usd", help="Start no new cell once this invocation has spent this much") run_p.add_argument("--force", action="store_true", help="Rerun cells that already have result.json") + run_p.add_argument("--label-host", action="store_true", dest="label_host", + help="Run a task whose task.json `host` requirements this host misses anyway, and record " + "per field what was met and what was not (host_match in result.json) instead of marking " + "the cells unsupported; results carry the host fingerprint, so `repo report` groups them by it") run_p.set_defaults(func=cmd_repo_run) report_p = repo_sub.add_parser("report", help="Per (task, arm, model) table over a batch directory") diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index 4c885e1d6..6d96dcb7e 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -23,15 +23,16 @@ from __future__ import annotations +import hashlib import json import math -import re import uuid from pathlib import Path from typing import Any from .container import ContainerError, ContainerSpec from .grading import EvalContext +from .host_fingerprint import IDENTITY_COLLECTORS, Probe, collect, cpuinfo_field, scrub from .host_spec import HOST_KEYS, KNOWN_VENDORS, HostRequirements, HostSpecError, parse_host # noqa: F401 from .useful_spec import parse_useful from .workspace import Task @@ -171,23 +172,71 @@ def host_facts(raw: dict[str, Any]) -> dict[str, Any]: quota = cgroup_cpus(raw) counts = [n for n in (affinity, quota) if n] text = str(raw["cpuinfo"]) if isinstance(raw.get("cpuinfo"), str) else "" - flags = _field(text, "flags") or _field(text, "Features") # x86 | arm64 + flags = cpuinfo_field(text, "flags") or cpuinfo_field(text, "Features") # x86 | arm64 return {"cpus": min(counts) if counts else None, "affinity_cpus": affinity, "cgroup_cpus": quota, - "cpu_vendor": _field(text, "vendor_id"), "cpu_model": _field(text, "model name"), + "cpu_vendor": cpuinfo_field(text, "vendor_id"), "cpu_model": cpuinfo_field(text, "model name"), "cpu_flags": sorted(set(flags.split())) if flags else []} -def _field(text: str, name: str) -> str | None: - match = re.search(rf"^{re.escape(name)}\s*:\s*(.*)$", text, re.M) - return match.group(1).strip() if match else None +def probe_host(ctx: EvalContext, work_dir: Path, probe: Probe | None = None) -> dict[str, Any]: + """Facts of a container like the grading one, plus the host fingerprint (:mod:`.host_fingerprint`). + The container's facts are the top-level keys, ``{"error": ...}`` in place of them when the probe + could not run. The spec carries what decides the CPU view: the evaluation's image and runner + (whose flags every container shares); no mounts, no network. + """ + return {**_container_facts(ctx, work_dir), **collect(probe)} + + +def fingerprint_id(facts: dict[str, Any]) -> str: + """A short id of the host: the same machine gives the same id on every run (see ``IDENTITY_COLLECTORS``).""" + keys = ("cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags") + identity = {**{k: facts.get(k) for k in keys}, + **{name: (facts.get("collectors") or {}).get(name) for name in IDENTITY_COLLECTORS}} + return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()[:12] -def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: - """Facts of a container like the grading one, or ``{"error": ...}`` when the probe could not run. - The spec carries what decides the CPU view: the evaluation's image and runner (whose flags - every container shares); no mounts, no network. +def match_report(task: Task, facts: dict[str, Any] | None) -> dict[str, Any]: + """Per declared requirement, whether this host meets it, for a run that labels a miss instead of blocking it. + + ``met`` is ``True`` or ``False`` per field, ``None`` when the probe could not tell (the rule of :func:`unmet`); + the report's ``met`` is ``False`` if any field is, else ``None`` if any is undecided, else ``True``. """ + facts = facts or {} + need = requirements(task) + fields = _match_fields(need, facts) if need is not None else [] + verdicts = {f["met"] for f in fields} + met = False if False in verdicts else None if None in verdicts else True + return {"fingerprint": fingerprint_id(facts), "met": met, "fields": fields} + + +def match_summary(match: dict[str, Any]) -> str: + """The fields a labelled run's host misses, or those it could not tell: ``cpu_flags.avx512f missing``.""" + missed = [f"{f['field']} missing" for f in match["fields"] if f["met"] is False] + undecided = [f"{f['field']} unknown" for f in match["fields"] if f["met"] is None] + return ", ".join(missed + undecided) or "nothing unmet" + + +def _match_fields(need: HostRequirements, facts: dict[str, Any]) -> list[dict[str, Any]]: + """One entry per declared requirement; undecided (``met`` None) where :func:`unmet` says the probe cannot tell.""" + cpus, vendor = facts.get("cpus"), facts.get("cpu_vendor") + have = {f.lower() for f in facts.get("cpu_flags") or []} # docs write AVX512F, cpuinfo avx512f + decided = cpus is not None and bool(vendor or have) # no cpuinfo vendor or flags: a miss would be a guess + fields: list[dict[str, Any]] = [] + if need.min_cpus > 1: + fields.append({"field": "min_cpus", "required": need.min_cpus, "actual": cpus, + "met": cpus >= need.min_cpus if cpus is not None else None}) + if need.cpu_vendors: + ok = (vendor or "").lower() in {v.lower() for v in need.cpu_vendors} if decided else None + fields.append({"field": "cpu_vendors", "required": list(need.cpu_vendors), "actual": vendor, "met": ok}) + for flag in need.cpu_flags: + ok = flag.lower() in have if decided else None + fields.append({"field": f"cpu_flags.{flag}", "required": flag, "actual": ok, "met": ok}) + return fields + + +def _container_facts(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: + """The probe container's facts, or ``{"error": ...}`` when it could not run.""" work_dir.mkdir(parents=True, exist_ok=True) out, err = work_dir / "host-probe.out", work_dir / "host-probe.err" # HOME is the probe container's own /tmp, never a host path. @@ -206,7 +255,7 @@ def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: if not isinstance(raw, dict): tail = (err.read_text(encoding="utf-8", errors="replace") if err.is_file() else "").strip()[-300:] how = "timed out" if result.timed_out else f"exited {result.returncode}" - return {"error": f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}"} + return {"error": scrub(f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}")} return host_facts(raw) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py new file mode 100644 index 000000000..270c4bc50 --- /dev/null +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -0,0 +1,246 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The host fingerprint: what the machine that runs dokimi is, one collector per field group. + +Each collector returns a flat dict of JSON scalars. A collector that fails returns ``{"error": ...}`` +and the others still run, so one unreadable file never costs the probe. Nothing that identifies a +person or a machine is read or recorded: no host name, user name, IP address, environment value or +absolute path (an error message is scrubbed of paths before it is kept). Spawns go through +:mod:`dokimi.process`, and only where the platform has the tool: nothing is shelled to on Windows +except the cross-platform ``--version`` calls and ``nvidia-smi`` when it is on the path. +""" + +from __future__ import annotations + +import os +import platform +import re +import shutil +import socket +import sysconfig +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from dokimi import process +from dokimi.process import Env, Runner + +COLLECTORS_VERSION = 1 +SPAWN_TIMEOUT_S = 10 +# Only what a spawned tool needs from the environment: the binary's search path, and on Windows its system root. +_SPAWN_ENV = ("PATH", "SYSTEMROOT") +# The collectors whose values decide a host's identity (the id of `fingerprint_id`); network state is not one: +# a docker bridge that appears between two runs must not split a batch. +IDENTITY_COLLECTORS = ("os", "cpu", "memory", "gpu", "python", "tools", "virt") +TOOLS = ("git", "docker", "cmake", "gcc", "clang", "make") # version is the first number on the first line of `--version` +PROXY_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy") +# DMI strings of hypervisors, matched in lower case against the vendor and product name. +VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), + ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) +_PATH = re.compile(r"(?:/[\w.+@%-]+)+|[A-Za-z]:\\\S*") + + +def _read_text(path: str) -> str | None: + try: + with open(path, encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return None + + +def _windows_total_memory() -> int: + """Physical memory of a Windows host, from the kernel's ``GlobalMemoryStatusEx`` (no shell).""" + import ctypes # Windows only: the windll attribute does not exist elsewhere + + class MemoryStatus(ctypes.Structure): + _fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong), ("total_phys", ctypes.c_ulonglong), + ("avail_phys", ctypes.c_ulonglong), ("total_page", ctypes.c_ulonglong), + ("avail_page", ctypes.c_ulonglong), ("total_virtual", ctypes.c_ulonglong), + ("avail_virtual", ctypes.c_ulonglong), ("avail_extended", ctypes.c_ulonglong)] + + status = MemoryStatus(length=ctypes.sizeof(MemoryStatus)) + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)) # type: ignore[attr-defined] + return int(status.total_phys) + + +@dataclass(frozen=True) +class Probe: + """What the collectors read and run. Tests pass fakes; the defaults read this machine.""" + + system: str = field(default_factory=platform.system) + read: Callable[[str], str | None] = _read_text + which: Callable[[str], str | None] = shutil.which + run: Runner = process.run + env: Mapping[str, str] = field(default_factory=lambda: dict(os.environ)) # read for booleans only + windows_memory: Callable[[], int] = _windows_total_memory + + +def _spawn(probe: Probe, argv: list[str]) -> str | None: + """stdout of a command that exits 0, else ``None``: a tool that fails is a fact about the host, not an error.""" + done = probe.run(argv, timeout=SPAWN_TIMEOUT_S, env=Env.only(_SPAWN_ENV)) + return done.stdout if done.ok else None + + +def scrub(text: str) -> str: + """``text`` with every absolute path replaced: an error message may name a file under the user's home.""" + return _PATH.sub("", text)[:300] + + +def cpuinfo_field(text: str, name: str) -> str | None: + match = re.search(rf"^{re.escape(name)}\s*:\s*(.*)$", text, re.M) + return match.group(1).strip() if match else None + + +def _os(probe: Probe) -> dict[str, Any]: + out: dict[str, Any] = {"system": probe.system, "release": platform.release(), "machine": platform.machine()} + if probe.system == "Linux": + out["distro"] = _os_release(probe) + elif probe.system == "Darwin": + out["macos_version"] = platform.mac_ver()[0] or None + return out + + +def _os_release(probe: Probe) -> str | None: + text = probe.read("/etc/os-release") or probe.read("/usr/lib/os-release") or "" + match = re.search(r'^PRETTY_NAME="?([^"\n]*)"?', text, re.M) + return match.group(1) if match else None + + +def _cpu(probe: Probe) -> dict[str, Any]: + """The machine's CPU (the container's view is in the probe's top-level keys, from its cgroup).""" + out: dict[str, Any] = {"logical_cpus": os.cpu_count(), "vendor": None, "model": None, "flags": None} + if probe.system == "Linux": + text = probe.read("/proc/cpuinfo") or "" + out.update(vendor=cpuinfo_field(text, "vendor_id"), model=cpuinfo_field(text, "model name"), + flags=cpuinfo_field(text, "flags") or cpuinfo_field(text, "Features")) + elif probe.system == "Darwin": + out.update(model=_sysctl(probe, "machdep.cpu.brand_string"), vendor=_sysctl(probe, "machdep.cpu.vendor")) + features = _sysctl(probe, "machdep.cpu.features") + out["flags"] = features.lower() if features else None + elif probe.system == "Windows": + out["model"] = platform.processor() or None # the kernel's string names the vendor, read from it + vendor = re.search(r"GenuineIntel|AuthenticAMD", out["model"] or "") + out["vendor"] = vendor.group(0) if vendor else None + return out + + +def _sysctl(probe: Probe, name: str) -> str | None: + value = _spawn(probe, ["sysctl", "-n", name]) + return value.strip() if value and value.strip() else None + + +def _memory(probe: Probe) -> dict[str, Any]: + if probe.system == "Linux": + match = re.search(r"^MemTotal:\s+(\d+) kB", probe.read("/proc/meminfo") or "", re.M) + return {"total_bytes": int(match.group(1)) * 1024} if match else {"error": "no MemTotal line in the meminfo file"} + if probe.system == "Darwin": + value = _sysctl(probe, "hw.memsize") + return {"total_bytes": int(value)} if value and value.isdigit() else {"error": "sysctl hw.memsize gave no number"} + if probe.system == "Windows": + return {"total_bytes": probe.windows_memory()} + return {"error": f"no memory collector for {probe.system}"} + + +def _gpu(probe: Probe) -> dict[str, Any]: + """NVIDIA only: ``nvidia-smi`` is run when it is on the path; other GPUs report ``present`` false.""" + if probe.which("nvidia-smi") is None: + return {"present": False} + rows = _spawn(probe, ["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"]) + if rows is None: + return {"present": True, "error": "nvidia-smi --query-gpu=name,driver_version failed"} + table = [[cell.strip() for cell in line.split(",")] for line in rows.splitlines() if line.strip()] + banner = _spawn(probe, ["nvidia-smi"]) or "" + cuda = re.search(r"CUDA Version:\s*(\d+(?:\.\d+)+)", banner) + first = table[0] if table else [] + return {"present": True, "count": len(table), "name": first[0] if first else None, + "driver_version": first[1] if len(first) > 1 else None, "cuda_version": cuda.group(1) if cuda else None} + + +def _python(probe: Probe) -> dict[str, Any]: + return {"implementation": platform.python_implementation(), "version": platform.python_version(), + "free_threaded": bool(sysconfig.get_config_var("Py_GIL_DISABLED"))} + + +def _tools(probe: Probe) -> dict[str, str | None]: + """Per tool: its version, ``present`` when it runs without a readable version, ``None`` when absent.""" + out: dict[str, str | None] = {} + for name in TOOLS: + if probe.which(name) is None: + out[name] = None + continue + text = _spawn(probe, [name, "--version"]) or "" + number = re.search(r"\d+(?:\.\d+)+", text.split("\n", 1)[0]) + out[name] = number.group(0) if number else "present" + return out + + +def _network(probe: Probe) -> dict[str, Any]: + """Counts and booleans only: no interface name, address or proxy value.""" + return {"interfaces": len(socket.if_nameindex()), "default_route": _default_route(probe), + "proxy_env": any(probe.env.get(name) for name in PROXY_VARS)} + + +def _default_route(probe: Probe) -> bool | None: + if probe.system != "Linux": + return None + table = probe.read("/proc/net/route") + if table is None: + return None + return any(line.split()[1:2] == ["00000000"] for line in table.splitlines()[1:] if len(line.split()) > 1) + + +def _virt(probe: Probe) -> dict[str, Any]: + kind = _container_kind(probe) + vm, vendor = _vm(probe) + return {"container": kind is not None, "container_kind": kind, "vm": vm, "vm_vendor": vendor} + + +def _container_kind(probe: Probe) -> str | None: + """``docker``, ``podman``, ``kubernetes`` or ``lxc`` when this process runs in one; ``None`` otherwise or unknown.""" + if probe.system != "Linux": + return None + if probe.read("/.dockerenv") is not None: + return "docker" + if probe.read("/run/.containerenv") is not None: + return "podman" + cgroup = probe.read("/proc/self/cgroup") or "" + for marker, kind in {"kubepods": "kubernetes", "libpod": "podman", "lxc": "lxc", "docker": "docker"}.items(): + if marker in cgroup: + return kind + return None + + +def _vm(probe: Probe) -> tuple[bool | None, str | None]: + """``(is a VM, its hypervisor)``; ``None`` when this platform cannot tell.""" + if probe.system == "Darwin": + value = _sysctl(probe, "kern.hv_vmm_present") + return (None, None) if value is None else (value == "1", None) + if probe.system != "Linux": + return None, None + dmi = " ".join(probe.read(f"/sys/class/dmi/id/{name}") or "" for name in ("sys_vendor", "product_name")).lower() + for needle, name in VM_MARKERS: + if needle in dmi: + return True, name + flags = cpuinfo_field(probe.read("/proc/cpuinfo") or "", "flags") or "" + if "hypervisor" in flags.split(): + return True, None + return (False if dmi.strip() else None), None + + +COLLECTORS: dict[str, Callable[[Probe], dict[str, Any]]] = { + "os": _os, "cpu": _cpu, "memory": _memory, "gpu": _gpu, "python": _python, + "tools": _tools, "network": _network, "virt": _virt, +} + + +def collect(probe: Probe | None = None) -> dict[str, Any]: + """``{"collectors_version", "collectors": {name: flat dict}}``; a collector that raises is ``{"error": ...}``.""" + probe = probe or Probe() + out: dict[str, Any] = {} + for name, collector in COLLECTORS.items(): + try: + out[name] = collector(probe) + except Exception as exc: # noqa: BLE001 - one collector failing must not fail the probe + out[name] = {"error": scrub(f"{type(exc).__name__}: {exc}")} + return {"collectors_version": COLLECTORS_VERSION, "collectors": out} diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index e9901dfec..9e1fc635a 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -210,18 +210,40 @@ def load_unsupported(batch_dir: Path) -> list[dict[str, Any]]: return out +def _labelled_match(recs: list[dict[str, Any]]) -> dict[str, Any]: + """``{"host_match": ...}`` for a labelled row: which of the task's host requirements its host missed.""" + labelled = [r["host_match"] for r in recs if isinstance(r.get("host_match"), dict)] + if not labelled: + return {} + return {"host_match": {"met": labelled[0]["met"], + "unmet": [f["field"] for f in labelled[0]["fields"] if f["met"] is False]}} + + +def _skill_counts(recs: list[dict[str, Any]]) -> dict[str, Any]: + used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)] + if not used: # cells whose trace was unreadable are not counted: no rate from a missing measurement + return {} + out: dict[str, Any] = {"skill_measured": len(used), "skill_invoked": sum(bool(u["skill_invoked"]) for u in used)} + expecting = [u for u in used if u.get("expected_skill")] # a task naming no skill has nothing to select + if expecting: + out["skill_expected"] = len(expecting) + out["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting) + return out + + def summarize(batch_dir: Path) -> dict[str, Any]: cells, missing = load_cells(batch_dir) - groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + # One row per (task, arm, model) and host fingerprint: cells run on different hosts never pool in a row. + groups: dict[tuple[str, str, str, str | None], list[dict[str, Any]]] = defaultdict(list) for rec in cells: - groups[(rec["task"], rec["arm"], rec["model"])].append(rec) + groups[(rec["task"], rec["arm"], rec["model"], rec.get("host_fingerprint"))].append(rec) missing_by: dict[tuple[str, str, str], int] = defaultdict(int) for name in missing: parts = name.split("__") if len(parts) == 4: missing_by[(parts[0], parts[1], parts[2])] += 1 rows = [] - for key in sorted(groups): + for key in sorted(groups, key=lambda k: (k[0], k[1], k[2], k[3] or "")): recs = groups[key] with reading_cells([str(batch_dir / r["cell"] / "result.json") for r in recs]): n = len(recs) @@ -233,7 +255,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "agent_errors": sum(bool(r["agent"]["is_error"]) for r in recs), "over_budget": sum(bool(r["agent"]["over_budget"]) for r in recs), "cost_unknown": sum(r["agent"]["cost_usd"] is None for r in recs), - "missing": missing_by.get((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0), + "missing": missing_by.pop((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0), "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]), "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6), "turns": _mean([r["agent"]["num_turns"] for r in recs]), @@ -241,6 +263,8 @@ def summarize(batch_dir: Path) -> dict[str, Any]: **{k: _mean([r["agent"].get(k) for r in recs]) for k in _TOKENS}, "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} + row["host_fingerprint"] = key[3] + row.update(_labelled_match(recs)) uptake = uptake_counts(recs) if uptake is not None: row["uptake"] = uptake @@ -253,14 +277,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: else: wins = sum(bool(r["grade"]["success"]) for r in recs) row.update(kind="gate", successes=wins, success_rate=round(wins / n, 4), ci95=wilson(wins, n)) - used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)] - if used: # cells whose trace was unreadable are not counted: no rate from a missing measurement - row["skill_measured"] = len(used) - row["skill_invoked"] = sum(bool(u["skill_invoked"]) for u in used) - expecting = [u for u in used if u.get("expected_skill")] # a task naming no skill has nothing to select - if expecting: - row["skill_expected"] = len(expecting) - row["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting) + row.update(_skill_counts(recs)) row["parse_errors"] = sum(r["grade"]["status"] != "grader_failure" # counted in grader_failures and _metrics(batch_dir / r["cell"]).get("parse_error") is True for r in recs) rows.append(row) @@ -319,6 +336,10 @@ def _row_notes(r: dict[str, Any]) -> list[str]: if "uptake" in r: notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)" + unmeasured_note(r["uptake"])) + if "host_match" in r: + match = r["host_match"] + what = "met" if match["met"] is True else f"unmet: {', '.join(match['unmet']) or 'undecided'}" + notes.append(f"host {r['host_fingerprint']} (labelled, {what})") return notes diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a5809f29a..2fcff5579 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -74,7 +74,7 @@ from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile -from .host import probe_host, unmet, validate_tasks +from .host import fingerprint_id, match_report, match_summary, probe_host, unmet, validate_tasks from .manifest import match_glob, scratch_name from .process_metrics import process_metrics from .workspace import OVERLAY_ARM, PrepError, Task, parse_arm, prepare_workspace @@ -203,10 +203,12 @@ def _workspace_args(task: Task, opts: AgentOptions, rd: Path, meta: dict[str, An def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, opts: AgentOptions, - force: bool = False, host: dict[str, Any] | None = None) -> dict[str, Any]: + force: bool = False, host: dict[str, Any] | None = None, + match: dict[str, Any] | None = None) -> dict[str, Any]: """Prepare, run the agent, grade. Returns the cell's ``result.json`` (``skipped`` when it existed). - ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``.""" + ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``; ``match`` (a + labelled run's :func:`.host.match_report`) too, and in ``result.json`` as ``host_match``.""" rd = batch_dir / cell_name(task.id, arm, opts.model, rep) execution = execution_record(opts, arm) done = batch_ledger.reused_result(rd, execution, force) @@ -221,6 +223,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) + if match is not None: + meta["host_match"] = match present = _workspace_args(task, opts, rd, meta) _write_json(rd / "meta.json", meta) skills: dict[str, Any] | None = None @@ -298,7 +302,10 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, - "process": process, "first_useful": first_useful(task.useful, useful_events), "grade": grade} + "process": process, "first_useful": first_useful(task.useful, useful_events), "grade": grade, + "host_fingerprint": fingerprint_id(host) if host is not None else None} + if match is not None: + record["host_match"] = match _add_arm_use(record, meta, task, skills, uptake_of, events) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away @@ -361,21 +368,37 @@ def _clear_unsupported_markers(batch_dir: Path, cells: list, unsupported: dict) (batch_dir / cell_name(t.id, a, o.model, r) / UNSUPPORTED_FILE).unlink(missing_ok=True) +def _verdicts(tasks: list[Task], host: dict[str, Any], label_host: bool) -> tuple[dict, dict]: + """``(unmet verdicts, match reports)``: a block-mode batch has the verdicts, a labelled one the reports.""" + if label_host: + return {}, {t.id: match_report(t, host) for t in tasks} + return {t.id: unmet(t, host) for t in tasks}, {} + + +def _log_labelled(matches: dict[str, dict[str, Any]], log) -> None: + for task_id, match in matches.items(): + if match["met"] is not True: + log(f"{task_id}: host {match['fingerprint']} is labelled, not blocked: {match_summary(match)}") + + def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: list[AgentOptions], reps: int, batch_dir: Path, jobs: int = 1, force: bool = False, max_batch_usd: float | None = None, - log=print) -> dict[str, Any]: + log=print, label_host: bool = False) -> dict[str, Any]: """Every (task, arm, model, rep) cell; returns counts. Cells that crash are reported, not raised. The host is probed once, before any cell (``_eval/host.json``); a cell of a task whose ``host`` requirements it does not meet is marked unsupported instead of run. One that already has a ``result.json`` (from a host that met them) keeps it, even with ``force``; one that cannot be read is - that cell's error.""" + that cell's error. + + ``label_host`` runs every cell whatever the host: a cell of a task whose requirements it misses runs and + records its :func:`.host.match_report` (``host_match`` in ``meta.json`` and ``result.json``) instead.""" validate_tasks(tasks) cells = [(t, a, o, r) for t in tasks for a in arms for o in options for r in range(1, reps + 1)] check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells} if force else None) host = probe_host(ctx, batch_dir / "_eval") _write_json(batch_dir / "_eval" / "host.json", host) - verdicts = {t.id: unmet(t, host) for t in tasks} + verdicts, matches = _verdicts(tasks, host, label_host) unsupported = {task_id: why for task_id, (why, _) in verdicts.items() if why} if force and unsupported: # --force keeps their finished cells, so those must match this run too check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells @@ -384,6 +407,7 @@ def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: lis probe_failed = _probe_failed(batch_dir, cells, verdicts) for task_id, why in unsupported.items(): log(f"{task_id}: UNSUPPORTED, its cells are not run: {why}") + _log_labelled(matches, log) _clear_unsupported_markers(batch_dir, cells, unsupported) lock = threading.Lock() state: dict[str, Any] = {"spent": 0.0, "ran": 0, "skipped": 0, "errors": [], "budget_stopped": 0, @@ -408,7 +432,7 @@ def one(cell: tuple[Task, str, AgentOptions, int]) -> None: state["budget_stopped"] += 1 return try: - rec = run_cell(ctx, task, arm, rep, batch_dir, opts, force=force, host=host) + rec = run_cell(ctx, task, arm, rep, batch_dir, opts, force=force, host=host, match=matches.get(task.id)) except CellCrash as exc: with lock: state["spent"] += exc.charged_usd diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..af96ea6e7 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -265,3 +265,4 @@ --canaries-file # repo security: the same canaries one per line (#360) --tool # skill novelty ground (#472): the command whose --help is the source; no registry concept yet --tool-version # skill novelty ground (#472): recorded with the help snapshot; no registry concept yet +--label-host # host label mode (contract §4.3) has no registry concept yet; F1b moves it with --allow-unsupported diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index ac45bf111..8a6f9eed6 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -108,6 +108,7 @@ "grade", "harness", "harness_version", + "host_fingerprint", "model", "model_resolved", "process", diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py new file mode 100644 index 000000000..9ac874614 --- /dev/null +++ b/dokimi/tests/test_host_fingerprint.py @@ -0,0 +1,293 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The host fingerprint's collectors (contract §3): per platform, failure isolation, privacy. + +No docker and no real tool is run: spawns go to a :class:`FakeRunner`, files come from a dict. +The privacy tests also run the collectors on this machine, with its real host and user names. +""" + +from __future__ import annotations + +import getpass +import json +import os +import platform +import re +import socket +import sys +from pathlib import Path + +import pytest + +from dokimi.process import FakeRunner, reply +from dokimi.repos import host_fingerprint as hf +from dokimi.repos.host import fingerprint_id, match_report, probe_host +from dokimi.repos.container import ContainerResult +from dokimi.repos.grading import EvalContext +from dokimi.repos.manifest import parse_manifest +from dokimi.repos.workspace import Task +from repo_eval_fixture import MANIFEST + +INTEL_CPUINFO = ("processor\t: 0\nvendor_id\t: GenuineIntel\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n" + "flags\t\t: fpu sse2 avx2 avx512f hypervisor\n") +ROUTE = ("Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\n" + "eth0\t00000000\t0102A8C0\t0003\t0\t0\t100\t00000000\n") + + +def _files(**paths: str) -> dict[str, str]: + """A fake filesystem: ``_files(**{"/proc/meminfo": ...})`` keys are paths, written with ``__`` for ``/``.""" + return {p.replace("__", "/"): text for p, text in paths.items()} + + +def _probe(system: str = "Linux", files: dict[str, str] | None = None, calls: dict | None = None, **kw): + """A :class:`Probe` whose files are ``files`` and whose spawns answer ``calls[argv[0]]`` (or nothing).""" + files = files or {} + answers = calls or {} + + def handler(call): + key = " ".join(call.argv) + if key in answers: + return reply(answers[key]) + return reply(returncode=1) + + runner = FakeRunner(handler) + kw.setdefault("which", lambda name: None) + return hf.Probe(system=system, read=lambda path: files.get(path), run=runner, + env=kw.pop("env", {}), **kw), runner + + +def _leaves(value): + if isinstance(value, dict): + for item in value.values(): + yield from _leaves(item) + else: + yield value + + +def _paths_in(text: str) -> list[str]: + """Absolute-looking paths in a serialised record: a slash or drive letter at the start of a token.""" + return re.findall(r"(?:^|[\s\"'=(,])(/[\w.+@%-]|[A-Za-z]:\\)", text) + + +# ------------------------------------------------------------------ the registry + + +def test_each_collector_gives_a_flat_dict_of_json_scalars_and_the_version_is_recorded(): + probe, _ = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO, "/proc__meminfo": "MemTotal: 2048 kB\n", + "/proc__net__route": ROUTE})) + out = hf.collect(probe) + assert out["collectors_version"] == hf.COLLECTORS_VERSION == 1 + assert sorted(out["collectors"]) == sorted(hf.COLLECTORS) == [ + "cpu", "gpu", "memory", "network", "os", "python", "tools", "virt"] + for name, facts in out["collectors"].items(): + assert isinstance(facts, dict), name + for item in facts.values(): + assert item is None or isinstance(item, (str, int, float, bool)), (name, item) + assert out["collectors"]["memory"] == {"total_bytes": 2048 * 1024} + assert out["collectors"]["cpu"]["vendor"] == "GenuineIntel" + + +def test_a_failing_collector_reports_its_error_and_the_others_still_run(monkeypatch): + def broken(probe): + raise OSError(f"[Errno 2] No such file or directory: '{Path.home()}/nvidia/smi'") + + monkeypatch.setitem(hf.COLLECTORS, "gpu", broken) + probe, _ = _probe(files=_files(**{"/proc__meminfo": "MemTotal: 1024 kB\n"})) + out = hf.collect(probe) + assert out["collectors"]["gpu"]["error"].startswith("OSError: ") + assert out["collectors"]["memory"] == {"total_bytes": 1024 * 1024} + assert "nvidia" not in out["collectors"]["gpu"]["error"] and "" in out["collectors"]["gpu"]["error"] + + +def test_probe_host_keeps_the_container_keys_and_adds_the_fingerprint(tmp_path): + """The existing keys (cpus, cpu_vendor, ...) are unchanged; the fingerprint sits beside them.""" + ctx = EvalContext(manifest=parse_manifest(MANIFEST), git_dir=tmp_path, content_dir=tmp_path, image="img:1", + runner=_ContainerAnswer({"affinity": 8, "cpu_max": "max 100000\n", "cfs_quota_us": None, + "cfs_period_us": None, "cpuinfo": INTEL_CPUINFO})) + facts = probe_host(ctx, tmp_path / "w", probe=_probe()[0]) + assert {"cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags"} <= set(facts) + assert facts["cpus"] == 8 and facts["cpu_vendor"] == "GenuineIntel" + assert facts["collectors_version"] == 1 and set(facts["collectors"]) == set(hf.COLLECTORS) + + +class _ContainerAnswer: + """A runner of the probe container that prints ``answer`` as its JSON line.""" + + def __init__(self, answer): + self.answer = answer + + def run(self, spec, *, stdout, stderr, timeout_s): + stdout.write_text(json.dumps(self.answer) + "\n", encoding="utf-8") + return ContainerResult(returncode=0, timed_out=False, wall_s=0.0) + + +# ------------------------------------------------------------------ per platform + + +def test_linux_host_reads_its_files_and_spawns_nothing_it_does_not_need(): + probe, runner = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO, "/proc__meminfo": "MemTotal: 4 kB\n", + "/etc__os-release": 'NAME="Ubuntu"\nPRETTY_NAME="Ubuntu 24.04 LTS"\n'})) + out = hf.collect(probe)["collectors"] + assert out["os"] == {"system": "Linux", "release": platform.release(), "machine": platform.machine(), + "distro": "Ubuntu 24.04 LTS"} + assert out["cpu"]["flags"] == "fpu sse2 avx2 avx512f hypervisor" + assert out["virt"] == {"container": False, "container_kind": None, "vm": True, "vm_vendor": None} + assert runner.argvs() == [] # no tool on the path, so nothing ran + + +def test_macos_reads_sysctl_and_no_proc_file(tmp_path): + answers = {"sysctl -n hw.memsize": "17179869184\n", "sysctl -n machdep.cpu.brand_string": "Apple M2\n", + "sysctl -n kern.hv_vmm_present": "0\n"} + probe, runner = _probe(system="Darwin", calls=answers, files=_files(**{"/proc__meminfo": "MemTotal: 1 kB\n"})) + out = hf.collect(probe)["collectors"] + assert out["memory"] == {"total_bytes": 17179869184} + assert out["cpu"]["model"] == "Apple M2" and out["virt"]["vm"] is False + assert out["os"]["system"] == "Darwin" and "distro" not in out["os"] + assert {argv[0] for argv in runner.argvs()} == {"sysctl"} + + +def test_macos_without_a_number_from_sysctl_is_an_error_for_that_field_only(): + probe, _ = _probe(system="Darwin", calls={"sysctl -n hw.memsize": "not a number\n"}) + assert hf.collect(probe)["collectors"]["memory"] == {"error": "sysctl hw.memsize gave no number"} + + +def test_windows_spawns_only_cross_platform_calls_and_reads_no_proc(monkeypatch): + monkeypatch.setattr(platform, "processor", lambda: "Intel64 Family 6 Model 140, GenuineIntel") + which = lambda name: rf"C:\tools\{name}.exe" if name in ("git", "nvidia-smi") else None # noqa: E731 + calls = {"git --version": "git version 2.45.1.windows.1\n", "nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": + "NVIDIA RTX A4000, 551.23\n", "nvidia-smi": "CUDA Version: 12.4"} + probe, runner = _probe(system="Windows", which=which, calls=calls, windows_memory=lambda: 16 * 2**30, + files=_files(**{"/proc__meminfo": "MemTotal: 1 kB\n"})) + out = hf.collect(probe)["collectors"] + assert out["memory"] == {"total_bytes": 16 * 2**30} + assert out["cpu"]["vendor"] == "GenuineIntel" and out["cpu"]["flags"] is None + assert out["tools"]["git"] == "2.45.1" and out["tools"]["docker"] is None + assert out["gpu"] == {"present": True, "count": 1, "name": "NVIDIA RTX A4000", "driver_version": "551.23", + "cuda_version": "12.4"} + assert out["virt"] == {"container": False, "container_kind": None, "vm": None, "vm_vendor": None} + assert {argv[0] for argv in runner.argvs()} <= {"git", "nvidia-smi"} # never sysctl, lscpu, free, cat + + +@pytest.mark.skipif(sys.platform != "win32", reason="reads the kernel's memory status on Windows only") +def test_windows_memory_is_the_kernels_total(): + assert hf.Probe(system="Windows").windows_memory() > 0 + + +# ------------------------------------------------------------------ gpu, virt, network, tools, python + + +def test_gpu_reports_driver_and_cuda_when_nvidia_smi_is_on_the_path(): + rows = "NVIDIA A100-SXM4-80GB, 550.54.14\nNVIDIA A100-SXM4-80GB, 550.54.14\n" + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": rows, + "nvidia-smi": "| NVIDIA-SMI 550.54.14 Driver Version: 550.54.14 CUDA Version: 12.4 |"} + probe, _ = _probe(calls=calls, which=lambda name: "/usr/bin/nvidia-smi" if name == "nvidia-smi" else None) + assert hf.collect(probe)["collectors"]["gpu"] == { + "present": True, "count": 2, "name": "NVIDIA A100-SXM4-80GB", "driver_version": "550.54.14", + "cuda_version": "12.4"} + + +def test_gpu_without_nvidia_smi_is_absent_and_a_failing_query_is_an_error(): + assert hf.collect(_probe()[0])["collectors"]["gpu"] == {"present": False} + probe, _ = _probe(which=lambda name: "/usr/bin/nvidia-smi") # on the path, but the query exits 1 + assert hf.collect(probe)["collectors"]["gpu"] == { + "present": True, "error": "nvidia-smi --query-gpu=name,driver_version failed"} + + +@pytest.mark.parametrize("files, expected", [ + ({"/.dockerenv": ""}, {"container": True, "container_kind": "docker"}), + ({"/run__.containerenv": ""}, {"container": True, "container_kind": "podman"}), + ({"/proc__self__cgroup": "0::/kubepods/burstable/pod1/abc\n"}, {"container": True, "container_kind": "kubernetes"}), + ({"/proc__self__cgroup": "0::/init.scope\n"}, {"container": False, "container_kind": None}), +]) +def test_virt_detects_a_container_from_its_markers(files, expected): + probe, _ = _probe(files=_files(**files)) + virt = hf.collect(probe)["collectors"]["virt"] + assert {key: virt[key] for key in expected} == expected + + +def test_virt_names_a_vm_from_dmi_or_the_hypervisor_flag(): + dmi = {"/sys__class__dmi__id__sys_vendor": "QEMU\n", "/sys__class__dmi__id__product_name": "Standard PC\n"} + probe, _ = _probe(files=_files(**dmi)) + assert hf.collect(probe)["collectors"]["virt"]["vm_vendor"] == "qemu" + probe, _ = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO})) # `hypervisor` in the flags + assert hf.collect(probe)["collectors"]["virt"]["vm"] is True + probe, _ = _probe(files=_files(**{"/sys__class__dmi__id__sys_vendor": "Dell Inc.\n"})) + assert hf.collect(probe)["collectors"]["virt"]["vm"] is False + + +def test_network_counts_and_booleans_and_never_an_address_or_a_proxy_value(monkeypatch): + monkeypatch.setattr(socket, "if_nameindex", lambda: [(1, "lo"), (2, "eth0")]) + probe, _ = _probe(files=_files(**{"/proc__net__route": ROUTE}), env={"HTTPS_PROXY": "http://10.1.2.3:3128"}) + net = hf.collect(probe)["collectors"]["network"] + assert net == {"interfaces": 2, "default_route": True, "proxy_env": True} + assert "10.1.2.3" not in json.dumps(net) + + +def test_tools_record_the_version_number_and_no_path(): + probe, _ = _probe(which=lambda name: f"/usr/bin/{name}" if name in ("git", "gcc") else None, + calls={"git --version": "git version 2.43.0\n", "gcc --version": "gcc (GCC) 13.2.0\nCopyright"}) + tools = hf.collect(probe)["collectors"]["tools"] + assert tools == {"git": "2.43.0", "docker": None, "cmake": None, "gcc": "13.2.0", "clang": None, "make": None} + probe, _ = _probe(which=lambda name: f"/usr/bin/{name}" if name == "make" else None, calls={}) + assert hf.collect(probe)["collectors"]["tools"]["make"] == "present" # on the path, its version not printed + + +def test_python_names_the_implementation_and_not_the_executable(): + python = hf.collect(_probe()[0])["collectors"]["python"] + assert python["version"] == platform.python_version() and isinstance(python["free_threaded"], bool) + assert sys.executable not in json.dumps(python) + + +# ------------------------------------------------------------------ privacy + + +def test_privacy_real_host_and_user_names_and_addresses_never_appear(): + text = json.dumps(hf.collect(hf.Probe()), sort_keys=True) + for secret in {socket.gethostname(), getpass.getuser(), os.path.expanduser("~")}: + if len(secret) >= 4: + assert secret not in text, "a host or user name leaked into the fingerprint" + assert not re.search(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", text), "an IPv4 address leaked into the fingerprint" + assert _paths_in(text) == [], "an absolute path leaked into the fingerprint" + + +def test_privacy_monkeypatched_identities_and_paths_never_appear(monkeypatch, tmp_path): + monkeypatch.setattr(platform, "node", lambda: "tower-7f3a") + monkeypatch.setattr(socket, "gethostname", lambda: "tower-7f3a") + monkeypatch.setattr(socket, "if_nameindex", lambda: [(1, "lo"), (2, "eth0")]) + monkeypatch.setattr(getpass, "getuser", lambda: "jdoe") + monkeypatch.setenv("HOME", "/home/jdoe") + files = _files(**{"/proc__net__route": ROUTE.replace("eth0", "/home/jdoe/eth0"), + "/proc__cpuinfo": INTEL_CPUINFO, "/etc__os-release": 'PRETTY_NAME="Ubuntu 24.04 LTS"\n', + "/sys__class__dmi__id__sys_vendor": "Lenovo at 10.20.30.40\n"}) + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": "A, 1\n", "nvidia-smi": "CUDA Version: 12.4"} + probe, _ = _probe(files=files, calls=calls, env={"HOME": "/home/jdoe", "HTTP_PROXY": "http://10.20.30.40:3128"}, + which=lambda name: f"/home/jdoe/bin/{name}" if name == "nvidia-smi" else None) + monkeypatch.setattr(hf, "COLLECTORS", {**hf.COLLECTORS, "tools": lambda p: 1 / 0}) # a failing collector's text + text = json.dumps(hf.collect(probe), sort_keys=True) + for leak in ("tower-7f3a", "jdoe", "10.20.30.40", "/home", str(tmp_path)): + assert leak not in text, leak + assert "ZeroDivisionError: division by zero" in text and _paths_in(text) == [] + + +def test_error_text_is_scrubbed_of_paths_and_truncated(): + assert hf.scrub("missing '/home/jdoe/.cache/x' and C:\\Users\\jdoe\\y") == "missing '' and " + assert len(hf.scrub("x" * 1000)) == 300 + + +# ------------------------------------------------------------------ the identity of a host + + +def test_the_fingerprint_id_is_stable_and_ignores_the_network_but_not_the_cpu(): + facts = {"cpus": 4, "cpu_vendor": "GenuineIntel", "cpu_model": "Xeon", "cpu_flags": ["avx2"], + "collectors": {"os": {"release": "6.8"}, "network": {"interfaces": 2}, "virt": {"container": False}}} + moved = {**facts, "collectors": {**facts["collectors"], "network": {"interfaces": 9, "proxy_env": True}}} + assert fingerprint_id(facts) == fingerprint_id(moved) + assert fingerprint_id(facts) != fingerprint_id({**facts, "cpu_model": "Epyc"}) + assert len(fingerprint_id(facts)) == 12 + + +def test_match_report_for_a_host_with_no_task_requirement_is_met_without_fields(): + task = Task("t", {"id": "t"}, Path(".")) + assert match_report(task, {"cpus": 2})["met"] is True + assert match_report(task, {"cpus": 2})["fields"] == [] diff --git a/dokimi/tests/test_repo_host_label.py b/dokimi/tests/test_repo_host_label.py new file mode 100644 index 000000000..2e6a57752 --- /dev/null +++ b/dokimi/tests/test_repo_host_label.py @@ -0,0 +1,177 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo run --label-host`: run a task the host misses, and record which requirements it met (contract §4.3). + +Four host scenarios end to end: an AVX-512 task on an AMD host, a Windows host, a container, and a GPU host. +The host's fingerprint comes from a fake :class:`Probe`; the grading containers are a :class:`FakeRunner`. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.host import host_facts, match_report, match_summary, probe_host as real_probe_host +from dokimi.repos.host_fingerprint import Probe +from dokimi.process import FakeRunner as HostFakeRunner +from dokimi.process import reply +from dokimi.repos.report import format_report, summarize +from dokimi.repos.workspace import Task +from repo_eval_fixture import make_repo +from test_repo_host import AMD_CPUINFO, INTEL_CPUINFO, RACE, HostRunner, _probe_out, _run + +AVX = dict(RACE, host={"cpu_flags": ["avx512f"]}) # no CPU count, no vendor: a flag is the only requirement + + +def _task(host) -> Task: + return Task("t", {"id": "t", "host": host}, Path(".")) + + +def _repo(tmp_path): + root, _ = make_repo(tmp_path, {"fix_avx": AVX, "fix_race": RACE}) + return root + + +def _host(system="Linux", files=None, calls=None, **kw) -> Probe: + """A fake host: ``files`` are what its /proc, /sys and /etc read; ``calls`` answer its spawns.""" + files = files or {} + answers = calls or {} + + def handler(call): + text = answers.get(" ".join(call.argv)) + return reply(text) if text is not None else reply(returncode=1) + + kw.setdefault("which", lambda name: None) + return Probe(system=system, read=lambda path: files.get(path), run=HostFakeRunner(handler), env={}, **kw) + + +def _labelled(monkeypatch, host: Probe) -> None: + """Probe this batch's host with ``host`` (the container probe is still the fake runner's).""" + monkeypatch.setattr("dokimi.repos.runner.probe_host", lambda ctx, work: real_probe_host(ctx, work, probe=host)) + + +def _cell_json(out: Path, task: str, name: str) -> dict: + """The first rep of ``task``'s cell file ``name`` (``result.json`` or ``meta.json``), as an object.""" + return json.loads(next((out / "b1").glob(f"{task}__*__r1/{name}")).read_text(encoding="utf-8")) + + +# ------------------------------------------------------------------ the match report + + +def test_an_avx512_task_on_an_amd_host_is_a_labelled_miss_on_the_flag_alone(): + amd = host_facts(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)) + report = match_report(_task({"cpu_flags": ["avx512f"]}), amd) + assert report["met"] is False + assert report["fields"] == [{"field": "cpu_flags.avx512f", "required": "avx512f", "actual": False, "met": False}] + assert match_summary(report) == "cpu_flags.avx512f missing" + + +def test_a_met_requirement_and_a_probe_that_could_not_tell_are_recorded_as_such(): + intel = host_facts(_probe_out(affinity=8, cpuinfo=INTEL_CPUINFO)) + met = match_report(_task({"min_cpus": 4, "cpu_vendors": ["GenuineIntel"], "cpu_flags": ["avx2"]}), intel) + assert met["met"] is True and [f["met"] for f in met["fields"]] == [True, True, True] + undecided = match_report(_task({"min_cpus": 4, "cpu_flags": ["avx2"]}), {"error": "host probe exited 127"}) + assert undecided["met"] is None and [f["met"] for f in undecided["fields"]] == [None, None] + assert match_summary(undecided) == "min_cpus unknown, cpu_flags.avx2 unknown" + + +# ------------------------------------------------------------------ the four host scenarios + + +def test_label_runs_the_avx512_task_on_an_amd_host_and_records_the_miss(tmp_path, monkeypatch, capsys): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + cell = _cell_json(out, "fix_avx", "result.json") + assert cell["host_match"]["met"] is False + assert cell["host_match"]["fingerprint"] == cell["host_fingerprint"] + assert [f["field"] for f in cell["host_match"]["fields"] if f["met"] is False] == ["cpu_flags.avx512f"] + assert not list((out / "b1").glob("fix_avx__*/unsupported.json")) # ran, not marked unsupported + logged = capsys.readouterr() + assert "labelled, not blocked" in logged.out + logged.err + + +def test_without_label_the_same_task_on_the_same_host_is_unsupported(tmp_path, monkeypatch): + """Regression: block stays the default; the labelled cells exist only with --label-host.""" + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + with pytest.raises(SystemExit) as exc: # every cell unsupported: `repo run` has nothing and exits 1 + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--task", "fix_avx") + assert exc.value.code == 1 + assert len(list((out / "b1").glob("fix_avx__*/unsupported.json"))) == 2 + assert not list((out / "b1").glob("fix_avx__*/result.json")) + + +def test_label_on_a_windows_host_groups_under_the_windows_fingerprint(tmp_path, monkeypatch): + """The requirements are checked against the grading container's CPU view, so a Linux container on a + Windows Docker host meets them; the host's own collectors are what the fingerprint records.""" + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(system="Windows", windows_memory=lambda: 8 * 2**30, + calls={"git --version": "git version 2.45.1\n"}, + which=lambda name: r"C:\git.exe" if name == "git" else None)) + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--label-host", "--task", "fix_avx") + cell = _cell_json(out, "fix_avx", "result.json") + assert cell["host_match"]["met"] is True and cell["host_match"]["fields"][0]["field"] == "cpu_flags.avx512f" + meta = _cell_json(out, "fix_avx", "meta.json") + assert meta["host"]["collectors"]["os"]["system"] == "Windows" + assert meta["host"]["collectors"]["memory"] == {"total_bytes": 8 * 2**30} + assert meta["host"]["collectors"]["tools"]["git"] == "2.45.1" + + +def test_label_on_a_container_records_the_container_in_the_fingerprint(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/.dockerenv": "", "/proc/cpuinfo": INTEL_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--label-host", "--task", "fix_race") + meta = _cell_json(out, "fix_race", "meta.json") + assert meta["host"]["collectors"]["virt"]["container"] is True + assert meta["host"]["collectors"]["virt"]["container_kind"] == "docker" + assert meta["host_match"]["met"] is True # RACE needs 4 CPUs of GenuineIntel, the container's cgroup gives 8 + + +def test_label_on_a_gpu_host_records_the_driver_and_cuda_version(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": "A100, 550.54.14\n", + "nvidia-smi": "CUDA Version: 12.4"} + _labelled(monkeypatch, _host(calls=calls, which=lambda name: "/usr/bin/nvidia-smi" if name == "nvidia-smi" else None)) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + meta = _cell_json(out, "fix_avx", "meta.json") + assert meta["host"]["collectors"]["gpu"]["cuda_version"] == "12.4" + assert meta["host"]["collectors"]["gpu"]["driver_version"] == "550.54.14" + + +# ------------------------------------------------------------------ results group by the actual fingerprint + + +def test_report_splits_a_batch_by_the_fingerprint_of_the_host_that_ran_each_cell(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + first = sorted((out / "b1").glob("fix_avx__*/result.json"))[1] + first.unlink() # the second rep is run again, on another host + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": INTEL_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=INTEL_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + rows = [r for r in summarize(out / "b1")["rows"] if r["task"] == "fix_avx"] + assert len(rows) == 2 and rows[0]["host_fingerprint"] != rows[1]["host_fingerprint"] + assert [r["n"] for r in rows] == [1, 1] + assert sorted(r["host_match"]["met"] for r in rows) == [False, True] # AMD misses avx512f, Intel has it + assert "labelled, unmet: cpu_flags.avx512f" in format_report(summarize(out / "b1")) + + +def test_the_flag_is_off_by_default_and_on_with_label_host(): + argv = ["repo", "run", "/x@HEAD", "--arm", "guidance:raw", "--model", "m", "--batch", "b"] + assert build_parser().parse_args(argv).label_host is False + assert build_parser().parse_args([*argv, "--label-host"]).label_host is True From 881673e56f7d45d75896de1969ce1d80dbe54e10 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Sun, 15 Nov 2026 11:35:54 +0000 Subject: [PATCH 02/16] Say the Windows CPU model comes from platform.processor() Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 3 ++- dokimi/dokimi/repos/host_fingerprint.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 4cd7d5248..c2263456f 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -702,7 +702,8 @@ Its collectors are `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` an flat dict of JSON scalars under `collectors` in the probe's facts, beside the container's own keys (`cpus`, `cpu_vendor`, ...), which are unchanged. A collector that fails reports `{"error": ...}` and the others still run. Linux reads `/proc`, `/sys` and `/etc/os-release`; macOS reads `sysctl`; -Windows reads the kernel's memory status and spawns only `git --version`, `nvidia-smi` and the other +Windows reads the kernel's memory status, takes the CPU model from `platform.processor()` (which +Windows fills from `PROCESSOR_IDENTIFIER`) and spawns only `git --version`, `nvidia-smi` and the other tools' `--version` calls when they are on the path. `gpu` is NVIDIA only. `virt` reads a container from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from DMI or the `hypervisor` CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value, diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 270c4bc50..577050012 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -6,7 +6,8 @@ Each collector returns a flat dict of JSON scalars. A collector that fails returns ``{"error": ...}`` and the others still run, so one unreadable file never costs the probe. Nothing that identifies a person or a machine is read or recorded: no host name, user name, IP address, environment value or -absolute path (an error message is scrubbed of paths before it is kept). Spawns go through +absolute path (an error message is scrubbed of paths before it is kept). The Windows CPU model is +``platform.processor()``, which Windows fills from ``PROCESSOR_IDENTIFIER``. Spawns go through :mod:`dokimi.process`, and only where the platform has the tool: nothing is shelled to on Windows except the cross-platform ``--version`` calls and ``nvidia-smi`` when it is on the path. """ From a22fd3a3bd0176ec6d3a3bc7a7b8f53ab73acd50 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 09:09:25 -0600 Subject: [PATCH 03/16] fix(repo): host fingerprint report rows, path scrub, Windows CPU Fixes in the host fingerprint and its report rows: - scrub redacts quoted paths with spaces, UNC paths and ~ paths whole. - The Windows CPU model and vendor come from the registry, not from platform.processor(), which reads PROCESSOR_IDENTIFIER. - repo report refuses a malformed host_fingerprint or host_match with exit 2 instead of a traceback. - A started missing cell counts on its own host's row; an unstarted one only when the task, arm and model has a single host row. - Text rows name their host; an undecided labelled row is not "unmet". - Tests: IPv4 check skips version strings, every identity collector, vendor, min_cpus boundary, flag case and mixed verdicts are pinned. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- ...00_napetrov_host-fingerprint-collectors.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 +- dokimi/dokimi/repos/host_fingerprint.py | 30 +++++-- dokimi/dokimi/repos/report.py | 68 +++++++++++++- dokimi/tests/test_host_fingerprint.py | 62 +++++++++++-- dokimi/tests/test_repo_host_label.py | 17 ++++ dokimi/tests/test_repo_report_hosts.py | 90 +++++++++++++++++++ .../tests/test_repo_report_unreadable_cell.py | 9 ++ 8 files changed, 262 insertions(+), 20 deletions(-) create mode 100644 dokimi/tests/test_repo_report_hosts.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md index 1312d64eb..64abc6cc0 100644 --- a/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md +++ b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md @@ -1,4 +1,4 @@ ### Added - **The host probe fingerprints the machine with a registry of collectors** — `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`, each a flat dict under `collectors` (`collectors_version` 1), beside the container's unchanged keys. A failing collector reports `{"error": ...}` and the others still run; no host or user name, IP address, environment value or absolute path is recorded. -- **`repo run --label-host` runs a task the host misses instead of blocking it** — each cell records `host_match` (per requirement met, unmet or unknown) in `result.json` and `meta.json`, and `repo report` groups results by host fingerprint. Without the flag nothing changes. +- **`repo run --label-host` runs a task the host misses instead of blocking it** — each cell records `host_match` (per requirement met, unmet or unknown) in `result.json` and `meta.json`, and `repo report` groups results by host fingerprint. Without the flag, `repo report` still splits rows by host fingerprint when a batch's cells ran on different hosts. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c2263456f..56aa85db2 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -702,8 +702,8 @@ Its collectors are `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` an flat dict of JSON scalars under `collectors` in the probe's facts, beside the container's own keys (`cpus`, `cpu_vendor`, ...), which are unchanged. A collector that fails reports `{"error": ...}` and the others still run. Linux reads `/proc`, `/sys` and `/etc/os-release`; macOS reads `sysctl`; -Windows reads the kernel's memory status, takes the CPU model from `platform.processor()` (which -Windows fills from `PROCESSOR_IDENTIFIER`) and spawns only `git --version`, `nvidia-smi` and the other +Windows reads the kernel's memory status, takes the CPU model and vendor from the registry (not from +`platform.processor()`, which reads `PROCESSOR_IDENTIFIER`) and spawns only `git --version`, `nvidia-smi` and the other tools' `--version` calls when they are on the path. `gpu` is NVIDIA only. `virt` reads a container from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from DMI or the `hypervisor` CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value, diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 577050012..111d3d45f 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -6,10 +6,10 @@ Each collector returns a flat dict of JSON scalars. A collector that fails returns ``{"error": ...}`` and the others still run, so one unreadable file never costs the probe. Nothing that identifies a person or a machine is read or recorded: no host name, user name, IP address, environment value or -absolute path (an error message is scrubbed of paths before it is kept). The Windows CPU model is -``platform.processor()``, which Windows fills from ``PROCESSOR_IDENTIFIER``. Spawns go through -:mod:`dokimi.process`, and only where the platform has the tool: nothing is shelled to on Windows -except the cross-platform ``--version`` calls and ``nvidia-smi`` when it is on the path. +absolute path (an error message is scrubbed of paths before it is kept). The Windows CPU model and +vendor come from the registry, not from ``platform.processor()`` (which reads ``PROCESSOR_IDENTIFIER``). +Spawns go through :mod:`dokimi.process`, and only where the platform has the tool: nothing is shelled to +on Windows except the cross-platform ``--version`` calls and ``nvidia-smi`` when it is on the path. """ from __future__ import annotations @@ -39,7 +39,8 @@ # DMI strings of hypervisors, matched in lower case against the vendor and product name. VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) -_PATH = re.compile(r"(?:/[\w.+@%-]+)+|[A-Za-z]:\\\S*") +# A quoted path may hold spaces, so it runs to its closing quote; bare ones end at a space. ``~`` and UNC paths too. +_PATH = re.compile(r"""(?<=['"])(?:/|~|[A-Za-z]:\\|\\\\)[^'"\n]*(?=['"])|~(?:/[\w.+@%-]+)+|\\\\[^\s'"]+|(?:/[\w.+@%-]+)+|[A-Za-z]:\\\S*""") def _read_text(path: str) -> str | None: @@ -65,6 +66,20 @@ class MemoryStatus(ctypes.Structure): return int(status.total_phys) +def _windows_cpu() -> tuple[str | None, str | None]: + """Model and vendor of a Windows host's first CPU, from the registry the kernel fills (no environment).""" + import winreg # Windows only: the module does not exist elsewhere + + try: + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, # type: ignore[attr-defined] + r"HARDWARE\DESCRIPTION\System\CentralProcessor\0") as key: + model = winreg.QueryValueEx(key, "ProcessorNameString")[0] # type: ignore[attr-defined] + vendor = winreg.QueryValueEx(key, "VendorIdentifier")[0] # type: ignore[attr-defined] + except OSError: + return None, None + return str(model).strip() or None, str(vendor).strip() or None + + @dataclass(frozen=True) class Probe: """What the collectors read and run. Tests pass fakes; the defaults read this machine.""" @@ -75,6 +90,7 @@ class Probe: run: Runner = process.run env: Mapping[str, str] = field(default_factory=lambda: dict(os.environ)) # read for booleans only windows_memory: Callable[[], int] = _windows_total_memory + windows_cpu: Callable[[], tuple[str | None, str | None]] = _windows_cpu def _spawn(probe: Probe, argv: list[str]) -> str | None: @@ -120,9 +136,7 @@ def _cpu(probe: Probe) -> dict[str, Any]: features = _sysctl(probe, "machdep.cpu.features") out["flags"] = features.lower() if features else None elif probe.system == "Windows": - out["model"] = platform.processor() or None # the kernel's string names the vendor, read from it - vendor = re.search(r"GenuineIntel|AuthenticAMD", out["model"] or "") - out["vendor"] = vendor.group(0) if vendor else None + out["model"], out["vendor"] = probe.windows_cpu() return out diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 9e1fc635a..b382294af 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -63,6 +63,7 @@ from dokimi.utils import same_model from .grading import SCORE_BOUND, read_grade_json +from .host import fingerprint_id from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug from .useful import useful_line, useful_summary @@ -157,6 +158,24 @@ def reading_cells(files: list[str]) -> Iterator[None]: f"({type(exc).__name__}: {exc})") from exc +def _match_usable(match: Any) -> bool: + """True when ``match`` is a ``host_match`` the report can read: ``met`` and each field's ``met`` a bool or null.""" + if not isinstance(match, dict) or not isinstance(match.get("met"), (bool, type(None))): + return False + fields = match.get("fields") + return isinstance(fields, list) and all( + isinstance(f, dict) and isinstance(f.get("field"), str) and isinstance(f.get("met"), (bool, type(None))) + for f in fields) + + +def _check_host_fields(path: Path, rec: dict[str, Any]) -> None: + """Raise a :class:`CellFileError` unless ``host_fingerprint`` is a string or null and ``host_match`` is usable.""" + if not isinstance(rec.get("host_fingerprint"), (str, type(None))): + raise CellFileError(f"{path} has no usable `host_fingerprint`") + if "host_match" in rec and not _match_usable(rec["host_match"]): + raise CellFileError(f"{path} has no usable `host_match`") + + def _check_cell(path: Path, rec: dict[str, Any]) -> None: """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read.""" if not _grade_usable(rec) or not _optional_number(rec["grade"].get("score")): @@ -175,6 +194,7 @@ def _check_cell(path: Path, rec: dict[str, Any]) -> None: for key in ("task", "arm", "model"): if not isinstance(rec.get(key), str): raise CellFileError(f"{path} has no usable `{key}`") + _check_host_fields(path, rec) def load_cells(batch_dir: Path) -> tuple[list[dict[str, Any]], list[str]]: @@ -231,17 +251,54 @@ def _skill_counts(recs: list[dict[str, Any]]) -> dict[str, Any]: return out +_UNKNOWN: Any = object() # a cell with no meta.json (never started) or an unreadable one: no host to name + + +def _started_host(meta_path: Path) -> Any: + """The fingerprint of the host recorded in a started cell's ``meta.json`` (``None``: none was probed).""" + if not meta_path.is_file(): + return _UNKNOWN + try: + meta = _read_object(meta_path) + except CellFileError: + return _UNKNOWN + host = meta.get("host") + if host is not None and not (isinstance(host, dict) and isinstance(host.get("collectors") or {}, dict)): + return _UNKNOWN # a host that fingerprint_id cannot read names no row + return fingerprint_id(host) if host is not None else None + + +def _row_missing(missing_by: dict[tuple[str, str, str, str | None], int], unplaced: dict[tuple[str, str, str], int], + hosts_of: dict[tuple[str, str, str], int], key: tuple[str, str, str, str | None]) -> int: + """Missing cells of one host row: those that ran on its host, and the unplaced ones when it is the only row.""" + slug_key = (key[0], _cell_arm(key[1]), _cell_model(key[2])) + count = missing_by.get((*slug_key, key[3]), 0) + if hosts_of[slug_key] == 1: + count += unplaced.get(slug_key, 0) + return count + + def summarize(batch_dir: Path) -> dict[str, Any]: cells, missing = load_cells(batch_dir) # One row per (task, arm, model) and host fingerprint: cells run on different hosts never pool in a row. groups: dict[tuple[str, str, str, str | None], list[dict[str, Any]]] = defaultdict(list) for rec in cells: groups[(rec["task"], rec["arm"], rec["model"], rec.get("host_fingerprint"))].append(rec) - missing_by: dict[tuple[str, str, str], int] = defaultdict(int) + # A missing cell that started has its host in meta.json and counts on that host's row. One that never + # started has none: it counts on a row only when its (task, arm, model) has a single host row. + missing_by: dict[tuple[str, str, str, str | None], int] = defaultdict(int) + unplaced: dict[tuple[str, str, str], int] = defaultdict(int) for name in missing: parts = name.split("__") if len(parts) == 4: - missing_by[(parts[0], parts[1], parts[2])] += 1 + host = _started_host(batch_dir / name / "meta.json") + if host is _UNKNOWN: + unplaced[(parts[0], parts[1], parts[2])] += 1 + else: + missing_by[(parts[0], parts[1], parts[2], host)] += 1 + hosts_of: dict[tuple[str, str, str], int] = defaultdict(int) + for g in groups: + hosts_of[(g[0], _cell_arm(g[1]), _cell_model(g[2]))] += 1 rows = [] for key in sorted(groups, key=lambda k: (k[0], k[1], k[2], k[3] or "")): recs = groups[key] @@ -255,7 +312,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "agent_errors": sum(bool(r["agent"]["is_error"]) for r in recs), "over_budget": sum(bool(r["agent"]["over_budget"]) for r in recs), "cost_unknown": sum(r["agent"]["cost_usd"] is None for r in recs), - "missing": missing_by.pop((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0), + "missing": _row_missing(missing_by, unplaced, hosts_of, key), "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]), "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6), "turns": _mean([r["agent"]["num_turns"] for r in recs]), @@ -338,8 +395,11 @@ def _row_notes(r: dict[str, Any]) -> list[str]: + unmeasured_note(r["uptake"])) if "host_match" in r: match = r["host_match"] - what = "met" if match["met"] is True else f"unmet: {', '.join(match['unmet']) or 'undecided'}" + what = "met" if match["met"] is True else f"unmet: {', '.join(match['unmet'])}" if match["met"] is False \ + else "undecided" notes.append(f"host {r['host_fingerprint']} (labelled, {what})") + elif r.get("host_fingerprint"): + notes.append(f"host {r['host_fingerprint']}") return notes diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 9ac874614..9d8befe3e 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -152,16 +152,17 @@ def test_macos_without_a_number_from_sysctl_is_an_error_for_that_field_only(): assert hf.collect(probe)["collectors"]["memory"] == {"error": "sysctl hw.memsize gave no number"} -def test_windows_spawns_only_cross_platform_calls_and_reads_no_proc(monkeypatch): - monkeypatch.setattr(platform, "processor", lambda: "Intel64 Family 6 Model 140, GenuineIntel") +def test_windows_spawns_only_cross_platform_calls_and_reads_no_proc(): which = lambda name: rf"C:\tools\{name}.exe" if name in ("git", "nvidia-smi") else None # noqa: E731 calls = {"git --version": "git version 2.45.1.windows.1\n", "nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": "NVIDIA RTX A4000, 551.23\n", "nvidia-smi": "CUDA Version: 12.4"} probe, runner = _probe(system="Windows", which=which, calls=calls, windows_memory=lambda: 16 * 2**30, + windows_cpu=lambda: ("Intel(R) Core(TM) i7", "GenuineIntel"), files=_files(**{"/proc__meminfo": "MemTotal: 1 kB\n"})) out = hf.collect(probe)["collectors"] assert out["memory"] == {"total_bytes": 16 * 2**30} assert out["cpu"]["vendor"] == "GenuineIntel" and out["cpu"]["flags"] is None + assert out["cpu"]["model"] == "Intel(R) Core(TM) i7" assert out["tools"]["git"] == "2.45.1" and out["tools"]["docker"] is None assert out["gpu"] == {"present": True, "count": 1, "name": "NVIDIA RTX A4000", "driver_version": "551.23", "cuda_version": "12.4"} @@ -169,6 +170,21 @@ def test_windows_spawns_only_cross_platform_calls_and_reads_no_proc(monkeypatch) assert {argv[0] for argv in runner.argvs()} <= {"git", "nvidia-smi"} # never sysctl, lscpu, free, cat +def test_windows_cpu_model_never_reads_the_processor_identifier_environment_value(monkeypatch): + monkeypatch.setenv("PROCESSOR_IDENTIFIER", "GenuineIntel workstation-jdoe") + monkeypatch.setattr(platform, "processor", lambda: os.environ["PROCESSOR_IDENTIFIER"]) # CPython's Windows source + probe, _ = _probe(system="Windows", windows_memory=lambda: 2**30, windows_cpu=lambda: (None, None)) + out = hf.collect(probe) + assert "workstation-jdoe" not in json.dumps(out) + assert out["collectors"]["cpu"]["model"] is None + + +@pytest.mark.skipif(sys.platform != "win32", reason="reads the registry on Windows only") +def test_windows_cpu_is_read_from_the_registry(): + model, vendor = hf.Probe(system="Windows").windows_cpu() + assert model and vendor + + @pytest.mark.skipif(sys.platform != "win32", reason="reads the kernel's memory status on Windows only") def test_windows_memory_is_the_kernels_total(): assert hf.Probe(system="Windows").windows_memory() > 0 @@ -242,12 +258,18 @@ def test_python_names_the_implementation_and_not_the_executable(): # ------------------------------------------------------------------ privacy -def test_privacy_real_host_and_user_names_and_addresses_never_appear(): - text = json.dumps(hf.collect(hf.Probe()), sort_keys=True) +def test_privacy_real_host_and_user_names_and_addresses_never_appear(monkeypatch): + monkeypatch.setattr(platform, "release", lambda: "5.15.167.4-microsoft-standard-WSL2") # a WSL2 kernel release + facts = hf.collect(hf.Probe()) + text = json.dumps(facts, sort_keys=True) for secret in {socket.gethostname(), getpass.getuser(), os.path.expanduser("~")}: if len(secret) >= 4: assert secret not in text, "a host or user name leaked into the fingerprint" - assert not re.search(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", text), "an IPv4 address leaked into the fingerprint" + # Versions (a four-part kernel release such as 5.15.167.4, a tool version) are dotted quads, not addresses. + versions = {"os": ("release",), "python": ("version",), "tools": tuple(hf.TOOLS)} + unversioned = {name: {k: v for k, v in c.items() if k not in versions.get(name, ())} + for name, c in facts["collectors"].items() if isinstance(c, dict)} + assert not re.search(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", json.dumps(unversioned)), "an IPv4 address leaked" assert _paths_in(text) == [], "an absolute path leaked into the fingerprint" @@ -275,6 +297,28 @@ def test_error_text_is_scrubbed_of_paths_and_truncated(): assert len(hf.scrub("x" * 1000)) == 300 +@pytest.mark.parametrize("text, kept", [ + ("missing '/home/Jane Doe/private'", "missing ''"), + ("missing 'C:\\Users\\Jane Doe\\AppData\\x'", "missing ''"), + ("cannot open \\\\server\\users\\jdoe\\file", "cannot open "), + ("cannot open '\\\\server\\users\\Jane Doe\\file'", "cannot open ''"), + ("no such file ~/secret/x", "no such file "), + ("no such file '~/Jane Doe/x'", "no such file ''"), +]) +def test_scrub_redacts_quoted_unc_and_home_relative_paths_whole(text, kept): + assert hf.scrub(text) == kept + + +def test_a_collector_error_naming_a_path_with_a_space_is_stored_without_it(monkeypatch): + def boom(probe): + raise OSError(2, "No such file", "/home/Jane Doe/private") + + monkeypatch.setattr(hf, "COLLECTORS", {**hf.COLLECTORS, "tools": boom}) + probe, _ = _probe() + error = hf.collect(probe)["collectors"]["tools"]["error"] + assert "Jane" not in error and "Doe" not in error and "" in error + + # ------------------------------------------------------------------ the identity of a host @@ -287,6 +331,14 @@ def test_the_fingerprint_id_is_stable_and_ignores_the_network_but_not_the_cpu(): assert len(fingerprint_id(facts)) == 12 +@pytest.mark.parametrize("name", ["os", "cpu", "memory", "gpu", "python", "tools", "virt"]) +def test_the_fingerprint_id_changes_with_every_identity_collector(name): + base = {n: {"k": "a"} for n in ["os", "cpu", "memory", "gpu", "python", "tools", "virt"]} + facts = {"cpus": 4, "collectors": base} + changed = {"cpus": 4, "collectors": {**base, name: {"k": "b"}}} + assert fingerprint_id(facts) != fingerprint_id(changed) + + def test_match_report_for_a_host_with_no_task_requirement_is_met_without_fields(): task = Task("t", {"id": "t"}, Path(".")) assert match_report(task, {"cpus": 2})["met"] is True diff --git a/dokimi/tests/test_repo_host_label.py b/dokimi/tests/test_repo_host_label.py index 2e6a57752..9f18d71a3 100644 --- a/dokimi/tests/test_repo_host_label.py +++ b/dokimi/tests/test_repo_host_label.py @@ -79,6 +79,23 @@ def test_a_met_requirement_and_a_probe_that_could_not_tell_are_recorded_as_such( assert match_summary(undecided) == "min_cpus unknown, cpu_flags.avx2 unknown" +def test_a_vendor_miss_the_min_cpus_boundary_and_an_uppercase_flag_are_judged_exactly(): + intel = host_facts(_probe_out(affinity=4, cpuinfo=INTEL_CPUINFO)) + vendor = match_report(_task({"cpu_vendors": ["AuthenticAMD"]}), intel) + assert vendor["met"] is False and match_summary(vendor) == "cpu_vendors missing" + assert match_report(_task({"cpu_vendors": ["genuineintel"]}), intel)["met"] is True # vendor compare ignores case + assert match_report(_task({"min_cpus": 4}), intel)["met"] is True # exactly min_cpus is enough + assert match_report(_task({"min_cpus": 5}), intel)["met"] is False + assert match_report(_task({"cpu_flags": ["AVX2"]}), intel)["met"] is True # docs write AVX2, cpuinfo avx2 + + +def test_one_unmet_and_one_undecided_field_is_a_miss(): + facts = {"cpus": 2, "cpu_vendor": None, "cpu_flags": []} # the vendor and flags are unknown, the CPU count is not + report = match_report(_task({"min_cpus": 4, "cpu_flags": ["avx2"]}), facts) + assert [f["met"] for f in report["fields"]] == [False, None] + assert report["met"] is False + + # ------------------------------------------------------------------ the four host scenarios diff --git a/dokimi/tests/test_repo_report_hosts.py b/dokimi/tests/test_repo_report_hosts.py new file mode 100644 index 000000000..c3dea5830 --- /dev/null +++ b/dokimi/tests/test_repo_report_hosts.py @@ -0,0 +1,90 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo report` rows split by host fingerprint: what a row shows and which missing cells it counts.""" + +from __future__ import annotations + +import json + +from dokimi.repos.host import fingerprint_id +from dokimi.repos.report import format_report, summarize + +HOST_A = {"cpus": 4, "cpu_vendor": "GenuineIntel"} +HOST_B = {"cpus": 8, "cpu_vendor": "AuthenticAMD"} +FP_A, FP_B = fingerprint_id(HOST_A), fingerprint_id(HOST_B) + + +def _cell(batch, rep, *, host=None, fingerprint=None, result=True, **extra): + rd = batch / f"t__guidance-raw__m-1__r{rep}" + rd.mkdir(parents=True) + (rd / "meta.json").write_text(json.dumps({"host": host})) + if result: + rec = {"task": "t", "arm": "guidance:raw", "model": "m-1", "score_task": False, "score_max": None, + "grade": {"status": "graded", "success": True, "score_only": False, "score": None}, + "agent": {"timed_out": False, "is_error": False, "over_budget": False, "cost_usd": 0.2, + "num_turns": 3}, "host_fingerprint": fingerprint, **extra} + (rd / "result.json").write_text(json.dumps(rec)) + + +def _plan(batch, reps): + (batch / "_eval").mkdir(parents=True, exist_ok=True) + cells = [f"t__guidance-raw__m-1__r{r}" for r in reps] + (batch / "_eval" / "plan.json").write_text(json.dumps({"cells": cells})) + + +def test_a_started_missing_cell_counts_on_the_row_of_the_host_that_started_it(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _cell(batch, 3, host=HOST_B, result=False) # started on B, never finished + rows = {r["host_fingerprint"]: r for r in summarize(batch)["rows"]} + assert {fp: r["missing"] for fp, r in rows.items()} == {FP_A: 0, FP_B: 1} + + +def test_an_unstarted_missing_cell_is_on_no_row_when_two_hosts_could_own_it(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _plan(batch, [1, 2, 3]) + summary = summarize(batch) + assert [r["missing"] for r in summary["rows"]] == [0, 0] + assert summary["missing"] == ["t__guidance-raw__m-1__r3"] + + +def test_an_unstarted_missing_cell_stays_on_the_only_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _plan(batch, [1, 2]) + assert [r["missing"] for r in summarize(batch)["rows"]] == [1] + + +def test_text_report_names_the_host_of_every_fingerprinted_row_without_label_mode(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _cell(batch, 3) # written before fingerprints: no host to name + lines = format_report(summarize(batch)).splitlines() + assert [ln.strip() for ln in lines if ln.strip().startswith("host ")] == sorted( + [f"host {FP_A}", f"host {FP_B}"]) + + +def test_a_labelled_row_keeps_its_verdict_and_an_undecided_one_is_not_unmet(tmp_path): + batch = tmp_path / "b1" + miss = {"met": False, "fields": [{"field": "cpu_flags.avx512f", "met": False}]} + unknown = {"met": None, "fields": [{"field": "min_cpus", "met": None}]} + _cell(batch, 1, host=HOST_A, fingerprint=FP_A, host_match=miss) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B, host_match=unknown) + text = format_report(summarize(batch)) + assert f"host {FP_A} (labelled, unmet: cpu_flags.avx512f)" in text + assert f"host {FP_B} (labelled, undecided)" in text + + +def test_a_missing_cell_whose_meta_host_is_malformed_is_placed_on_no_host_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host={"collectors": [1]}, result=False) + _cell(batch, 3, host=[1], result=False) + summary = summarize(batch) + assert [r["missing"] for r in summary["rows"]] == [2] # unplaced, and the only row takes them + assert len(summary["missing"]) == 2 diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py index 53047eb90..901ac88e1 100644 --- a/dokimi/tests/test_repo_report_unreadable_cell.py +++ b/dokimi/tests/test_repo_report_unreadable_cell.py @@ -127,6 +127,15 @@ def test_a_result_file_without_a_usable_grade_exits_2(tmp_path, capsys, argv, co pytest.param("agent", {**_GOOD["agent"], "num_turns": "3"}, id="num-turns-a-string"), pytest.param("agent", {**_GOOD["agent"], "input_tokens": True}, id="input-tokens-a-bool"), pytest.param("skills", [], id="skills-not-a-mapping"), + pytest.param("host_fingerprint", {}, id="host-fingerprint-an-object"), + pytest.param("host_fingerprint", [], id="host-fingerprint-a-list"), + pytest.param("host_fingerprint", 5, id="host-fingerprint-a-number"), + pytest.param("host_match", [], id="host-match-a-list"), + pytest.param("host_match", {"met": False}, id="host-match-no-fields"), + pytest.param("host_match", {"met": "no", "fields": []}, id="host-match-met-a-string"), + pytest.param("host_match", {"met": False, "fields": [{"field": 42, "met": False}]}, id="host-match-field-a-number"), + pytest.param("host_match", {"met": False, "fields": [{"field": "min_cpus", "met": "no"}]}, + id="host-match-field-met-a-string"), pytest.param("skills", {"measured": True, "expected_skill": "s"}, id="skills-no-skill-invoked"), pytest.param("skills", {"measured": True}, id="skills-measured-no-skill-invoked"), pytest.param("skills", {"measured": True, "expected_skill": "s", "skill_invoked": True}, From f0420106bc48995ebf28668132c25fdd5cfb68c5 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:29:13 -0600 Subject: [PATCH 04/16] fix(repo): scrub whole paths, keep legacy host rows - scrub: close a quoted path on its own quote, run a bare Windows or UNC path to the end of the line, keep image names such as ghcr.io/a/b - report: a started cell whose meta.json host has no collectors counts on the unfingerprinted row, as its result.json does Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/repos/host_fingerprint.py | 7 ++++--- dokimi/dokimi/repos/report.py | 4 +++- dokimi/tests/test_host_fingerprint.py | 6 ++++++ dokimi/tests/test_repo_report_hosts.py | 13 +++++++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 111d3d45f..c6c059165 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -39,8 +39,9 @@ # DMI strings of hypervisors, matched in lower case against the vendor and product name. VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) -# A quoted path may hold spaces, so it runs to its closing quote; bare ones end at a space. ``~`` and UNC paths too. -_PATH = re.compile(r"""(?<=['"])(?:/|~|[A-Za-z]:\\|\\\\)[^'"\n]*(?=['"])|~(?:/[\w.+@%-]+)+|\\\\[^\s'"]+|(?:/[\w.+@%-]+)+|[A-Za-z]:\\\S*""") +# A quoted path may hold spaces, so it runs to its own closing quote; a bare Windows or UNC path runs to the end of +# the line. ``~`` paths too. A ``/`` path not preceded by a word character, ``.``, ``:`` or ``/`` is absolute (not an image name). +_PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~(?:/[\w.+@%-]+)+|\\\\[^\n]*|(? str | None: @@ -101,7 +102,7 @@ def _spawn(probe: Probe, argv: list[str]) -> str | None: def scrub(text: str) -> str: """``text`` with every absolute path replaced: an error message may name a file under the user's home.""" - return _PATH.sub("", text)[:300] + return _PATH.sub(lambda m: f"{m['q']}{m['q']}" if m["q"] else "", text)[:300] def cpuinfo_field(text: str, name: str) -> str | None: diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index b382294af..fa3a91f4e 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -265,7 +265,9 @@ def _started_host(meta_path: Path) -> Any: host = meta.get("host") if host is not None and not (isinstance(host, dict) and isinstance(host.get("collectors") or {}, dict)): return _UNKNOWN # a host that fingerprint_id cannot read names no row - return fingerprint_id(host) if host is not None else None + if host is None or not host.get("collectors"): + return None # facts from before the collectors: the cell's result.json carries no fingerprint either + return fingerprint_id(host) def _row_missing(missing_by: dict[tuple[str, str, str, str | None], int], unplaced: dict[tuple[str, str, str], int], diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 9d8befe3e..960b7152c 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -304,6 +304,12 @@ def test_error_text_is_scrubbed_of_paths_and_truncated(): ("cannot open '\\\\server\\users\\Jane Doe\\file'", "cannot open ''"), ("no such file ~/secret/x", "no such file "), ("no such file '~/Jane Doe/x'", "no such file ''"), + ("missing \"/home/Jane O'Connor/private\"", "missing \"\""), + ("missing C:\\Users\\Jane Doe\\AppData\\x", "missing "), + ("missing C:\\Users\\O'Connor\\private.txt", "missing "), + ("cannot open \\\\server\\users\\O'Connor\\file", "cannot open "), + ("host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom", + "host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom"), ]) def test_scrub_redacts_quoted_unc_and_home_relative_paths_whole(text, kept): assert hf.scrub(text) == kept diff --git a/dokimi/tests/test_repo_report_hosts.py b/dokimi/tests/test_repo_report_hosts.py index c3dea5830..7eeae61a1 100644 --- a/dokimi/tests/test_repo_report_hosts.py +++ b/dokimi/tests/test_repo_report_hosts.py @@ -10,8 +10,8 @@ from dokimi.repos.host import fingerprint_id from dokimi.repos.report import format_report, summarize -HOST_A = {"cpus": 4, "cpu_vendor": "GenuineIntel"} -HOST_B = {"cpus": 8, "cpu_vendor": "AuthenticAMD"} +HOST_A = {"cpus": 4, "cpu_vendor": "GenuineIntel", "collectors": {"os": {"system": "Linux"}}} +HOST_B = {"cpus": 8, "cpu_vendor": "AuthenticAMD", "collectors": {"os": {"system": "Linux"}}} FP_A, FP_B = fingerprint_id(HOST_A), fingerprint_id(HOST_B) @@ -88,3 +88,12 @@ def test_a_missing_cell_whose_meta_host_is_malformed_is_placed_on_no_host_row(tm summary = summarize(batch) assert [r["missing"] for r in summary["rows"]] == [2] # unplaced, and the only row takes them assert len(summary["missing"]) == 2 + + +def test_a_missing_cell_with_container_only_host_facts_counts_on_the_unfingerprinted_row(tmp_path): + batch = tmp_path / "b1" + legacy = {"cpus": 4, "cpu_vendor": "GenuineIntel"} # facts from before the collectors + _cell(batch, 1, host=legacy) # its result.json has no host_fingerprint + _cell(batch, 2, host=legacy, result=False) + summary = summarize(batch) + assert [(r["host_fingerprint"], r["missing"]) for r in summary["rows"]] == [(None, 1)] From 6d3945e8c23ac2a29fa115a72746aa90df94c8b2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:13:25 -0800 Subject: [PATCH 05/16] fix(repo): scrub a path that follows a colon scrub() left "cannot open path:/home/jdoe/private" unchanged: the colon before the slash excluded the path from redaction, and no test pinned that exclusion. Drop the colon from the lookbehind; URLs, proxy addresses and image references still come back as written. Thread: PRRT_kwDOVBg7g863vUNS Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host_fingerprint.py | 2 +- dokimi/tests/test_host_fingerprint.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index c6c059165..0caf6fc1d 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -41,7 +41,7 @@ ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) # A quoted path may hold spaces, so it runs to its own closing quote; a bare Windows or UNC path runs to the end of # the line. ``~`` paths too. A ``/`` path not preceded by a word character, ``.``, ``:`` or ``/`` is absolute (not an image name). -_PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~(?:/[\w.+@%-]+)+|\\\\[^\n]*|(?['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~(?:/[\w.+@%-]+)+|\\\\[^\n]*|(? str | None: diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 960b7152c..25991c21e 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -308,6 +308,7 @@ def test_error_text_is_scrubbed_of_paths_and_truncated(): ("missing C:\\Users\\Jane Doe\\AppData\\x", "missing "), ("missing C:\\Users\\O'Connor\\private.txt", "missing "), ("cannot open \\\\server\\users\\O'Connor\\file", "cannot open "), + ("cannot open path:/home/jdoe/private", "cannot open path:"), ("host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom", "host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom"), ]) From b4fdefe385f4c71c33d16094c34a597c495296d1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 23:14:14 -0800 Subject: [PATCH 06/16] Address review: record items already fixed on this branch Each item was fixed by an earlier commit of this PR and checked on head. Thread: PRRT_kwDOVBg7g863m2nt: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863m2nu: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863m2nw: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863m2nz: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863m2n2: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863m2n5: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQc: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQf: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQi: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQk: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQm: already fixed in 881673e56 Thread: PRRT_kwDOVBg7g863pBQr: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQt: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pBQw: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863pIBx: already fixed in a22fd3a3b Thread: PRRT_kwDOVBg7g863sCH1: already fixed in f0420106b Thread: PRRT_kwDOVBg7g863sCH4: already fixed in f0420106b Review: PRR_kwDOVBg7g88AAAABYEpmmA: already fixed in f0420106b Review: PRR_kwDOVBg7g88AAAABYEwXWw: already fixed in f0420106b Comment: IC_kwDOVBg7g88AAAABmy6k2g: PR description corrected in this round (body edit, no commit) Comment: IC_kwDOVBg7g88AAAABm3tl6w: already fixed in f0420106b Signed-off-by: Nikolay Petrov From f8757b966c065dd1a560d9348af83714612a24ee Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:08:57 -0800 Subject: [PATCH 07/16] Address review: redact a bare POSIX path with spaces whole A bare POSIX path now runs to the end of the line, as a bare Windows or UNC path already did, so the rest of a path with spaces is not left behind. The regression cases cover spaces, a Unicode name, a ~ path and a URL. Thread: PRRT_kwDOVBg7g8630nuX Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host_fingerprint.py | 2 +- dokimi/tests/test_host_fingerprint.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 0caf6fc1d..514de15a6 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -41,7 +41,7 @@ ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) # A quoted path may hold spaces, so it runs to its own closing quote; a bare Windows or UNC path runs to the end of # the line. ``~`` paths too. A ``/`` path not preceded by a word character, ``.``, ``:`` or ``/`` is absolute (not an image name). -_PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~(?:/[\w.+@%-]+)+|\\\\[^\n]*|(?['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~/[^\n]*|\\\\[^\n]*|(? str | None: diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 25991c21e..614d2dfd3 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -309,6 +309,11 @@ def test_error_text_is_scrubbed_of_paths_and_truncated(): ("missing C:\\Users\\O'Connor\\private.txt", "missing "), ("cannot open \\\\server\\users\\O'Connor\\file", "cannot open "), ("cannot open path:/home/jdoe/private", "cannot open path:"), + ("cannot open /home/Jane Doe/private", "cannot open "), + ("cannot open /home/Jane Doe/private: Permission denied", "cannot open "), + ("cannot open /home/张三/private", "cannot open "), + ("no such file ~/Jane Doe/x", "no such file "), + ("see https://example.com/a b", "see https://example.com/a b"), ("host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom", "host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom"), ]) From ed0ec8e3d7346e6329eeb02f0a1c85981e11838e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:09:04 -0800 Subject: [PATCH 08/16] Address review: the path comment lists the lookbehind's characters Thread: PRRT_kwDOVBg7g864FY2z Review: PRR_kwDOVBg7g88AAAABYTPWEA Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host_fingerprint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 514de15a6..d2b6172d2 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -39,8 +39,8 @@ # DMI strings of hypervisors, matched in lower case against the vendor and product name. VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) -# A quoted path may hold spaces, so it runs to its own closing quote; a bare Windows or UNC path runs to the end of -# the line. ``~`` paths too. A ``/`` path not preceded by a word character, ``.``, ``:`` or ``/`` is absolute (not an image name). +# A quoted path may hold spaces, so it runs to its own closing quote; a bare path (POSIX, ``~``, Windows or UNC) +# runs to the end of the line. A ``/`` path not preceded by a word character, ``.``, ``-`` or ``/`` is absolute (not an image name). _PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~/[^\n]*|\\\\[^\n]*|(? Date: Mon, 16 Nov 2026 12:08:45 -0800 Subject: [PATCH 09/16] fix(repo): redact file: URL and UNC paths in host-probe errors scrub() missed a path behind a file: scheme (file:///home/Jane Doe/x leaked whole) and a forward-slash UNC path (//server/share/x leaked). A file: path keeps its scheme, a quoted one keeps its quotes, and a UNC path is redacted to the end of the line. http and https URLs are unchanged. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host_fingerprint.py | 15 +++++++++++---- dokimi/tests/test_host_fingerprint.py | 8 ++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index d2b6172d2..e90caf29a 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -39,9 +39,9 @@ # DMI strings of hypervisors, matched in lower case against the vendor and product name. VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) -# A quoted path may hold spaces, so it runs to its own closing quote; a bare path (POSIX, ``~``, Windows or UNC) -# runs to the end of the line. A ``/`` path not preceded by a word character, ``.``, ``-`` or ``/`` is absolute (not an image name). -_PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\)(?:(?!(?P=q)).)*(?P=q)|~/[^\n]*|\\\\[^\n]*|(?['"])(?:/|~|[A-Za-z]:\\|\\\\|(?i:file):)(?:(?!(?P=q)).)*(?P=q)|(?(?i:file):(?://)?)/[^\n]*|~/[^\n]*|\\\\[^\n]*|(? str | None: @@ -102,7 +102,14 @@ def _spawn(probe: Probe, argv: list[str]) -> str | None: def scrub(text: str) -> str: """``text`` with every absolute path replaced: an error message may name a file under the user's home.""" - return _PATH.sub(lambda m: f"{m['q']}{m['q']}" if m["q"] else "", text)[:300] + return _PATH.sub(_redact, text)[:300] + + +def _redact(match: re.Match[str]) -> str: + """A quoted path keeps its quotes, a ``file:`` URL keeps its scheme, any other path is ````.""" + if match["q"]: + return f"{match['q']}{match['q']}" + return f"{match['f']}" if match["f"] else "" def cpuinfo_field(text: str, name: str) -> str | None: diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 614d2dfd3..623ca4bb9 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -313,7 +313,15 @@ def test_error_text_is_scrubbed_of_paths_and_truncated(): ("cannot open /home/Jane Doe/private: Permission denied", "cannot open "), ("cannot open /home/张三/private", "cannot open "), ("no such file ~/Jane Doe/x", "no such file "), + ("cannot open file:///home/Jane Doe/private", "cannot open file://"), + ("cannot open file:/home/Jane Doe/private", "cannot open file:"), + ("cannot open FILE:///home/jdoe/x", "cannot open FILE://"), + ("cannot open file://server/share/Jane Doe/x", "cannot open file:"), + ("missing \"file:///home/Jane O'Connor/x\", next", "missing \"\", next"), + ("cannot open //server/share/Jane Doe/file", "cannot open "), + ("cannot open '//server/share/Jane Doe/file'", "cannot open ''"), ("see https://example.com/a b", "see https://example.com/a b"), + ("see http://example.com/a/b c", "see http://example.com/a/b c"), ("host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom", "host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom"), ]) From 904654149e29e9e219d8f5ab633198e5f4818391 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:45:28 -0800 Subject: [PATCH 10/16] Address review: count a started missing cell whose host has no row A cell that started on a host with no finished cell was in the batch-level missing list only; the row for its host does not exist, so no row showed it. It now counts as unplaced, so the single-row rule places it. Thread: PRRT_kwDOVBg7g864ZGf7 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/report.py | 4 ++++ dokimi/tests/test_repo_report_hosts.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 0c18f496a..2428fac72 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -319,6 +319,10 @@ def summarize(batch_dir: Path) -> dict[str, Any]: hosts_of: dict[tuple[str, str, str], int] = defaultdict(int) for g in groups: hosts_of[(g[0], _cell_arm(g[1]), _cell_model(g[2]))] += 1 + # A started missing cell whose host has no finished cell has no row: it is unplaced, so the single-row rule applies. + row_keys = {(g[0], _cell_arm(g[1]), _cell_model(g[2]), g[3]) for g in groups} + for k in [k for k in missing_by if k not in row_keys]: + unplaced[(k[0], k[1], k[2])] += missing_by.pop(k) rows = [] for key in sorted(groups, key=lambda k: (k[0], k[1], k[2], k[3] or "")): recs = groups[key] diff --git a/dokimi/tests/test_repo_report_hosts.py b/dokimi/tests/test_repo_report_hosts.py index 7eeae61a1..c30a7b5d8 100644 --- a/dokimi/tests/test_repo_report_hosts.py +++ b/dokimi/tests/test_repo_report_hosts.py @@ -59,6 +59,15 @@ def test_an_unstarted_missing_cell_stays_on_the_only_row(tmp_path): assert [r["missing"] for r in summarize(batch)["rows"]] == [1] +def test_a_started_missing_cell_whose_host_has_no_row_counts_on_the_only_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, result=False) # started on B, crashed before result.json; B has no finished cell + summary = summarize(batch) + assert [(r["host_fingerprint"], r["missing"]) for r in summary["rows"]] == [(FP_A, 1)] + assert summary["missing"] == ["t__guidance-raw__m-1__r2"] + + def test_text_report_names_the_host_of_every_fingerprinted_row_without_label_mode(tmp_path): batch = tmp_path / "b1" _cell(batch, 1, host=HOST_A, fingerprint=FP_A) From a22850ccc2c10b49c818c57d3cd3cdd5ea828745 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:46:27 -0800 Subject: [PATCH 11/16] Address review: warn when pooled labelled cells missed their host A cell run with --label-host on a host that misses its task, or that could not be checked, was pooled into the delta with no warning. compare now names those tasks, as it does for a model mismatch. Thread: PRRT_kwDOVBg7g864ZGgO Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/compare.py | 5 +++++ dokimi/tests/test_repo_host_label.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 015f1e5ce..5152a5c45 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -335,6 +335,11 @@ def _arm_warnings(cells: list[dict[str, Any]], baseline: str, text: str, models: if uptake and uptake["cells"] and not uptake["used"]: out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); " "a delta measures the patch's text, not the tool") + # A labelled cell on a host that misses its task (or could not be checked) is pooled like any other: say so. + missed = sorted({c["task"] for arm in arms for c in arm if "host_match" in c and c["host_match"].get("met") is not True}) + if missed: + out.append(f"{text} vs {baseline}: cells run with --label-host on a host that misses the task's requirements " + f"or could not be checked are pooled (tasks {', '.join(missed)}); not a measured pass") return out diff --git a/dokimi/tests/test_repo_host_label.py b/dokimi/tests/test_repo_host_label.py index 9f18d71a3..8068a2cd7 100644 --- a/dokimi/tests/test_repo_host_label.py +++ b/dokimi/tests/test_repo_host_label.py @@ -192,3 +192,15 @@ def test_the_flag_is_off_by_default_and_on_with_label_host(): argv = ["repo", "run", "/x@HEAD", "--arm", "guidance:raw", "--model", "m", "--batch", "b"] assert build_parser().parse_args(argv).label_host is False assert build_parser().parse_args([*argv, "--label-host"]).label_host is True + + +def test_compare_warns_when_a_labelled_cell_ran_on_a_host_that_misses_its_task(tmp_path): + from dokimi.repos.compare import _arm_warnings + + def cell(arm, met): + rec = {"task": "race", "arm": arm, "model": "m-1", "batch": "b"} + return rec if met is None else {**rec, "host_match": {"met": met, "fields": [], "fingerprint": "f"}} + + cells = [cell("raw", None), cell("skill", False)] # the treatment ran one labelled cell on a host that misses + assert any("race" in w and "--label-host" in w for w in _arm_warnings(cells, "raw", "skill", None)) + assert not any("--label-host" in w for w in _arm_warnings([cell("raw", None), cell("skill", None)], "raw", "skill", None)) From 60a9c54c2abd4e25376ca5014804014c04ae2f99 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:46:54 -0800 Subject: [PATCH 12/16] Address review: a tool that cannot be started keeps the other tools An OSError from the spawn of a tool that `which` found (a Windows make.cmd shim) made the whole tools collector an error. The spawn failure is now a fact about the tool, as a failing --version already is. Thread: PRRT_kwDOVBg7g864ZGgT Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host_fingerprint.py | 5 ++++- dokimi/tests/test_host_fingerprint.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index e90caf29a..203ffe182 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -96,7 +96,10 @@ class Probe: def _spawn(probe: Probe, argv: list[str]) -> str | None: """stdout of a command that exits 0, else ``None``: a tool that fails is a fact about the host, not an error.""" - done = probe.run(argv, timeout=SPAWN_TIMEOUT_S, env=Env.only(_SPAWN_ENV)) + try: + done = probe.run(argv, timeout=SPAWN_TIMEOUT_S, env=Env.only(_SPAWN_ENV)) + except OSError: # a command `which` found that cannot be started (a Windows shim): still a fact, not an error + return None return done.stdout if done.ok else None diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 623ca4bb9..6ebcc0f39 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -249,6 +249,18 @@ def test_tools_record_the_version_number_and_no_path(): assert hf.collect(probe)["collectors"]["tools"]["make"] == "present" # on the path, its version not printed +def test_a_tool_that_cannot_be_started_keeps_the_versions_of_the_others(): + def handler(call): + if call.argv[0] == "make": + raise OSError("[Errno 2] No such file or directory: 'make'") # `which` found it; the spawn fails + return reply("git version 2.43.0\n") if " ".join(call.argv) == "git --version" else reply(returncode=1) + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(handler), env={}, + which=lambda name: f"/usr/bin/{name}" if name in ("git", "make") else None) + tools = hf.collect(probe)["collectors"]["tools"] + assert tools == {"git": "2.43.0", "docker": None, "cmake": None, "gcc": None, "clang": None, "make": "present"} + + def test_python_names_the_implementation_and_not_the_executable(): python = hf.collect(_probe()[0])["collectors"]["python"] assert python["version"] == platform.python_version() and isinstance(python["free_threaded"], bool) From 9b265dc382b88169a274426e589134f2fb77dd4b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:47:41 -0800 Subject: [PATCH 13/16] Address review: scrub the probe-did-not-start error The ContainerError text (docker stderr, up to 500 characters) went into _eval/host.json, meta.json and unsupported.json with paths unchanged. It is scrubbed like the other probe errors. Thread: PRRT_kwDOVBg7g864ZGgZ Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/host.py | 2 +- dokimi/tests/test_repo_host.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index 6d96dcb7e..51d488c36 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -246,7 +246,7 @@ def _container_facts(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: try: result = ctx.runner.run(spec, stdout=out, stderr=err, timeout_s=PROBE_TIMEOUT_S) except ContainerError as exc: - return {"error": f"host probe did not start: {str(exc)[:300]}"} + return {"error": scrub(f"host probe did not start: {str(exc)[:300]}")} text = out.read_text(encoding="utf-8", errors="replace") if out.is_file() else "" try: raw = json.loads(text.strip().splitlines()[-1]) if result.ok and text.strip() else None diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index 845d2071f..ab171e576 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -933,3 +933,16 @@ def test_a_skill_arm_cell_of_an_unsupported_task_is_marked_not_run(tmp_path, mon assert [p.name for p in (batch / "fix_race__skill-lint__m-1__r1").iterdir()] == ["unsupported.json"] assert (batch / "fix_add__skill-lint__m-1__r1/result.json").is_file() assert {u["cell"] for u in summarize(batch)["unsupported"]} >= {"fix_race__skill-lint__m-1__r1"} + + +def test_a_probe_container_that_fails_to_start_has_its_error_scrubbed(tmp_path): + from dokimi.repos.container import ContainerError + + class RuntimeFails(HostRunner): + def run(self, spec, *, stdout, stderr, timeout_s): + raise ContainerError("docker run failed: Error loading config file: /home/jdoe/.docker/config.json") + + ctx = EvalContext(manifest=parse_manifest(MANIFEST), git_dir=tmp_path, content_dir=tmp_path, image="img:1", + runner=RuntimeFails(None)) + error = probe_host(ctx, tmp_path / "w")["error"] + assert "/home/jdoe" not in error and "host probe did not start" in error From 7444d7c0aba0dbdf2271f157b490fc2da7e5e8ff Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:47:51 -0800 Subject: [PATCH 14/16] Address review: the contract says what no-flag runs do now Without --label-host a cell still carries host_fingerprint and repo report still splits rows by it; the contract said nothing changes. Thread: PRRT_kwDOVBg7g864ZGgd Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f9954b62a..1804db553 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -718,7 +718,7 @@ field per declared requirement with `met` true, false, or null where the probe c report's `met` is false if any field is, else null if any is undecided, else true. The requirements are checked against the grading container's CPU view, as in §3, not against the host's collectors, which only enter the fingerprint. `repo report` groups rows by `host_fingerprint` and notes a -labelled miss ("labelled, unmet: cpu_flags.avx512f"). Without the flag, nothing changes. +labelled miss ("labelled, unmet: cpu_flags.avx512f"). Without the flag, cells are not run on a host that misses the requirements, but results still carry `host_fingerprint` and `repo report` still splits rows by it. Both commands exit 2, before any build, on a source that does not parse (a local path that is not a directory, an unpinned remote) and on a pinned local path inside a repository that is not its root; a From 779a950a682f3ee90b6bbbe7056adfac4ab51b64 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:56:11 -0800 Subject: [PATCH 15/16] Address review: move the unplaced step of summarize into a helper summarize was over the complexity cap (ratchet) with the rowless-host step inline. Thread: PRRT_kwDOVBg7g864ZGf7 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/report.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 2428fac72..75226cd56 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -286,6 +286,14 @@ def _started_host(meta_path: Path) -> Any: return fingerprint_id(host) +def _unplace_rowless(missing_by: dict[tuple[str, str, str, str | None], int], + unplaced: dict[tuple[str, str, str], int], groups: dict[Any, list[dict[str, Any]]]) -> None: + """A started missing cell whose host has no finished cell has no row: it is unplaced, so the single-row rule applies.""" + row_keys = {(g[0], _cell_arm(g[1]), _cell_model(g[2]), g[3]) for g in groups} + for k in [k for k in missing_by if k not in row_keys]: + unplaced[(k[0], k[1], k[2])] += missing_by.pop(k) + + def _row_missing(missing_by: dict[tuple[str, str, str, str | None], int], unplaced: dict[tuple[str, str, str], int], hosts_of: dict[tuple[str, str, str], int], key: tuple[str, str, str, str | None]) -> int: """Missing cells of one host row: those that ran on its host, and the unplaced ones when it is the only row.""" @@ -319,10 +327,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: hosts_of: dict[tuple[str, str, str], int] = defaultdict(int) for g in groups: hosts_of[(g[0], _cell_arm(g[1]), _cell_model(g[2]))] += 1 - # A started missing cell whose host has no finished cell has no row: it is unplaced, so the single-row rule applies. - row_keys = {(g[0], _cell_arm(g[1]), _cell_model(g[2]), g[3]) for g in groups} - for k in [k for k in missing_by if k not in row_keys]: - unplaced[(k[0], k[1], k[2])] += missing_by.pop(k) + _unplace_rowless(missing_by, unplaced, groups) rows = [] for key in sorted(groups, key=lambda k: (k[0], k[1], k[2], k[3] or "")): recs = groups[key] From face9224515fbc1fbef573536a8db53da9bdd15d Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:30:27 -0800 Subject: [PATCH 16/16] Address review: tool identity is presence only, failing collectors are left out A collector that still fails after one retry is left out of fingerprint_id, and tools count by presence, so a timed-out --version does not split one host into two rows. Thread: PRRT_kwDOVBg7g864ZGgG Thread: PRR_kwDOVBg7g88AAAABYn7Jtw Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 3 +- dokimi/dokimi/repos/host.py | 13 ++++- dokimi/dokimi/repos/host_fingerprint.py | 21 +++++-- dokimi/tests/test_host_fingerprint.py | 74 ++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 9 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 86584a427..559312cd0 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -712,7 +712,8 @@ from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from D CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value, interface name or absolute path; an error text is cut to 300 characters with its paths replaced. `fingerprint_id` hashes the CPU and identity collectors but not `network`, so a bridge that appears -between runs does not split a batch. +between runs does not split a batch. `tools` count by presence only (a version change or a timed-out +`--version` does not split a host), and a collector that still fails after one retry is left out. `repo run --label-host` runs a task that the host misses, instead of marking it unsupported. Each cell's `result.json` and `meta.json` hold `host_match`: the `fingerprint`, the report's `met`, and one diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index 51d488c36..4098bda58 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -191,11 +191,22 @@ def probe_host(ctx: EvalContext, work_dir: Path, probe: Probe | None = None) -> def fingerprint_id(facts: dict[str, Any]) -> str: """A short id of the host: the same machine gives the same id on every run (see ``IDENTITY_COLLECTORS``).""" keys = ("cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags") + collectors = facts.get("collectors") or {} identity = {**{k: facts.get(k) for k in keys}, - **{name: (facts.get("collectors") or {}).get(name) for name in IDENTITY_COLLECTORS}} + **{name: _identity_value(name, collectors.get(name)) for name in IDENTITY_COLLECTORS}} return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()[:12] +def _identity_value(name: str, value: Any) -> Any: + """What a collector adds to the id: nothing when it failed (left out, as if absent), and for ``tools`` + only whether each tool is present, so a version change or a timed-out ``--version`` does not split a host.""" + if not isinstance(value, dict) or "error" in value: + return None + if name == "tools": + return {tool: version is not None for tool, version in value.items()} + return value + + def match_report(task: Task, facts: dict[str, Any] | None) -> dict[str, Any]: """Per declared requirement, whether this host meets it, for a run that labels a miss instead of blocking it. diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py index 203ffe182..e60cde035 100644 --- a/dokimi/dokimi/repos/host_fingerprint.py +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -260,13 +260,22 @@ def _vm(probe: Probe) -> tuple[bool | None, str | None]: } -def collect(probe: Probe | None = None) -> dict[str, Any]: - """``{"collectors_version", "collectors": {name: flat dict}}``; a collector that raises is ``{"error": ...}``.""" - probe = probe or Probe() +def _run_collector(probe: Probe, collector: Callable[[Probe], dict[str, Any]]) -> dict[str, Any]: + """One collector's flat dict; a collector that raises or reports an error is run once more (a transient + failure on a loaded host), and the retry is what is kept.""" out: dict[str, Any] = {} - for name, collector in COLLECTORS.items(): + for _attempt in range(2): try: - out[name] = collector(probe) + out = collector(probe) except Exception as exc: # noqa: BLE001 - one collector failing must not fail the probe - out[name] = {"error": scrub(f"{type(exc).__name__}: {exc}")} + out = {"error": scrub(f"{type(exc).__name__}: {exc}")} + if "error" not in out: + break + return out + + +def collect(probe: Probe | None = None) -> dict[str, Any]: + """``{"collectors_version", "collectors": {name: flat dict}}``; a collector that still fails after one retry is ``{"error": ...}``.""" + probe = probe or Probe() + out = {name: _run_collector(probe, collector) for name, collector in COLLECTORS.items()} return {"collectors_version": COLLECTORS_VERSION, "collectors": out} diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py index 6ebcc0f39..d21e12a2b 100644 --- a/dokimi/tests/test_host_fingerprint.py +++ b/dokimi/tests/test_host_fingerprint.py @@ -367,7 +367,9 @@ def test_the_fingerprint_id_is_stable_and_ignores_the_network_but_not_the_cpu(): def test_the_fingerprint_id_changes_with_every_identity_collector(name): base = {n: {"k": "a"} for n in ["os", "cpu", "memory", "gpu", "python", "tools", "virt"]} facts = {"cpus": 4, "collectors": base} - changed = {"cpus": 4, "collectors": {**base, name: {"k": "b"}}} + # tools count by presence only (a version change is no change): its change is a tool that goes away + changed_value = {"k": None} if name == "tools" else {"k": "b"} + changed = {"cpus": 4, "collectors": {**base, name: changed_value}} assert fingerprint_id(facts) != fingerprint_id(changed) @@ -375,3 +377,73 @@ def test_match_report_for_a_host_with_no_task_requirement_is_met_without_fields( task = Task("t", {"id": "t"}, Path(".")) assert match_report(task, {"cpus": 2})["met"] is True assert match_report(task, {"cpus": 2})["fields"] == [] + + +# ------------------------------------------------------------------ identity: presence, failures, one retry (ZGgG) + + +def test_a_tool_version_change_does_not_change_the_fingerprint_id(): + base = {"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": None}}} + upgraded = {"cpus": 4, "collectors": {"tools": {"git": "2.45.1", "docker": None}}} + timed_out = {"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": "present"}}} + assert fingerprint_id(base) == fingerprint_id(upgraded) + assert fingerprint_id(timed_out) == fingerprint_id({"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": "24.0.7"}}}) + + +def test_a_tool_that_appears_or_disappears_changes_the_fingerprint_id(): + absent = {"cpus": 4, "collectors": {"tools": {"docker": None}}} + present = {"cpus": 4, "collectors": {"tools": {"docker": "24.0.7"}}} + assert fingerprint_id(absent) != fingerprint_id(present) + + +def test_a_failing_collector_is_left_out_of_the_fingerprint_id(): + ok = {"cpus": 4, "collectors": {"gpu": {"present": True, "count": 1}, "os": {"release": "6.8"}}} + failed = {"cpus": 4, "collectors": {"gpu": {"present": True, "error": "nvidia-smi timed out"}, + "os": {"release": "6.8"}}} + left_out = {"cpus": 4, "collectors": {"os": {"release": "6.8"}}} + assert fingerprint_id(failed) == fingerprint_id(left_out) + assert fingerprint_id(failed) != fingerprint_id(ok) + + +def test_a_collector_that_fails_once_is_retried_and_its_retry_is_kept(): + calls = {"n": 0} + + def flaky(probe): + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("transient") + return {"total_bytes": 8} + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(lambda call: reply(returncode=1)), + env={}, which=lambda name: None) + original = dict(hf.COLLECTORS) + try: + hf.COLLECTORS.clear() + hf.COLLECTORS["memory"] = flaky + out = hf.collect(probe)["collectors"]["memory"] + finally: + hf.COLLECTORS.clear() + hf.COLLECTORS.update(original) + assert out == {"total_bytes": 8} + assert calls["n"] == 2 + + +def test_a_collector_that_fails_twice_is_one_retry_then_an_error(): + calls = {"n": 0} + + def broken(probe): + calls["n"] += 1 + return {"error": "still broken"} + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(lambda call: reply(returncode=1)), + env={}, which=lambda name: None) + original = dict(hf.COLLECTORS) + try: + hf.COLLECTORS.clear() + hf.COLLECTORS["memory"] = broken + out = hf.collect(probe)["collectors"]["memory"] + finally: + hf.COLLECTORS.clear() + hf.COLLECTORS.update(original) + assert out == {"error": "still broken"} + assert calls["n"] == 2