diff --git a/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md new file mode 100644 index 000000000..64abc6cc0 --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_host-fingerprint-collectors.md @@ -0,0 +1,4 @@ +### Added + +- **The host probe fingerprints the machine with a registry of collectors** — `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`, each a flat dict under `collectors` (`collectors_version` 1), beside the container's unchanged keys. A failing collector reports `{"error": ...}` and the others still run; no host or user name, IP address, environment value or absolute path is recorded. +- **`repo run --label-host` runs a task the host misses instead of blocking it** — each cell records `host_match` (per requirement met, unmet or unknown) in `result.json` and `meta.json`, and `repo report` groups results by host fingerprint. Without the flag, `repo report` still splits rows by host fingerprint when a batch's cells ran on different hosts. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a9b377d5b..483e1d0e6 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1389,6 +1389,7 @@ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--image", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--jobs/-j", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--label-host", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--manifest", "action": "_StoreAction", "choices": null, "const": null, "default": ".agent-eval/repo-eval.yaml", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--manifest-file", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--max-batch-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.repo_run._usd#591d6e2f0474"}, diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e0b1be55b..559312cd0 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -330,7 +330,7 @@ belongs to the repository's grader and is passed through unread, except `grader` | `gate` | no | framework | what counts as success; default `["pass"]`. Each entry is a boolean metric name, which must be `true`, or `{"metric": , "min": }` for a numeric metric | | `score` | for score-only tasks | framework | `{"metric": , "max": }`: the metric that is the result when `pass` is `null`, and the value the oracle must reach (§4.3) | | `timeout_s`, `max_turns`, `budget_usd` | no | framework | per-task caps, overriding run defaults; each a finite number > 0 | -| `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | +| `host` | no | framework | `{"min_cpus": , "cpu_vendors": [], "cpu_flags": []}`, every key optional: what the hidden test needs to show the bug (a thread race that needs 3+ CPUs, an ISA path). Vendor and flag names are `/proc/cpuinfo`'s (`GenuineIntel`, `avx512f`); both match in any case, and a vendor other than `GenuineIntel`, `AuthenticAMD`, `HygonGenuine`, `CentaurHauls`, `Shanghai` is refused (`repo validate` too). A probe that read neither vendor nor flags from `/proc/cpuinfo` is a probe failure, not a miss They are checked against a probe container started like the grading one: CPUs are its affinity mask capped by its cgroup CPU quota (`docker --cpus`). On a host that misses them the task is unsupported (§4.3), unless `repo run --label-host` runs it and records the miss, and the probed facts go to `meta.json` `host` of a cell that runs, or to its `unsupported.json` `host` when it does not | | `egress` | no | framework | extra hosts (DNS names, optional `:port`; `*` stands for one whole label and the last two labels are literal, so `*.pythonhosted.org` passes and `*` or `*.org` does not) this task's agent and entry points reach through the proxy (§2), for example `["pypi.org", "files.pythonhosted.org"]` | | `expected_skill` | no | framework | the name (directory name) of the skill the agent should use; a `skill-catalog:` arm must install it, or the arm is refused before anything runs. On a `guidance:as-skill` catalog arm the generated `-guidance` skill is refused too (its usage is `guidance_skill_loaded`). `result.json` `skills` records whether the agent used it and whether it used it first | | `useful` | no | framework | the definition of the run's first useful result, reported as `first_useful` in `result.json` (§4.1): `{"command": ""}` for a task whose grade runs the target tests, or `{"edit": ""}` (or a list of globs) for one that grades an edit to a target file. Omit it when no event is the useful result. A malformed value is refused by `repo validate`, `tasks check` and `repo run` | @@ -700,6 +700,29 @@ apart, neither failed nor missing. If the probe itself fails, a task with `host` requirements is not run either, and `repo run` exits 1 when one of its cells is left unrun; a cell that already has a `result.json` keeps it. +The probe also fingerprints the host (`dokimi/repos/host_fingerprint.py`, `collectors_version` 1). +Its collectors are `os`, `cpu`, `memory`, `gpu`, `python`, `tools`, `network` and `virt`; each is a +flat dict of JSON scalars under `collectors` in the probe's facts, beside the container's own keys +(`cpus`, `cpu_vendor`, ...), which are unchanged. A collector that fails reports `{"error": ...}` +and the others still run. Linux reads `/proc`, `/sys` and `/etc/os-release`; macOS reads `sysctl`; +Windows reads the kernel's memory status, takes the CPU model and vendor from the registry (not from +`platform.processor()`, which reads `PROCESSOR_IDENTIFIER`) and spawns only `git --version`, `nvidia-smi` and the other +tools' `--version` calls when they are on the path. `gpu` is NVIDIA only. `virt` reads a container +from `/.dockerenv`, podman's `/run/.containerenv` or the cgroup, and a VM from DMI or the `hypervisor` +CPU flag. Nothing identifying is recorded: no host or user name, IP address, environment value, +interface name or absolute path; an error text is cut to 300 characters with its paths replaced. +`fingerprint_id` hashes the CPU and identity collectors but not `network`, so a bridge that appears +between runs does not split a batch. `tools` count by presence only (a version change or a timed-out +`--version` does not split a host), and a collector that still fails after one retry is left out. + +`repo run --label-host` runs a task that the host misses, instead of marking it unsupported. Each +cell's `result.json` and `meta.json` hold `host_match`: the `fingerprint`, the report's `met`, and one +field per declared requirement with `met` true, false, or null where the probe could not tell. The +report's `met` is false if any field is, else null if any is undecided, else true. The requirements +are checked against the grading container's CPU view, as in §3, not against the host's collectors, +which only enter the fingerprint. `repo report` groups rows by `host_fingerprint` and notes a +labelled miss ("labelled, unmet: cpu_flags.avx512f"). Without the flag, cells are not run on a host that misses the requirements, but results still carry `host_fingerprint` and `repo report` still splits rows by it. + Both commands exit 2, before any build, on a source that does not parse (a local path that is not a directory, an unpinned remote), on a pinned local path inside a repository that is not its root and on a `--cache-dir` that is not a usable directory; a source that parses but cannot be opened (clone, fetch, revision) exits 1 with a `✗` line. A manifest that diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index a0b1b3c86..10992fe85 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -362,7 +362,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock if skill_pin is not None: write_skill_pin(batch_dir, skill_pin) summary = run_batch(ctx, tasks, arms, options, args.reps, batch_dir, jobs=args.jobs, force=args.force, - max_batch_usd=args.max_batch_usd, log=_progress) + max_batch_usd=args.max_batch_usd, log=_progress, label_host=args.label_host) except _errors() as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) @@ -554,6 +554,10 @@ def register(repo_sub, eval_sub, positive_int, source_help: str, cache_help: str help="Start no new cell once the batch has spent this much, earlier invocations included " "(and the budgets of the cells in flight)") run_p.add_argument("--force", action="store_true", help="Rerun cells that already have result.json") + run_p.add_argument("--label-host", action="store_true", dest="label_host", + help="Run a task whose task.json `host` requirements this host misses anyway, and record " + "per field what was met and what was not (host_match in result.json) instead of marking " + "the cells unsupported; results carry the host fingerprint, so `repo report` groups them by it") run_p.set_defaults(func=cmd_repo_run) report_p = repo_sub.add_parser("report", help="Per (task, arm, model) table over a batch directory") diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 1d79c57a8..0038fde27 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -346,6 +346,11 @@ def _arm_warnings(cells: list[dict[str, Any]], baseline: str, text: str, models: if uptake and uptake["cells"] and not uptake["used"]: out.append(f"{text}: the agent used none of what the arm added in {uptake['cells']} cell(s); " "a delta measures the patch's text, not the tool") + # A labelled cell on a host that misses its task (or could not be checked) is pooled like any other: say so. + missed = sorted({c["task"] for arm in arms for c in arm if "host_match" in c and c["host_match"].get("met") is not True}) + if missed: + out.append(f"{text} vs {baseline}: cells run with --label-host on a host that misses the task's requirements " + f"or could not be checked are pooled (tasks {', '.join(missed)}); not a measured pass") return out diff --git a/dokimi/dokimi/repos/host.py b/dokimi/dokimi/repos/host.py index 4c885e1d6..4098bda58 100644 --- a/dokimi/dokimi/repos/host.py +++ b/dokimi/dokimi/repos/host.py @@ -23,15 +23,16 @@ from __future__ import annotations +import hashlib import json import math -import re import uuid from pathlib import Path from typing import Any from .container import ContainerError, ContainerSpec from .grading import EvalContext +from .host_fingerprint import IDENTITY_COLLECTORS, Probe, collect, cpuinfo_field, scrub from .host_spec import HOST_KEYS, KNOWN_VENDORS, HostRequirements, HostSpecError, parse_host # noqa: F401 from .useful_spec import parse_useful from .workspace import Task @@ -171,23 +172,82 @@ def host_facts(raw: dict[str, Any]) -> dict[str, Any]: quota = cgroup_cpus(raw) counts = [n for n in (affinity, quota) if n] text = str(raw["cpuinfo"]) if isinstance(raw.get("cpuinfo"), str) else "" - flags = _field(text, "flags") or _field(text, "Features") # x86 | arm64 + flags = cpuinfo_field(text, "flags") or cpuinfo_field(text, "Features") # x86 | arm64 return {"cpus": min(counts) if counts else None, "affinity_cpus": affinity, "cgroup_cpus": quota, - "cpu_vendor": _field(text, "vendor_id"), "cpu_model": _field(text, "model name"), + "cpu_vendor": cpuinfo_field(text, "vendor_id"), "cpu_model": cpuinfo_field(text, "model name"), "cpu_flags": sorted(set(flags.split())) if flags else []} -def _field(text: str, name: str) -> str | None: - match = re.search(rf"^{re.escape(name)}\s*:\s*(.*)$", text, re.M) - return match.group(1).strip() if match else None +def probe_host(ctx: EvalContext, work_dir: Path, probe: Probe | None = None) -> dict[str, Any]: + """Facts of a container like the grading one, plus the host fingerprint (:mod:`.host_fingerprint`). + + The container's facts are the top-level keys, ``{"error": ...}`` in place of them when the probe + could not run. The spec carries what decides the CPU view: the evaluation's image and runner + (whose flags every container shares); no mounts, no network. + """ + return {**_container_facts(ctx, work_dir), **collect(probe)} + + +def fingerprint_id(facts: dict[str, Any]) -> str: + """A short id of the host: the same machine gives the same id on every run (see ``IDENTITY_COLLECTORS``).""" + keys = ("cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags") + collectors = facts.get("collectors") or {} + identity = {**{k: facts.get(k) for k in keys}, + **{name: _identity_value(name, collectors.get(name)) for name in IDENTITY_COLLECTORS}} + return hashlib.sha256(json.dumps(identity, sort_keys=True).encode("utf-8")).hexdigest()[:12] + + +def _identity_value(name: str, value: Any) -> Any: + """What a collector adds to the id: nothing when it failed (left out, as if absent), and for ``tools`` + only whether each tool is present, so a version change or a timed-out ``--version`` does not split a host.""" + if not isinstance(value, dict) or "error" in value: + return None + if name == "tools": + return {tool: version is not None for tool, version in value.items()} + return value -def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: - """Facts of a container like the grading one, or ``{"error": ...}`` when the probe could not run. +def match_report(task: Task, facts: dict[str, Any] | None) -> dict[str, Any]: + """Per declared requirement, whether this host meets it, for a run that labels a miss instead of blocking it. - The spec carries what decides the CPU view: the evaluation's image and runner (whose flags - every container shares); no mounts, no network. + ``met`` is ``True`` or ``False`` per field, ``None`` when the probe could not tell (the rule of :func:`unmet`); + the report's ``met`` is ``False`` if any field is, else ``None`` if any is undecided, else ``True``. """ + facts = facts or {} + need = requirements(task) + fields = _match_fields(need, facts) if need is not None else [] + verdicts = {f["met"] for f in fields} + met = False if False in verdicts else None if None in verdicts else True + return {"fingerprint": fingerprint_id(facts), "met": met, "fields": fields} + + +def match_summary(match: dict[str, Any]) -> str: + """The fields a labelled run's host misses, or those it could not tell: ``cpu_flags.avx512f missing``.""" + missed = [f"{f['field']} missing" for f in match["fields"] if f["met"] is False] + undecided = [f"{f['field']} unknown" for f in match["fields"] if f["met"] is None] + return ", ".join(missed + undecided) or "nothing unmet" + + +def _match_fields(need: HostRequirements, facts: dict[str, Any]) -> list[dict[str, Any]]: + """One entry per declared requirement; undecided (``met`` None) where :func:`unmet` says the probe cannot tell.""" + cpus, vendor = facts.get("cpus"), facts.get("cpu_vendor") + have = {f.lower() for f in facts.get("cpu_flags") or []} # docs write AVX512F, cpuinfo avx512f + decided = cpus is not None and bool(vendor or have) # no cpuinfo vendor or flags: a miss would be a guess + fields: list[dict[str, Any]] = [] + if need.min_cpus > 1: + fields.append({"field": "min_cpus", "required": need.min_cpus, "actual": cpus, + "met": cpus >= need.min_cpus if cpus is not None else None}) + if need.cpu_vendors: + ok = (vendor or "").lower() in {v.lower() for v in need.cpu_vendors} if decided else None + fields.append({"field": "cpu_vendors", "required": list(need.cpu_vendors), "actual": vendor, "met": ok}) + for flag in need.cpu_flags: + ok = flag.lower() in have if decided else None + fields.append({"field": f"cpu_flags.{flag}", "required": flag, "actual": ok, "met": ok}) + return fields + + +def _container_facts(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: + """The probe container's facts, or ``{"error": ...}`` when it could not run.""" work_dir.mkdir(parents=True, exist_ok=True) out, err = work_dir / "host-probe.out", work_dir / "host-probe.err" # HOME is the probe container's own /tmp, never a host path. @@ -197,7 +257,7 @@ def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: try: result = ctx.runner.run(spec, stdout=out, stderr=err, timeout_s=PROBE_TIMEOUT_S) except ContainerError as exc: - return {"error": f"host probe did not start: {str(exc)[:300]}"} + return {"error": scrub(f"host probe did not start: {str(exc)[:300]}")} text = out.read_text(encoding="utf-8", errors="replace") if out.is_file() else "" try: raw = json.loads(text.strip().splitlines()[-1]) if result.ok and text.strip() else None @@ -206,7 +266,7 @@ def probe_host(ctx: EvalContext, work_dir: Path) -> dict[str, Any]: if not isinstance(raw, dict): tail = (err.read_text(encoding="utf-8", errors="replace") if err.is_file() else "").strip()[-300:] how = "timed out" if result.timed_out else f"exited {result.returncode}" - return {"error": f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}"} + return {"error": scrub(f"host probe {how} in {ctx.image} (python3 -c): {tail or text.strip()[-300:]}")} return host_facts(raw) diff --git a/dokimi/dokimi/repos/host_fingerprint.py b/dokimi/dokimi/repos/host_fingerprint.py new file mode 100644 index 000000000..e60cde035 --- /dev/null +++ b/dokimi/dokimi/repos/host_fingerprint.py @@ -0,0 +1,281 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The host fingerprint: what the machine that runs dokimi is, one collector per field group. + +Each collector returns a flat dict of JSON scalars. A collector that fails returns ``{"error": ...}`` +and the others still run, so one unreadable file never costs the probe. Nothing that identifies a +person or a machine is read or recorded: no host name, user name, IP address, environment value or +absolute path (an error message is scrubbed of paths before it is kept). The Windows CPU model and +vendor come from the registry, not from ``platform.processor()`` (which reads ``PROCESSOR_IDENTIFIER``). +Spawns go through :mod:`dokimi.process`, and only where the platform has the tool: nothing is shelled to +on Windows except the cross-platform ``--version`` calls and ``nvidia-smi`` when it is on the path. +""" + +from __future__ import annotations + +import os +import platform +import re +import shutil +import socket +import sysconfig +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from dokimi import process +from dokimi.process import Env, Runner + +COLLECTORS_VERSION = 1 +SPAWN_TIMEOUT_S = 10 +# Only what a spawned tool needs from the environment: the binary's search path, and on Windows its system root. +_SPAWN_ENV = ("PATH", "SYSTEMROOT") +# The collectors whose values decide a host's identity (the id of `fingerprint_id`); network state is not one: +# a docker bridge that appears between two runs must not split a batch. +IDENTITY_COLLECTORS = ("os", "cpu", "memory", "gpu", "python", "tools", "virt") +TOOLS = ("git", "docker", "cmake", "gcc", "clang", "make") # version is the first number on the first line of `--version` +PROXY_VARS = ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy") +# DMI strings of hypervisors, matched in lower case against the vendor and product name. +VM_MARKERS = (("vmware", "vmware"), ("virtualbox", "virtualbox"), ("innotek", "virtualbox"), ("qemu", "qemu"), + ("kvm", "kvm"), ("xen", "xen"), ("parallels", "parallels"), ("virtual machine", "hyper-v")) +# A quoted path runs to its closing quote; a bare path (POSIX, ``~``, Windows, UNC, ``file:``) runs to the end of the line. +# A ``/`` path not preceded by a word character, ``.``, ``-`` or ``/`` is absolute (not an image name); ``//`` is UNC, not ``https://``. +_PATH = re.compile(r"""(?P['"])(?:/|~|[A-Za-z]:\\|\\\\|(?i:file):)(?:(?!(?P=q)).)*(?P=q)|(?(?i:file):(?://)?)/[^\n]*|~/[^\n]*|\\\\[^\n]*|(? str | None: + try: + with open(path, encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return None + + +def _windows_total_memory() -> int: + """Physical memory of a Windows host, from the kernel's ``GlobalMemoryStatusEx`` (no shell).""" + import ctypes # Windows only: the windll attribute does not exist elsewhere + + class MemoryStatus(ctypes.Structure): + _fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong), ("total_phys", ctypes.c_ulonglong), + ("avail_phys", ctypes.c_ulonglong), ("total_page", ctypes.c_ulonglong), + ("avail_page", ctypes.c_ulonglong), ("total_virtual", ctypes.c_ulonglong), + ("avail_virtual", ctypes.c_ulonglong), ("avail_extended", ctypes.c_ulonglong)] + + status = MemoryStatus(length=ctypes.sizeof(MemoryStatus)) + ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)) # type: ignore[attr-defined] + return int(status.total_phys) + + +def _windows_cpu() -> tuple[str | None, str | None]: + """Model and vendor of a Windows host's first CPU, from the registry the kernel fills (no environment).""" + import winreg # Windows only: the module does not exist elsewhere + + try: + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, # type: ignore[attr-defined] + r"HARDWARE\DESCRIPTION\System\CentralProcessor\0") as key: + model = winreg.QueryValueEx(key, "ProcessorNameString")[0] # type: ignore[attr-defined] + vendor = winreg.QueryValueEx(key, "VendorIdentifier")[0] # type: ignore[attr-defined] + except OSError: + return None, None + return str(model).strip() or None, str(vendor).strip() or None + + +@dataclass(frozen=True) +class Probe: + """What the collectors read and run. Tests pass fakes; the defaults read this machine.""" + + system: str = field(default_factory=platform.system) + read: Callable[[str], str | None] = _read_text + which: Callable[[str], str | None] = shutil.which + run: Runner = process.run + env: Mapping[str, str] = field(default_factory=lambda: dict(os.environ)) # read for booleans only + windows_memory: Callable[[], int] = _windows_total_memory + windows_cpu: Callable[[], tuple[str | None, str | None]] = _windows_cpu + + +def _spawn(probe: Probe, argv: list[str]) -> str | None: + """stdout of a command that exits 0, else ``None``: a tool that fails is a fact about the host, not an error.""" + try: + done = probe.run(argv, timeout=SPAWN_TIMEOUT_S, env=Env.only(_SPAWN_ENV)) + except OSError: # a command `which` found that cannot be started (a Windows shim): still a fact, not an error + return None + return done.stdout if done.ok else None + + +def scrub(text: str) -> str: + """``text`` with every absolute path replaced: an error message may name a file under the user's home.""" + return _PATH.sub(_redact, text)[:300] + + +def _redact(match: re.Match[str]) -> str: + """A quoted path keeps its quotes, a ``file:`` URL keeps its scheme, any other path is ````.""" + if match["q"]: + return f"{match['q']}{match['q']}" + return f"{match['f']}" if match["f"] else "" + + +def cpuinfo_field(text: str, name: str) -> str | None: + match = re.search(rf"^{re.escape(name)}\s*:\s*(.*)$", text, re.M) + return match.group(1).strip() if match else None + + +def _os(probe: Probe) -> dict[str, Any]: + out: dict[str, Any] = {"system": probe.system, "release": platform.release(), "machine": platform.machine()} + if probe.system == "Linux": + out["distro"] = _os_release(probe) + elif probe.system == "Darwin": + out["macos_version"] = platform.mac_ver()[0] or None + return out + + +def _os_release(probe: Probe) -> str | None: + text = probe.read("/etc/os-release") or probe.read("/usr/lib/os-release") or "" + match = re.search(r'^PRETTY_NAME="?([^"\n]*)"?', text, re.M) + return match.group(1) if match else None + + +def _cpu(probe: Probe) -> dict[str, Any]: + """The machine's CPU (the container's view is in the probe's top-level keys, from its cgroup).""" + out: dict[str, Any] = {"logical_cpus": os.cpu_count(), "vendor": None, "model": None, "flags": None} + if probe.system == "Linux": + text = probe.read("/proc/cpuinfo") or "" + out.update(vendor=cpuinfo_field(text, "vendor_id"), model=cpuinfo_field(text, "model name"), + flags=cpuinfo_field(text, "flags") or cpuinfo_field(text, "Features")) + elif probe.system == "Darwin": + out.update(model=_sysctl(probe, "machdep.cpu.brand_string"), vendor=_sysctl(probe, "machdep.cpu.vendor")) + features = _sysctl(probe, "machdep.cpu.features") + out["flags"] = features.lower() if features else None + elif probe.system == "Windows": + out["model"], out["vendor"] = probe.windows_cpu() + return out + + +def _sysctl(probe: Probe, name: str) -> str | None: + value = _spawn(probe, ["sysctl", "-n", name]) + return value.strip() if value and value.strip() else None + + +def _memory(probe: Probe) -> dict[str, Any]: + if probe.system == "Linux": + match = re.search(r"^MemTotal:\s+(\d+) kB", probe.read("/proc/meminfo") or "", re.M) + return {"total_bytes": int(match.group(1)) * 1024} if match else {"error": "no MemTotal line in the meminfo file"} + if probe.system == "Darwin": + value = _sysctl(probe, "hw.memsize") + return {"total_bytes": int(value)} if value and value.isdigit() else {"error": "sysctl hw.memsize gave no number"} + if probe.system == "Windows": + return {"total_bytes": probe.windows_memory()} + return {"error": f"no memory collector for {probe.system}"} + + +def _gpu(probe: Probe) -> dict[str, Any]: + """NVIDIA only: ``nvidia-smi`` is run when it is on the path; other GPUs report ``present`` false.""" + if probe.which("nvidia-smi") is None: + return {"present": False} + rows = _spawn(probe, ["nvidia-smi", "--query-gpu=name,driver_version", "--format=csv,noheader"]) + if rows is None: + return {"present": True, "error": "nvidia-smi --query-gpu=name,driver_version failed"} + table = [[cell.strip() for cell in line.split(",")] for line in rows.splitlines() if line.strip()] + banner = _spawn(probe, ["nvidia-smi"]) or "" + cuda = re.search(r"CUDA Version:\s*(\d+(?:\.\d+)+)", banner) + first = table[0] if table else [] + return {"present": True, "count": len(table), "name": first[0] if first else None, + "driver_version": first[1] if len(first) > 1 else None, "cuda_version": cuda.group(1) if cuda else None} + + +def _python(probe: Probe) -> dict[str, Any]: + return {"implementation": platform.python_implementation(), "version": platform.python_version(), + "free_threaded": bool(sysconfig.get_config_var("Py_GIL_DISABLED"))} + + +def _tools(probe: Probe) -> dict[str, str | None]: + """Per tool: its version, ``present`` when it runs without a readable version, ``None`` when absent.""" + out: dict[str, str | None] = {} + for name in TOOLS: + if probe.which(name) is None: + out[name] = None + continue + text = _spawn(probe, [name, "--version"]) or "" + number = re.search(r"\d+(?:\.\d+)+", text.split("\n", 1)[0]) + out[name] = number.group(0) if number else "present" + return out + + +def _network(probe: Probe) -> dict[str, Any]: + """Counts and booleans only: no interface name, address or proxy value.""" + return {"interfaces": len(socket.if_nameindex()), "default_route": _default_route(probe), + "proxy_env": any(probe.env.get(name) for name in PROXY_VARS)} + + +def _default_route(probe: Probe) -> bool | None: + if probe.system != "Linux": + return None + table = probe.read("/proc/net/route") + if table is None: + return None + return any(line.split()[1:2] == ["00000000"] for line in table.splitlines()[1:] if len(line.split()) > 1) + + +def _virt(probe: Probe) -> dict[str, Any]: + kind = _container_kind(probe) + vm, vendor = _vm(probe) + return {"container": kind is not None, "container_kind": kind, "vm": vm, "vm_vendor": vendor} + + +def _container_kind(probe: Probe) -> str | None: + """``docker``, ``podman``, ``kubernetes`` or ``lxc`` when this process runs in one; ``None`` otherwise or unknown.""" + if probe.system != "Linux": + return None + if probe.read("/.dockerenv") is not None: + return "docker" + if probe.read("/run/.containerenv") is not None: + return "podman" + cgroup = probe.read("/proc/self/cgroup") or "" + for marker, kind in {"kubepods": "kubernetes", "libpod": "podman", "lxc": "lxc", "docker": "docker"}.items(): + if marker in cgroup: + return kind + return None + + +def _vm(probe: Probe) -> tuple[bool | None, str | None]: + """``(is a VM, its hypervisor)``; ``None`` when this platform cannot tell.""" + if probe.system == "Darwin": + value = _sysctl(probe, "kern.hv_vmm_present") + return (None, None) if value is None else (value == "1", None) + if probe.system != "Linux": + return None, None + dmi = " ".join(probe.read(f"/sys/class/dmi/id/{name}") or "" for name in ("sys_vendor", "product_name")).lower() + for needle, name in VM_MARKERS: + if needle in dmi: + return True, name + flags = cpuinfo_field(probe.read("/proc/cpuinfo") or "", "flags") or "" + if "hypervisor" in flags.split(): + return True, None + return (False if dmi.strip() else None), None + + +COLLECTORS: dict[str, Callable[[Probe], dict[str, Any]]] = { + "os": _os, "cpu": _cpu, "memory": _memory, "gpu": _gpu, "python": _python, + "tools": _tools, "network": _network, "virt": _virt, +} + + +def _run_collector(probe: Probe, collector: Callable[[Probe], dict[str, Any]]) -> dict[str, Any]: + """One collector's flat dict; a collector that raises or reports an error is run once more (a transient + failure on a loaded host), and the retry is what is kept.""" + out: dict[str, Any] = {} + for _attempt in range(2): + try: + out = collector(probe) + except Exception as exc: # noqa: BLE001 - one collector failing must not fail the probe + out = {"error": scrub(f"{type(exc).__name__}: {exc}")} + if "error" not in out: + break + return out + + +def collect(probe: Probe | None = None) -> dict[str, Any]: + """``{"collectors_version", "collectors": {name: flat dict}}``; a collector that still fails after one retry is ``{"error": ...}``.""" + probe = probe or Probe() + out = {name: _run_collector(probe, collector) for name, collector in COLLECTORS.items()} + return {"collectors_version": COLLECTORS_VERSION, "collectors": out} diff --git a/dokimi/dokimi/repos/report.py b/dokimi/dokimi/repos/report.py index 28c9eaa79..75226cd56 100644 --- a/dokimi/dokimi/repos/report.py +++ b/dokimi/dokimi/repos/report.py @@ -51,19 +51,19 @@ from __future__ import annotations import json +import math from collections import defaultdict from collections.abc import Iterator from contextlib import contextmanager from pathlib import Path from typing import Any -import math - from dokimi.measure import as_float, is_number from dokimi.utils import same_model -from .grading import SCORE_BOUND, read_grade_json from .batch_ledger import recorded_execution +from .grading import SCORE_BOUND, read_grade_json +from .host import fingerprint_id from .process_counts import process_line, process_means from .runner import UNSUPPORTED_FILE, slug from .useful import useful_line, useful_summary @@ -158,6 +158,24 @@ def reading_cells(files: list[str]) -> Iterator[None]: f"({type(exc).__name__}: {exc})") from exc +def _match_usable(match: Any) -> bool: + """True when ``match`` is a ``host_match`` the report can read: ``met`` and each field's ``met`` a bool or null.""" + if not isinstance(match, dict) or not isinstance(match.get("met"), (bool, type(None))): + return False + fields = match.get("fields") + return isinstance(fields, list) and all( + isinstance(f, dict) and isinstance(f.get("field"), str) and isinstance(f.get("met"), (bool, type(None))) + for f in fields) + + +def _check_host_fields(path: Path, rec: dict[str, Any]) -> None: + """Raise a :class:`CellFileError` unless ``host_fingerprint`` is a string or null and ``host_match`` is usable.""" + if not isinstance(rec.get("host_fingerprint"), (str, type(None))): + raise CellFileError(f"{path} has no usable `host_fingerprint`") + if "host_match" in rec and not _match_usable(rec["host_match"]): + raise CellFileError(f"{path} has no usable `host_match`") + + def _check_cell(path: Path, rec: dict[str, Any]) -> None: """Raise a :class:`CellFileError` unless ``rec`` has the fields the report and compare read.""" if not _grade_usable(rec) or not _optional_number(rec["grade"].get("score")): @@ -176,6 +194,7 @@ def _check_cell(path: Path, rec: dict[str, Any]) -> None: for key in ("task", "arm", "model"): if not isinstance(rec.get(key), str): raise CellFileError(f"{path} has no usable `{key}`") + _check_host_fields(path, rec) if "batch" in rec and not isinstance(rec["batch"], str): raise CellFileError(f"{path} has no usable `batch`") @@ -227,20 +246,90 @@ def load_unsupported(batch_dir: Path) -> list[dict[str, Any]]: return out +def _labelled_match(recs: list[dict[str, Any]]) -> dict[str, Any]: + """``{"host_match": ...}`` for a labelled row: which of the task's host requirements its host missed.""" + labelled = [r["host_match"] for r in recs if isinstance(r.get("host_match"), dict)] + if not labelled: + return {} + return {"host_match": {"met": labelled[0]["met"], + "unmet": [f["field"] for f in labelled[0]["fields"] if f["met"] is False]}} + + +def _skill_counts(recs: list[dict[str, Any]]) -> dict[str, Any]: + used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)] + if not used: # cells whose trace was unreadable are not counted: no rate from a missing measurement + return {} + out: dict[str, Any] = {"skill_measured": len(used), "skill_invoked": sum(bool(u["skill_invoked"]) for u in used)} + expecting = [u for u in used if u.get("expected_skill")] # a task naming no skill has nothing to select + if expecting: + out["skill_expected"] = len(expecting) + out["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting) + return out + + +_UNKNOWN: Any = object() # a cell with no meta.json (never started) or an unreadable one: no host to name + + +def _started_host(meta_path: Path) -> Any: + """The fingerprint of the host recorded in a started cell's ``meta.json`` (``None``: none was probed).""" + if not meta_path.is_file(): + return _UNKNOWN + try: + meta = _read_object(meta_path) + except CellFileError: + return _UNKNOWN + host = meta.get("host") + if host is not None and not (isinstance(host, dict) and isinstance(host.get("collectors") or {}, dict)): + return _UNKNOWN # a host that fingerprint_id cannot read names no row + if host is None or not host.get("collectors"): + return None # facts from before the collectors: the cell's result.json carries no fingerprint either + return fingerprint_id(host) + + +def _unplace_rowless(missing_by: dict[tuple[str, str, str, str | None], int], + unplaced: dict[tuple[str, str, str], int], groups: dict[Any, list[dict[str, Any]]]) -> None: + """A started missing cell whose host has no finished cell has no row: it is unplaced, so the single-row rule applies.""" + row_keys = {(g[0], _cell_arm(g[1]), _cell_model(g[2]), g[3]) for g in groups} + for k in [k for k in missing_by if k not in row_keys]: + unplaced[(k[0], k[1], k[2])] += missing_by.pop(k) + + +def _row_missing(missing_by: dict[tuple[str, str, str, str | None], int], unplaced: dict[tuple[str, str, str], int], + hosts_of: dict[tuple[str, str, str], int], key: tuple[str, str, str, str | None]) -> int: + """Missing cells of one host row: those that ran on its host, and the unplaced ones when it is the only row.""" + slug_key = (key[0], _cell_arm(key[1]), _cell_model(key[2])) + count = missing_by.get((*slug_key, key[3]), 0) + if hosts_of[slug_key] == 1: + count += unplaced.get(slug_key, 0) + return count + + def summarize(batch_dir: Path) -> dict[str, Any]: cells, missing = load_cells(batch_dir) # one directory is one batch: the rows below pool by task, arm and model, so the batch name cannot split them refuse_mixed_executions([{**c, "batch": ""} for c in cells], str(batch_dir)) - groups: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list) + # One row per (task, arm, model) and host fingerprint: cells run on different hosts never pool in a row. + groups: dict[tuple[str, str, str, str | None], list[dict[str, Any]]] = defaultdict(list) for rec in cells: - groups[(rec["task"], rec["arm"], rec["model"])].append(rec) - missing_by: dict[tuple[str, str, str], int] = defaultdict(int) + groups[(rec["task"], rec["arm"], rec["model"], rec.get("host_fingerprint"))].append(rec) + # A missing cell that started has its host in meta.json and counts on that host's row. One that never + # started has none: it counts on a row only when its (task, arm, model) has a single host row. + missing_by: dict[tuple[str, str, str, str | None], int] = defaultdict(int) + unplaced: dict[tuple[str, str, str], int] = defaultdict(int) for name in missing: parts = name.split("__") if len(parts) == 4: - missing_by[(parts[0], parts[1], parts[2])] += 1 + host = _started_host(batch_dir / name / "meta.json") + if host is _UNKNOWN: + unplaced[(parts[0], parts[1], parts[2])] += 1 + else: + missing_by[(parts[0], parts[1], parts[2], host)] += 1 + hosts_of: dict[tuple[str, str, str], int] = defaultdict(int) + for g in groups: + hosts_of[(g[0], _cell_arm(g[1]), _cell_model(g[2]))] += 1 + _unplace_rowless(missing_by, unplaced, groups) rows = [] - for key in sorted(groups): + for key in sorted(groups, key=lambda k: (k[0], k[1], k[2], k[3] or "")): recs = groups[key] with reading_cells([str(batch_dir / r["cell"] / "result.json") for r in recs]): n = len(recs) @@ -252,7 +341,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: "agent_errors": sum(bool(r["agent"]["is_error"]) for r in recs), "over_budget": sum(bool(r["agent"]["over_budget"]) for r in recs), "cost_unknown": sum(r["agent"]["cost_usd"] is None for r in recs), - "missing": missing_by.get((key[0], _cell_arm(key[1]), _cell_model(key[2])), 0), + "missing": _row_missing(missing_by, unplaced, hosts_of, key), "cost_usd": _mean([r["agent"]["cost_usd"] for r in recs]), "total_cost_usd": round(sum(r["agent"]["cost_usd"] or 0 for r in recs), 6), "turns": _mean([r["agent"]["num_turns"] for r in recs]), @@ -260,6 +349,8 @@ def summarize(batch_dir: Path) -> dict[str, Any]: **{k: _mean([r["agent"].get(k) for r in recs]) for k in _TOKENS}, "networks": sorted({str(r["agent"].get("network")) for r in recs}), "model_resolved": sorted({m for r in recs for m in resolved_models(r)})} + row["host_fingerprint"] = key[3] + row.update(_labelled_match(recs)) uptake = uptake_counts(recs) if uptake is not None: row["uptake"] = uptake @@ -272,14 +363,7 @@ def summarize(batch_dir: Path) -> dict[str, Any]: else: wins = sum(bool(r["grade"]["success"]) for r in recs) row.update(kind="gate", successes=wins, success_rate=round(wins / n, 4), ci95=wilson(wins, n)) - used = [r["skills"] for r in recs if r.get("skills") and r["skills"].get("measured", True)] - if used: # cells whose trace was unreadable are not counted: no rate from a missing measurement - row["skill_measured"] = len(used) - row["skill_invoked"] = sum(bool(u["skill_invoked"]) for u in used) - expecting = [u for u in used if u.get("expected_skill")] # a task naming no skill has nothing to select - if expecting: - row["skill_expected"] = len(expecting) - row["skill_selected_correct"] = sum(bool(u["skill_selected_correct"]) for u in expecting) + row.update(_skill_counts(recs)) row["parse_errors"] = sum(r["grade"]["status"] != "grader_failure" # counted in grader_failures and _metrics(batch_dir / r["cell"]).get("parse_error") is True for r in recs) rows.append(row) @@ -338,6 +422,13 @@ def _row_notes(r: dict[str, Any]) -> list[str]: if "uptake" in r: notes.append(f"arm's additions used in {r['uptake']['used']}/{r['uptake']['cells']} cell(s)" + unmeasured_note(r["uptake"])) + if "host_match" in r: + match = r["host_match"] + what = "met" if match["met"] is True else f"unmet: {', '.join(match['unmet'])}" if match["met"] is False \ + else "undecided" + notes.append(f"host {r['host_fingerprint']} (labelled, {what})") + elif r.get("host_fingerprint"): + notes.append(f"host {r['host_fingerprint']}") return notes diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a08eefaa4..98cb1b3f8 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -77,7 +77,7 @@ from .useful import first_useful from .grading import EvalContext, failed_outcome, outcome, run_entry from .harness_profile import HarnessProfile -from .host import probe_host, unmet, validate_tasks +from .host import fingerprint_id, match_report, match_summary, probe_host, unmet, validate_tasks from .manifest import match_glob, scratch_name from .process_metrics import process_metrics from .workspace import OVERLAY_ARM, PrepError, Task, parse_arm, prepare_workspace @@ -206,10 +206,12 @@ def _workspace_args(task: Task, opts: AgentOptions, rd: Path, meta: dict[str, An def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, opts: AgentOptions, - force: bool = False, host: dict[str, Any] | None = None) -> dict[str, Any]: + force: bool = False, host: dict[str, Any] | None = None, + match: dict[str, Any] | None = None) -> dict[str, Any]: """Prepare, run the agent, grade. Returns the cell's ``result.json`` (``skipped`` when it existed). - ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``.""" + ``host`` (the probed facts of :func:`.host.probe_host`) is recorded in ``meta.json``; ``match`` (a + labelled run's :func:`.host.match_report`) too, and in ``result.json`` as ``host_match``.""" rd = batch_dir / cell_name(task.id, arm, opts.model, rep) execution = execution_record(opts, arm) done = batch_ledger.reused_result(rd, execution, force) @@ -224,6 +226,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) + if match is not None: + meta["host_match"] = match present = _workspace_args(task, opts, rd, meta) _write_json(rd / "meta.json", meta) skills: dict[str, Any] | None = None @@ -305,7 +309,10 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, "harness": opts.profile.name, "harness_version": meta["harness_version"], "base_date": meta["base_date"], "execution": execution, "score_task": bool(task.score), "score_max": task.score["max"] if task.score else None, "agent": agent, - "process": process, "first_useful": first_useful(task.useful, useful_events), "grade": grade} + "process": process, "first_useful": first_useful(task.useful, useful_events), "grade": grade, + "host_fingerprint": fingerprint_id(host) if host is not None else None} + if match is not None: + record["host_match"] = match _add_arm_use(record, meta, task, skills, uptake_of, events) try: _own_dir(rd) # the grader had the run dir read-write, and may have taken write access away @@ -368,22 +375,38 @@ def _clear_unsupported_markers(batch_dir: Path, cells: list, unsupported: dict) (batch_dir / cell_name(t.id, a, o.model, r) / UNSUPPORTED_FILE).unlink(missing_ok=True) +def _verdicts(tasks: list[Task], host: dict[str, Any], label_host: bool) -> tuple[dict, dict]: + """``(unmet verdicts, match reports)``: a block-mode batch has the verdicts, a labelled one the reports.""" + if label_host: + return {}, {t.id: match_report(t, host) for t in tasks} + return {t.id: unmet(t, host) for t in tasks}, {} + + +def _log_labelled(matches: dict[str, dict[str, Any]], log) -> None: + for task_id, match in matches.items(): + if match["met"] is not True: + log(f"{task_id}: host {match['fingerprint']} is labelled, not blocked: {match_summary(match)}") + + def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: list[AgentOptions], reps: int, batch_dir: Path, jobs: int = 1, force: bool = False, max_batch_usd: float | None = None, - log=print) -> dict[str, Any]: + log=print, label_host: bool = False) -> dict[str, Any]: """Every (task, arm, model, rep) cell; returns counts. Cells that crash are reported, not raised. The host is probed once, before any cell (``_eval/host.json``); a cell of a task whose ``host`` requirements it does not meet is marked unsupported instead of run. One that already has a ``result.json`` (from a host that met them) keeps it, even with ``force``; one that cannot be read is - that cell's error.""" + that cell's error. + + ``label_host`` runs every cell whatever the host: a cell of a task whose requirements it misses runs and + records its :func:`.host.match_report` (``host_match`` in ``meta.json`` and ``result.json``) instead.""" validate_tasks(tasks) cells = [(t, a, o, r) for t in tasks for a in arms for o in options for r in range(1, reps + 1)] _refuse_shared_cells(cells) check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells} if force else None) host = probe_host(ctx, batch_dir / "_eval") _write_json(batch_dir / "_eval" / "host.json", host) - verdicts = {t.id: unmet(t, host) for t in tasks} + verdicts, matches = _verdicts(tasks, host, label_host) unsupported = {task_id: why for task_id, (why, _) in verdicts.items() if why} if force and unsupported: # --force keeps their finished cells, so those must match this run too check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells @@ -392,6 +415,7 @@ def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: lis probe_failed = _probe_failed(batch_dir, cells, verdicts) for task_id, why in unsupported.items(): log(f"{task_id}: UNSUPPORTED, its cells are not run: {why}") + _log_labelled(matches, log) _clear_unsupported_markers(batch_dir, cells, unsupported) lock = threading.Lock() # The cap counts what earlier invocations of this batch spent (under the batch lock the caller holds), and @@ -421,7 +445,7 @@ def one(cell: tuple[Task, str, AgentOptions, int]) -> None: return charged, rec = 0.0, None try: - rec = run_cell(ctx, task, arm, rep, batch_dir, opts, force=force, host=host) + rec = run_cell(ctx, task, arm, rep, batch_dir, opts, force=force, host=host, match=matches.get(task.id)) except CellCrash as exc: charged = exc.charged_usd with lock: diff --git a/dokimi/tests/data/cli_flags_allowlist.txt b/dokimi/tests/data/cli_flags_allowlist.txt index fbf2192fa..af96ea6e7 100644 --- a/dokimi/tests/data/cli_flags_allowlist.txt +++ b/dokimi/tests/data/cli_flags_allowlist.txt @@ -265,3 +265,4 @@ --canaries-file # repo security: the same canaries one per line (#360) --tool # skill novelty ground (#472): the command whose --help is the source; no registry concept yet --tool-version # skill novelty ground (#472): recorded with the help snapshot; no registry concept yet +--label-host # host label mode (contract §4.3) has no registry concept yet; F1b moves it with --allow-unsupported diff --git a/dokimi/tests/test_batch_ledger_golden.py b/dokimi/tests/test_batch_ledger_golden.py index 7b0f823ce..35da34caf 100644 --- a/dokimi/tests/test_batch_ledger_golden.py +++ b/dokimi/tests/test_batch_ledger_golden.py @@ -109,6 +109,7 @@ "grade", "harness", "harness_version", + "host_fingerprint", "model", "model_resolved", "process", diff --git a/dokimi/tests/test_host_fingerprint.py b/dokimi/tests/test_host_fingerprint.py new file mode 100644 index 000000000..d21e12a2b --- /dev/null +++ b/dokimi/tests/test_host_fingerprint.py @@ -0,0 +1,449 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The host fingerprint's collectors (contract §3): per platform, failure isolation, privacy. + +No docker and no real tool is run: spawns go to a :class:`FakeRunner`, files come from a dict. +The privacy tests also run the collectors on this machine, with its real host and user names. +""" + +from __future__ import annotations + +import getpass +import json +import os +import platform +import re +import socket +import sys +from pathlib import Path + +import pytest + +from dokimi.process import FakeRunner, reply +from dokimi.repos import host_fingerprint as hf +from dokimi.repos.host import fingerprint_id, match_report, probe_host +from dokimi.repos.container import ContainerResult +from dokimi.repos.grading import EvalContext +from dokimi.repos.manifest import parse_manifest +from dokimi.repos.workspace import Task +from repo_eval_fixture import MANIFEST + +INTEL_CPUINFO = ("processor\t: 0\nvendor_id\t: GenuineIntel\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n" + "flags\t\t: fpu sse2 avx2 avx512f hypervisor\n") +ROUTE = ("Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\n" + "eth0\t00000000\t0102A8C0\t0003\t0\t0\t100\t00000000\n") + + +def _files(**paths: str) -> dict[str, str]: + """A fake filesystem: ``_files(**{"/proc/meminfo": ...})`` keys are paths, written with ``__`` for ``/``.""" + return {p.replace("__", "/"): text for p, text in paths.items()} + + +def _probe(system: str = "Linux", files: dict[str, str] | None = None, calls: dict | None = None, **kw): + """A :class:`Probe` whose files are ``files`` and whose spawns answer ``calls[argv[0]]`` (or nothing).""" + files = files or {} + answers = calls or {} + + def handler(call): + key = " ".join(call.argv) + if key in answers: + return reply(answers[key]) + return reply(returncode=1) + + runner = FakeRunner(handler) + kw.setdefault("which", lambda name: None) + return hf.Probe(system=system, read=lambda path: files.get(path), run=runner, + env=kw.pop("env", {}), **kw), runner + + +def _leaves(value): + if isinstance(value, dict): + for item in value.values(): + yield from _leaves(item) + else: + yield value + + +def _paths_in(text: str) -> list[str]: + """Absolute-looking paths in a serialised record: a slash or drive letter at the start of a token.""" + return re.findall(r"(?:^|[\s\"'=(,])(/[\w.+@%-]|[A-Za-z]:\\)", text) + + +# ------------------------------------------------------------------ the registry + + +def test_each_collector_gives_a_flat_dict_of_json_scalars_and_the_version_is_recorded(): + probe, _ = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO, "/proc__meminfo": "MemTotal: 2048 kB\n", + "/proc__net__route": ROUTE})) + out = hf.collect(probe) + assert out["collectors_version"] == hf.COLLECTORS_VERSION == 1 + assert sorted(out["collectors"]) == sorted(hf.COLLECTORS) == [ + "cpu", "gpu", "memory", "network", "os", "python", "tools", "virt"] + for name, facts in out["collectors"].items(): + assert isinstance(facts, dict), name + for item in facts.values(): + assert item is None or isinstance(item, (str, int, float, bool)), (name, item) + assert out["collectors"]["memory"] == {"total_bytes": 2048 * 1024} + assert out["collectors"]["cpu"]["vendor"] == "GenuineIntel" + + +def test_a_failing_collector_reports_its_error_and_the_others_still_run(monkeypatch): + def broken(probe): + raise OSError(f"[Errno 2] No such file or directory: '{Path.home()}/nvidia/smi'") + + monkeypatch.setitem(hf.COLLECTORS, "gpu", broken) + probe, _ = _probe(files=_files(**{"/proc__meminfo": "MemTotal: 1024 kB\n"})) + out = hf.collect(probe) + assert out["collectors"]["gpu"]["error"].startswith("OSError: ") + assert out["collectors"]["memory"] == {"total_bytes": 1024 * 1024} + assert "nvidia" not in out["collectors"]["gpu"]["error"] and "" in out["collectors"]["gpu"]["error"] + + +def test_probe_host_keeps_the_container_keys_and_adds_the_fingerprint(tmp_path): + """The existing keys (cpus, cpu_vendor, ...) are unchanged; the fingerprint sits beside them.""" + ctx = EvalContext(manifest=parse_manifest(MANIFEST), git_dir=tmp_path, content_dir=tmp_path, image="img:1", + runner=_ContainerAnswer({"affinity": 8, "cpu_max": "max 100000\n", "cfs_quota_us": None, + "cfs_period_us": None, "cpuinfo": INTEL_CPUINFO})) + facts = probe_host(ctx, tmp_path / "w", probe=_probe()[0]) + assert {"cpus", "affinity_cpus", "cgroup_cpus", "cpu_vendor", "cpu_model", "cpu_flags"} <= set(facts) + assert facts["cpus"] == 8 and facts["cpu_vendor"] == "GenuineIntel" + assert facts["collectors_version"] == 1 and set(facts["collectors"]) == set(hf.COLLECTORS) + + +class _ContainerAnswer: + """A runner of the probe container that prints ``answer`` as its JSON line.""" + + def __init__(self, answer): + self.answer = answer + + def run(self, spec, *, stdout, stderr, timeout_s): + stdout.write_text(json.dumps(self.answer) + "\n", encoding="utf-8") + return ContainerResult(returncode=0, timed_out=False, wall_s=0.0) + + +# ------------------------------------------------------------------ per platform + + +def test_linux_host_reads_its_files_and_spawns_nothing_it_does_not_need(): + probe, runner = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO, "/proc__meminfo": "MemTotal: 4 kB\n", + "/etc__os-release": 'NAME="Ubuntu"\nPRETTY_NAME="Ubuntu 24.04 LTS"\n'})) + out = hf.collect(probe)["collectors"] + assert out["os"] == {"system": "Linux", "release": platform.release(), "machine": platform.machine(), + "distro": "Ubuntu 24.04 LTS"} + assert out["cpu"]["flags"] == "fpu sse2 avx2 avx512f hypervisor" + assert out["virt"] == {"container": False, "container_kind": None, "vm": True, "vm_vendor": None} + assert runner.argvs() == [] # no tool on the path, so nothing ran + + +def test_macos_reads_sysctl_and_no_proc_file(tmp_path): + answers = {"sysctl -n hw.memsize": "17179869184\n", "sysctl -n machdep.cpu.brand_string": "Apple M2\n", + "sysctl -n kern.hv_vmm_present": "0\n"} + probe, runner = _probe(system="Darwin", calls=answers, files=_files(**{"/proc__meminfo": "MemTotal: 1 kB\n"})) + out = hf.collect(probe)["collectors"] + assert out["memory"] == {"total_bytes": 17179869184} + assert out["cpu"]["model"] == "Apple M2" and out["virt"]["vm"] is False + assert out["os"]["system"] == "Darwin" and "distro" not in out["os"] + assert {argv[0] for argv in runner.argvs()} == {"sysctl"} + + +def test_macos_without_a_number_from_sysctl_is_an_error_for_that_field_only(): + probe, _ = _probe(system="Darwin", calls={"sysctl -n hw.memsize": "not a number\n"}) + assert hf.collect(probe)["collectors"]["memory"] == {"error": "sysctl hw.memsize gave no number"} + + +def test_windows_spawns_only_cross_platform_calls_and_reads_no_proc(): + which = lambda name: rf"C:\tools\{name}.exe" if name in ("git", "nvidia-smi") else None # noqa: E731 + calls = {"git --version": "git version 2.45.1.windows.1\n", "nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": + "NVIDIA RTX A4000, 551.23\n", "nvidia-smi": "CUDA Version: 12.4"} + probe, runner = _probe(system="Windows", which=which, calls=calls, windows_memory=lambda: 16 * 2**30, + windows_cpu=lambda: ("Intel(R) Core(TM) i7", "GenuineIntel"), + files=_files(**{"/proc__meminfo": "MemTotal: 1 kB\n"})) + out = hf.collect(probe)["collectors"] + assert out["memory"] == {"total_bytes": 16 * 2**30} + assert out["cpu"]["vendor"] == "GenuineIntel" and out["cpu"]["flags"] is None + assert out["cpu"]["model"] == "Intel(R) Core(TM) i7" + assert out["tools"]["git"] == "2.45.1" and out["tools"]["docker"] is None + assert out["gpu"] == {"present": True, "count": 1, "name": "NVIDIA RTX A4000", "driver_version": "551.23", + "cuda_version": "12.4"} + assert out["virt"] == {"container": False, "container_kind": None, "vm": None, "vm_vendor": None} + assert {argv[0] for argv in runner.argvs()} <= {"git", "nvidia-smi"} # never sysctl, lscpu, free, cat + + +def test_windows_cpu_model_never_reads_the_processor_identifier_environment_value(monkeypatch): + monkeypatch.setenv("PROCESSOR_IDENTIFIER", "GenuineIntel workstation-jdoe") + monkeypatch.setattr(platform, "processor", lambda: os.environ["PROCESSOR_IDENTIFIER"]) # CPython's Windows source + probe, _ = _probe(system="Windows", windows_memory=lambda: 2**30, windows_cpu=lambda: (None, None)) + out = hf.collect(probe) + assert "workstation-jdoe" not in json.dumps(out) + assert out["collectors"]["cpu"]["model"] is None + + +@pytest.mark.skipif(sys.platform != "win32", reason="reads the registry on Windows only") +def test_windows_cpu_is_read_from_the_registry(): + model, vendor = hf.Probe(system="Windows").windows_cpu() + assert model and vendor + + +@pytest.mark.skipif(sys.platform != "win32", reason="reads the kernel's memory status on Windows only") +def test_windows_memory_is_the_kernels_total(): + assert hf.Probe(system="Windows").windows_memory() > 0 + + +# ------------------------------------------------------------------ gpu, virt, network, tools, python + + +def test_gpu_reports_driver_and_cuda_when_nvidia_smi_is_on_the_path(): + rows = "NVIDIA A100-SXM4-80GB, 550.54.14\nNVIDIA A100-SXM4-80GB, 550.54.14\n" + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": rows, + "nvidia-smi": "| NVIDIA-SMI 550.54.14 Driver Version: 550.54.14 CUDA Version: 12.4 |"} + probe, _ = _probe(calls=calls, which=lambda name: "/usr/bin/nvidia-smi" if name == "nvidia-smi" else None) + assert hf.collect(probe)["collectors"]["gpu"] == { + "present": True, "count": 2, "name": "NVIDIA A100-SXM4-80GB", "driver_version": "550.54.14", + "cuda_version": "12.4"} + + +def test_gpu_without_nvidia_smi_is_absent_and_a_failing_query_is_an_error(): + assert hf.collect(_probe()[0])["collectors"]["gpu"] == {"present": False} + probe, _ = _probe(which=lambda name: "/usr/bin/nvidia-smi") # on the path, but the query exits 1 + assert hf.collect(probe)["collectors"]["gpu"] == { + "present": True, "error": "nvidia-smi --query-gpu=name,driver_version failed"} + + +@pytest.mark.parametrize("files, expected", [ + ({"/.dockerenv": ""}, {"container": True, "container_kind": "docker"}), + ({"/run__.containerenv": ""}, {"container": True, "container_kind": "podman"}), + ({"/proc__self__cgroup": "0::/kubepods/burstable/pod1/abc\n"}, {"container": True, "container_kind": "kubernetes"}), + ({"/proc__self__cgroup": "0::/init.scope\n"}, {"container": False, "container_kind": None}), +]) +def test_virt_detects_a_container_from_its_markers(files, expected): + probe, _ = _probe(files=_files(**files)) + virt = hf.collect(probe)["collectors"]["virt"] + assert {key: virt[key] for key in expected} == expected + + +def test_virt_names_a_vm_from_dmi_or_the_hypervisor_flag(): + dmi = {"/sys__class__dmi__id__sys_vendor": "QEMU\n", "/sys__class__dmi__id__product_name": "Standard PC\n"} + probe, _ = _probe(files=_files(**dmi)) + assert hf.collect(probe)["collectors"]["virt"]["vm_vendor"] == "qemu" + probe, _ = _probe(files=_files(**{"/proc__cpuinfo": INTEL_CPUINFO})) # `hypervisor` in the flags + assert hf.collect(probe)["collectors"]["virt"]["vm"] is True + probe, _ = _probe(files=_files(**{"/sys__class__dmi__id__sys_vendor": "Dell Inc.\n"})) + assert hf.collect(probe)["collectors"]["virt"]["vm"] is False + + +def test_network_counts_and_booleans_and_never_an_address_or_a_proxy_value(monkeypatch): + monkeypatch.setattr(socket, "if_nameindex", lambda: [(1, "lo"), (2, "eth0")]) + probe, _ = _probe(files=_files(**{"/proc__net__route": ROUTE}), env={"HTTPS_PROXY": "http://10.1.2.3:3128"}) + net = hf.collect(probe)["collectors"]["network"] + assert net == {"interfaces": 2, "default_route": True, "proxy_env": True} + assert "10.1.2.3" not in json.dumps(net) + + +def test_tools_record_the_version_number_and_no_path(): + probe, _ = _probe(which=lambda name: f"/usr/bin/{name}" if name in ("git", "gcc") else None, + calls={"git --version": "git version 2.43.0\n", "gcc --version": "gcc (GCC) 13.2.0\nCopyright"}) + tools = hf.collect(probe)["collectors"]["tools"] + assert tools == {"git": "2.43.0", "docker": None, "cmake": None, "gcc": "13.2.0", "clang": None, "make": None} + probe, _ = _probe(which=lambda name: f"/usr/bin/{name}" if name == "make" else None, calls={}) + assert hf.collect(probe)["collectors"]["tools"]["make"] == "present" # on the path, its version not printed + + +def test_a_tool_that_cannot_be_started_keeps_the_versions_of_the_others(): + def handler(call): + if call.argv[0] == "make": + raise OSError("[Errno 2] No such file or directory: 'make'") # `which` found it; the spawn fails + return reply("git version 2.43.0\n") if " ".join(call.argv) == "git --version" else reply(returncode=1) + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(handler), env={}, + which=lambda name: f"/usr/bin/{name}" if name in ("git", "make") else None) + tools = hf.collect(probe)["collectors"]["tools"] + assert tools == {"git": "2.43.0", "docker": None, "cmake": None, "gcc": None, "clang": None, "make": "present"} + + +def test_python_names_the_implementation_and_not_the_executable(): + python = hf.collect(_probe()[0])["collectors"]["python"] + assert python["version"] == platform.python_version() and isinstance(python["free_threaded"], bool) + assert sys.executable not in json.dumps(python) + + +# ------------------------------------------------------------------ privacy + + +def test_privacy_real_host_and_user_names_and_addresses_never_appear(monkeypatch): + monkeypatch.setattr(platform, "release", lambda: "5.15.167.4-microsoft-standard-WSL2") # a WSL2 kernel release + facts = hf.collect(hf.Probe()) + text = json.dumps(facts, sort_keys=True) + for secret in {socket.gethostname(), getpass.getuser(), os.path.expanduser("~")}: + if len(secret) >= 4: + assert secret not in text, "a host or user name leaked into the fingerprint" + # Versions (a four-part kernel release such as 5.15.167.4, a tool version) are dotted quads, not addresses. + versions = {"os": ("release",), "python": ("version",), "tools": tuple(hf.TOOLS)} + unversioned = {name: {k: v for k, v in c.items() if k not in versions.get(name, ())} + for name, c in facts["collectors"].items() if isinstance(c, dict)} + assert not re.search(r"\b\d{1,3}(?:\.\d{1,3}){3}\b", json.dumps(unversioned)), "an IPv4 address leaked" + assert _paths_in(text) == [], "an absolute path leaked into the fingerprint" + + +def test_privacy_monkeypatched_identities_and_paths_never_appear(monkeypatch, tmp_path): + monkeypatch.setattr(platform, "node", lambda: "tower-7f3a") + monkeypatch.setattr(socket, "gethostname", lambda: "tower-7f3a") + monkeypatch.setattr(socket, "if_nameindex", lambda: [(1, "lo"), (2, "eth0")]) + monkeypatch.setattr(getpass, "getuser", lambda: "jdoe") + monkeypatch.setenv("HOME", "/home/jdoe") + files = _files(**{"/proc__net__route": ROUTE.replace("eth0", "/home/jdoe/eth0"), + "/proc__cpuinfo": INTEL_CPUINFO, "/etc__os-release": 'PRETTY_NAME="Ubuntu 24.04 LTS"\n', + "/sys__class__dmi__id__sys_vendor": "Lenovo at 10.20.30.40\n"}) + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": "A, 1\n", "nvidia-smi": "CUDA Version: 12.4"} + probe, _ = _probe(files=files, calls=calls, env={"HOME": "/home/jdoe", "HTTP_PROXY": "http://10.20.30.40:3128"}, + which=lambda name: f"/home/jdoe/bin/{name}" if name == "nvidia-smi" else None) + monkeypatch.setattr(hf, "COLLECTORS", {**hf.COLLECTORS, "tools": lambda p: 1 / 0}) # a failing collector's text + text = json.dumps(hf.collect(probe), sort_keys=True) + for leak in ("tower-7f3a", "jdoe", "10.20.30.40", "/home", str(tmp_path)): + assert leak not in text, leak + assert "ZeroDivisionError: division by zero" in text and _paths_in(text) == [] + + +def test_error_text_is_scrubbed_of_paths_and_truncated(): + assert hf.scrub("missing '/home/jdoe/.cache/x' and C:\\Users\\jdoe\\y") == "missing '' and " + assert len(hf.scrub("x" * 1000)) == 300 + + +@pytest.mark.parametrize("text, kept", [ + ("missing '/home/Jane Doe/private'", "missing ''"), + ("missing 'C:\\Users\\Jane Doe\\AppData\\x'", "missing ''"), + ("cannot open \\\\server\\users\\jdoe\\file", "cannot open "), + ("cannot open '\\\\server\\users\\Jane Doe\\file'", "cannot open ''"), + ("no such file ~/secret/x", "no such file "), + ("no such file '~/Jane Doe/x'", "no such file ''"), + ("missing \"/home/Jane O'Connor/private\"", "missing \"\""), + ("missing C:\\Users\\Jane Doe\\AppData\\x", "missing "), + ("missing C:\\Users\\O'Connor\\private.txt", "missing "), + ("cannot open \\\\server\\users\\O'Connor\\file", "cannot open "), + ("cannot open path:/home/jdoe/private", "cannot open path:"), + ("cannot open /home/Jane Doe/private", "cannot open "), + ("cannot open /home/Jane Doe/private: Permission denied", "cannot open "), + ("cannot open /home/张三/private", "cannot open "), + ("no such file ~/Jane Doe/x", "no such file "), + ("cannot open file:///home/Jane Doe/private", "cannot open file://"), + ("cannot open file:/home/Jane Doe/private", "cannot open file:"), + ("cannot open FILE:///home/jdoe/x", "cannot open FILE://"), + ("cannot open file://server/share/Jane Doe/x", "cannot open file:"), + ("missing \"file:///home/Jane O'Connor/x\", next", "missing \"\", next"), + ("cannot open //server/share/Jane Doe/file", "cannot open "), + ("cannot open '//server/share/Jane Doe/file'", "cannot open ''"), + ("see https://example.com/a b", "see https://example.com/a b"), + ("see http://example.com/a/b c", "see http://example.com/a/b c"), + ("host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom", + "host probe exited 1 in ghcr.io/intel/dokimi-grader:1 (python3 -c): boom"), +]) +def test_scrub_redacts_quoted_unc_and_home_relative_paths_whole(text, kept): + assert hf.scrub(text) == kept + + +def test_a_collector_error_naming_a_path_with_a_space_is_stored_without_it(monkeypatch): + def boom(probe): + raise OSError(2, "No such file", "/home/Jane Doe/private") + + monkeypatch.setattr(hf, "COLLECTORS", {**hf.COLLECTORS, "tools": boom}) + probe, _ = _probe() + error = hf.collect(probe)["collectors"]["tools"]["error"] + assert "Jane" not in error and "Doe" not in error and "" in error + + +# ------------------------------------------------------------------ the identity of a host + + +def test_the_fingerprint_id_is_stable_and_ignores_the_network_but_not_the_cpu(): + facts = {"cpus": 4, "cpu_vendor": "GenuineIntel", "cpu_model": "Xeon", "cpu_flags": ["avx2"], + "collectors": {"os": {"release": "6.8"}, "network": {"interfaces": 2}, "virt": {"container": False}}} + moved = {**facts, "collectors": {**facts["collectors"], "network": {"interfaces": 9, "proxy_env": True}}} + assert fingerprint_id(facts) == fingerprint_id(moved) + assert fingerprint_id(facts) != fingerprint_id({**facts, "cpu_model": "Epyc"}) + assert len(fingerprint_id(facts)) == 12 + + +@pytest.mark.parametrize("name", ["os", "cpu", "memory", "gpu", "python", "tools", "virt"]) +def test_the_fingerprint_id_changes_with_every_identity_collector(name): + base = {n: {"k": "a"} for n in ["os", "cpu", "memory", "gpu", "python", "tools", "virt"]} + facts = {"cpus": 4, "collectors": base} + # tools count by presence only (a version change is no change): its change is a tool that goes away + changed_value = {"k": None} if name == "tools" else {"k": "b"} + changed = {"cpus": 4, "collectors": {**base, name: changed_value}} + assert fingerprint_id(facts) != fingerprint_id(changed) + + +def test_match_report_for_a_host_with_no_task_requirement_is_met_without_fields(): + task = Task("t", {"id": "t"}, Path(".")) + assert match_report(task, {"cpus": 2})["met"] is True + assert match_report(task, {"cpus": 2})["fields"] == [] + + +# ------------------------------------------------------------------ identity: presence, failures, one retry (ZGgG) + + +def test_a_tool_version_change_does_not_change_the_fingerprint_id(): + base = {"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": None}}} + upgraded = {"cpus": 4, "collectors": {"tools": {"git": "2.45.1", "docker": None}}} + timed_out = {"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": "present"}}} + assert fingerprint_id(base) == fingerprint_id(upgraded) + assert fingerprint_id(timed_out) == fingerprint_id({"cpus": 4, "collectors": {"tools": {"git": "2.43.0", "docker": "24.0.7"}}}) + + +def test_a_tool_that_appears_or_disappears_changes_the_fingerprint_id(): + absent = {"cpus": 4, "collectors": {"tools": {"docker": None}}} + present = {"cpus": 4, "collectors": {"tools": {"docker": "24.0.7"}}} + assert fingerprint_id(absent) != fingerprint_id(present) + + +def test_a_failing_collector_is_left_out_of_the_fingerprint_id(): + ok = {"cpus": 4, "collectors": {"gpu": {"present": True, "count": 1}, "os": {"release": "6.8"}}} + failed = {"cpus": 4, "collectors": {"gpu": {"present": True, "error": "nvidia-smi timed out"}, + "os": {"release": "6.8"}}} + left_out = {"cpus": 4, "collectors": {"os": {"release": "6.8"}}} + assert fingerprint_id(failed) == fingerprint_id(left_out) + assert fingerprint_id(failed) != fingerprint_id(ok) + + +def test_a_collector_that_fails_once_is_retried_and_its_retry_is_kept(): + calls = {"n": 0} + + def flaky(probe): + calls["n"] += 1 + if calls["n"] == 1: + raise OSError("transient") + return {"total_bytes": 8} + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(lambda call: reply(returncode=1)), + env={}, which=lambda name: None) + original = dict(hf.COLLECTORS) + try: + hf.COLLECTORS.clear() + hf.COLLECTORS["memory"] = flaky + out = hf.collect(probe)["collectors"]["memory"] + finally: + hf.COLLECTORS.clear() + hf.COLLECTORS.update(original) + assert out == {"total_bytes": 8} + assert calls["n"] == 2 + + +def test_a_collector_that_fails_twice_is_one_retry_then_an_error(): + calls = {"n": 0} + + def broken(probe): + calls["n"] += 1 + return {"error": "still broken"} + + probe = hf.Probe(system="Linux", read=lambda path: None, run=FakeRunner(lambda call: reply(returncode=1)), + env={}, which=lambda name: None) + original = dict(hf.COLLECTORS) + try: + hf.COLLECTORS.clear() + hf.COLLECTORS["memory"] = broken + out = hf.collect(probe)["collectors"]["memory"] + finally: + hf.COLLECTORS.clear() + hf.COLLECTORS.update(original) + assert out == {"error": "still broken"} + assert calls["n"] == 2 diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index 845d2071f..ab171e576 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -933,3 +933,16 @@ def test_a_skill_arm_cell_of_an_unsupported_task_is_marked_not_run(tmp_path, mon assert [p.name for p in (batch / "fix_race__skill-lint__m-1__r1").iterdir()] == ["unsupported.json"] assert (batch / "fix_add__skill-lint__m-1__r1/result.json").is_file() assert {u["cell"] for u in summarize(batch)["unsupported"]} >= {"fix_race__skill-lint__m-1__r1"} + + +def test_a_probe_container_that_fails_to_start_has_its_error_scrubbed(tmp_path): + from dokimi.repos.container import ContainerError + + class RuntimeFails(HostRunner): + def run(self, spec, *, stdout, stderr, timeout_s): + raise ContainerError("docker run failed: Error loading config file: /home/jdoe/.docker/config.json") + + ctx = EvalContext(manifest=parse_manifest(MANIFEST), git_dir=tmp_path, content_dir=tmp_path, image="img:1", + runner=RuntimeFails(None)) + error = probe_host(ctx, tmp_path / "w")["error"] + assert "/home/jdoe" not in error and "host probe did not start" in error diff --git a/dokimi/tests/test_repo_host_label.py b/dokimi/tests/test_repo_host_label.py new file mode 100644 index 000000000..8068a2cd7 --- /dev/null +++ b/dokimi/tests/test_repo_host_label.py @@ -0,0 +1,206 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo run --label-host`: run a task the host misses, and record which requirements it met (contract §4.3). + +Four host scenarios end to end: an AVX-512 task on an AMD host, a Windows host, a container, and a GPU host. +The host's fingerprint comes from a fake :class:`Probe`; the grading containers are a :class:`FakeRunner`. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from dokimi.cli import build_parser +from dokimi.repos.host import host_facts, match_report, match_summary, probe_host as real_probe_host +from dokimi.repos.host_fingerprint import Probe +from dokimi.process import FakeRunner as HostFakeRunner +from dokimi.process import reply +from dokimi.repos.report import format_report, summarize +from dokimi.repos.workspace import Task +from repo_eval_fixture import make_repo +from test_repo_host import AMD_CPUINFO, INTEL_CPUINFO, RACE, HostRunner, _probe_out, _run + +AVX = dict(RACE, host={"cpu_flags": ["avx512f"]}) # no CPU count, no vendor: a flag is the only requirement + + +def _task(host) -> Task: + return Task("t", {"id": "t", "host": host}, Path(".")) + + +def _repo(tmp_path): + root, _ = make_repo(tmp_path, {"fix_avx": AVX, "fix_race": RACE}) + return root + + +def _host(system="Linux", files=None, calls=None, **kw) -> Probe: + """A fake host: ``files`` are what its /proc, /sys and /etc read; ``calls`` answer its spawns.""" + files = files or {} + answers = calls or {} + + def handler(call): + text = answers.get(" ".join(call.argv)) + return reply(text) if text is not None else reply(returncode=1) + + kw.setdefault("which", lambda name: None) + return Probe(system=system, read=lambda path: files.get(path), run=HostFakeRunner(handler), env={}, **kw) + + +def _labelled(monkeypatch, host: Probe) -> None: + """Probe this batch's host with ``host`` (the container probe is still the fake runner's).""" + monkeypatch.setattr("dokimi.repos.runner.probe_host", lambda ctx, work: real_probe_host(ctx, work, probe=host)) + + +def _cell_json(out: Path, task: str, name: str) -> dict: + """The first rep of ``task``'s cell file ``name`` (``result.json`` or ``meta.json``), as an object.""" + return json.loads(next((out / "b1").glob(f"{task}__*__r1/{name}")).read_text(encoding="utf-8")) + + +# ------------------------------------------------------------------ the match report + + +def test_an_avx512_task_on_an_amd_host_is_a_labelled_miss_on_the_flag_alone(): + amd = host_facts(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)) + report = match_report(_task({"cpu_flags": ["avx512f"]}), amd) + assert report["met"] is False + assert report["fields"] == [{"field": "cpu_flags.avx512f", "required": "avx512f", "actual": False, "met": False}] + assert match_summary(report) == "cpu_flags.avx512f missing" + + +def test_a_met_requirement_and_a_probe_that_could_not_tell_are_recorded_as_such(): + intel = host_facts(_probe_out(affinity=8, cpuinfo=INTEL_CPUINFO)) + met = match_report(_task({"min_cpus": 4, "cpu_vendors": ["GenuineIntel"], "cpu_flags": ["avx2"]}), intel) + assert met["met"] is True and [f["met"] for f in met["fields"]] == [True, True, True] + undecided = match_report(_task({"min_cpus": 4, "cpu_flags": ["avx2"]}), {"error": "host probe exited 127"}) + assert undecided["met"] is None and [f["met"] for f in undecided["fields"]] == [None, None] + assert match_summary(undecided) == "min_cpus unknown, cpu_flags.avx2 unknown" + + +def test_a_vendor_miss_the_min_cpus_boundary_and_an_uppercase_flag_are_judged_exactly(): + intel = host_facts(_probe_out(affinity=4, cpuinfo=INTEL_CPUINFO)) + vendor = match_report(_task({"cpu_vendors": ["AuthenticAMD"]}), intel) + assert vendor["met"] is False and match_summary(vendor) == "cpu_vendors missing" + assert match_report(_task({"cpu_vendors": ["genuineintel"]}), intel)["met"] is True # vendor compare ignores case + assert match_report(_task({"min_cpus": 4}), intel)["met"] is True # exactly min_cpus is enough + assert match_report(_task({"min_cpus": 5}), intel)["met"] is False + assert match_report(_task({"cpu_flags": ["AVX2"]}), intel)["met"] is True # docs write AVX2, cpuinfo avx2 + + +def test_one_unmet_and_one_undecided_field_is_a_miss(): + facts = {"cpus": 2, "cpu_vendor": None, "cpu_flags": []} # the vendor and flags are unknown, the CPU count is not + report = match_report(_task({"min_cpus": 4, "cpu_flags": ["avx2"]}), facts) + assert [f["met"] for f in report["fields"]] == [False, None] + assert report["met"] is False + + +# ------------------------------------------------------------------ the four host scenarios + + +def test_label_runs_the_avx512_task_on_an_amd_host_and_records_the_miss(tmp_path, monkeypatch, capsys): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + cell = _cell_json(out, "fix_avx", "result.json") + assert cell["host_match"]["met"] is False + assert cell["host_match"]["fingerprint"] == cell["host_fingerprint"] + assert [f["field"] for f in cell["host_match"]["fields"] if f["met"] is False] == ["cpu_flags.avx512f"] + assert not list((out / "b1").glob("fix_avx__*/unsupported.json")) # ran, not marked unsupported + logged = capsys.readouterr() + assert "labelled, not blocked" in logged.out + logged.err + + +def test_without_label_the_same_task_on_the_same_host_is_unsupported(tmp_path, monkeypatch): + """Regression: block stays the default; the labelled cells exist only with --label-host.""" + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + with pytest.raises(SystemExit) as exc: # every cell unsupported: `repo run` has nothing and exits 1 + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--task", "fix_avx") + assert exc.value.code == 1 + assert len(list((out / "b1").glob("fix_avx__*/unsupported.json"))) == 2 + assert not list((out / "b1").glob("fix_avx__*/result.json")) + + +def test_label_on_a_windows_host_groups_under_the_windows_fingerprint(tmp_path, monkeypatch): + """The requirements are checked against the grading container's CPU view, so a Linux container on a + Windows Docker host meets them; the host's own collectors are what the fingerprint records.""" + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(system="Windows", windows_memory=lambda: 8 * 2**30, + calls={"git --version": "git version 2.45.1\n"}, + which=lambda name: r"C:\git.exe" if name == "git" else None)) + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--label-host", "--task", "fix_avx") + cell = _cell_json(out, "fix_avx", "result.json") + assert cell["host_match"]["met"] is True and cell["host_match"]["fields"][0]["field"] == "cpu_flags.avx512f" + meta = _cell_json(out, "fix_avx", "meta.json") + assert meta["host"]["collectors"]["os"]["system"] == "Windows" + assert meta["host"]["collectors"]["memory"] == {"total_bytes": 8 * 2**30} + assert meta["host"]["collectors"]["tools"]["git"] == "2.45.1" + + +def test_label_on_a_container_records_the_container_in_the_fingerprint(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/.dockerenv": "", "/proc/cpuinfo": INTEL_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8)), monkeypatch, "--label-host", "--task", "fix_race") + meta = _cell_json(out, "fix_race", "meta.json") + assert meta["host"]["collectors"]["virt"]["container"] is True + assert meta["host"]["collectors"]["virt"]["container_kind"] == "docker" + assert meta["host_match"]["met"] is True # RACE needs 4 CPUs of GenuineIntel, the container's cgroup gives 8 + + +def test_label_on_a_gpu_host_records_the_driver_and_cuda_version(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + calls = {"nvidia-smi --query-gpu=name,driver_version --format=csv,noheader": "A100, 550.54.14\n", + "nvidia-smi": "CUDA Version: 12.4"} + _labelled(monkeypatch, _host(calls=calls, which=lambda name: "/usr/bin/nvidia-smi" if name == "nvidia-smi" else None)) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + meta = _cell_json(out, "fix_avx", "meta.json") + assert meta["host"]["collectors"]["gpu"]["cuda_version"] == "12.4" + assert meta["host"]["collectors"]["gpu"]["driver_version"] == "550.54.14" + + +# ------------------------------------------------------------------ results group by the actual fingerprint + + +def test_report_splits_a_batch_by_the_fingerprint_of_the_host_that_ran_each_cell(tmp_path, monkeypatch): + root = _repo(tmp_path) + out = tmp_path / "runs" + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": AMD_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=AMD_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + first = sorted((out / "b1").glob("fix_avx__*/result.json"))[1] + first.unlink() # the second rep is run again, on another host + _labelled(monkeypatch, _host(files={"/proc/cpuinfo": INTEL_CPUINFO})) + _run(root, out, HostRunner(_probe_out(affinity=8, cpuinfo=INTEL_CPUINFO)), monkeypatch, "--label-host", + "--task", "fix_avx") + rows = [r for r in summarize(out / "b1")["rows"] if r["task"] == "fix_avx"] + assert len(rows) == 2 and rows[0]["host_fingerprint"] != rows[1]["host_fingerprint"] + assert [r["n"] for r in rows] == [1, 1] + assert sorted(r["host_match"]["met"] for r in rows) == [False, True] # AMD misses avx512f, Intel has it + assert "labelled, unmet: cpu_flags.avx512f" in format_report(summarize(out / "b1")) + + +def test_the_flag_is_off_by_default_and_on_with_label_host(): + argv = ["repo", "run", "/x@HEAD", "--arm", "guidance:raw", "--model", "m", "--batch", "b"] + assert build_parser().parse_args(argv).label_host is False + assert build_parser().parse_args([*argv, "--label-host"]).label_host is True + + +def test_compare_warns_when_a_labelled_cell_ran_on_a_host_that_misses_its_task(tmp_path): + from dokimi.repos.compare import _arm_warnings + + def cell(arm, met): + rec = {"task": "race", "arm": arm, "model": "m-1", "batch": "b"} + return rec if met is None else {**rec, "host_match": {"met": met, "fields": [], "fingerprint": "f"}} + + cells = [cell("raw", None), cell("skill", False)] # the treatment ran one labelled cell on a host that misses + assert any("race" in w and "--label-host" in w for w in _arm_warnings(cells, "raw", "skill", None)) + assert not any("--label-host" in w for w in _arm_warnings([cell("raw", None), cell("skill", None)], "raw", "skill", None)) diff --git a/dokimi/tests/test_repo_report_hosts.py b/dokimi/tests/test_repo_report_hosts.py new file mode 100644 index 000000000..c30a7b5d8 --- /dev/null +++ b/dokimi/tests/test_repo_report_hosts.py @@ -0,0 +1,108 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`repo report` rows split by host fingerprint: what a row shows and which missing cells it counts.""" + +from __future__ import annotations + +import json + +from dokimi.repos.host import fingerprint_id +from dokimi.repos.report import format_report, summarize + +HOST_A = {"cpus": 4, "cpu_vendor": "GenuineIntel", "collectors": {"os": {"system": "Linux"}}} +HOST_B = {"cpus": 8, "cpu_vendor": "AuthenticAMD", "collectors": {"os": {"system": "Linux"}}} +FP_A, FP_B = fingerprint_id(HOST_A), fingerprint_id(HOST_B) + + +def _cell(batch, rep, *, host=None, fingerprint=None, result=True, **extra): + rd = batch / f"t__guidance-raw__m-1__r{rep}" + rd.mkdir(parents=True) + (rd / "meta.json").write_text(json.dumps({"host": host})) + if result: + rec = {"task": "t", "arm": "guidance:raw", "model": "m-1", "score_task": False, "score_max": None, + "grade": {"status": "graded", "success": True, "score_only": False, "score": None}, + "agent": {"timed_out": False, "is_error": False, "over_budget": False, "cost_usd": 0.2, + "num_turns": 3}, "host_fingerprint": fingerprint, **extra} + (rd / "result.json").write_text(json.dumps(rec)) + + +def _plan(batch, reps): + (batch / "_eval").mkdir(parents=True, exist_ok=True) + cells = [f"t__guidance-raw__m-1__r{r}" for r in reps] + (batch / "_eval" / "plan.json").write_text(json.dumps({"cells": cells})) + + +def test_a_started_missing_cell_counts_on_the_row_of_the_host_that_started_it(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _cell(batch, 3, host=HOST_B, result=False) # started on B, never finished + rows = {r["host_fingerprint"]: r for r in summarize(batch)["rows"]} + assert {fp: r["missing"] for fp, r in rows.items()} == {FP_A: 0, FP_B: 1} + + +def test_an_unstarted_missing_cell_is_on_no_row_when_two_hosts_could_own_it(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _plan(batch, [1, 2, 3]) + summary = summarize(batch) + assert [r["missing"] for r in summary["rows"]] == [0, 0] + assert summary["missing"] == ["t__guidance-raw__m-1__r3"] + + +def test_an_unstarted_missing_cell_stays_on_the_only_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _plan(batch, [1, 2]) + assert [r["missing"] for r in summarize(batch)["rows"]] == [1] + + +def test_a_started_missing_cell_whose_host_has_no_row_counts_on_the_only_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, result=False) # started on B, crashed before result.json; B has no finished cell + summary = summarize(batch) + assert [(r["host_fingerprint"], r["missing"]) for r in summary["rows"]] == [(FP_A, 1)] + assert summary["missing"] == ["t__guidance-raw__m-1__r2"] + + +def test_text_report_names_the_host_of_every_fingerprinted_row_without_label_mode(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B) + _cell(batch, 3) # written before fingerprints: no host to name + lines = format_report(summarize(batch)).splitlines() + assert [ln.strip() for ln in lines if ln.strip().startswith("host ")] == sorted( + [f"host {FP_A}", f"host {FP_B}"]) + + +def test_a_labelled_row_keeps_its_verdict_and_an_undecided_one_is_not_unmet(tmp_path): + batch = tmp_path / "b1" + miss = {"met": False, "fields": [{"field": "cpu_flags.avx512f", "met": False}]} + unknown = {"met": None, "fields": [{"field": "min_cpus", "met": None}]} + _cell(batch, 1, host=HOST_A, fingerprint=FP_A, host_match=miss) + _cell(batch, 2, host=HOST_B, fingerprint=FP_B, host_match=unknown) + text = format_report(summarize(batch)) + assert f"host {FP_A} (labelled, unmet: cpu_flags.avx512f)" in text + assert f"host {FP_B} (labelled, undecided)" in text + + +def test_a_missing_cell_whose_meta_host_is_malformed_is_placed_on_no_host_row(tmp_path): + batch = tmp_path / "b1" + _cell(batch, 1, host=HOST_A, fingerprint=FP_A) + _cell(batch, 2, host={"collectors": [1]}, result=False) + _cell(batch, 3, host=[1], result=False) + summary = summarize(batch) + assert [r["missing"] for r in summary["rows"]] == [2] # unplaced, and the only row takes them + assert len(summary["missing"]) == 2 + + +def test_a_missing_cell_with_container_only_host_facts_counts_on_the_unfingerprinted_row(tmp_path): + batch = tmp_path / "b1" + legacy = {"cpus": 4, "cpu_vendor": "GenuineIntel"} # facts from before the collectors + _cell(batch, 1, host=legacy) # its result.json has no host_fingerprint + _cell(batch, 2, host=legacy, result=False) + summary = summarize(batch) + assert [(r["host_fingerprint"], r["missing"]) for r in summary["rows"]] == [(None, 1)] diff --git a/dokimi/tests/test_repo_report_unreadable_cell.py b/dokimi/tests/test_repo_report_unreadable_cell.py index 46a51cd5d..79317d0a6 100644 --- a/dokimi/tests/test_repo_report_unreadable_cell.py +++ b/dokimi/tests/test_repo_report_unreadable_cell.py @@ -127,6 +127,15 @@ def test_a_result_file_without_a_usable_grade_exits_2(tmp_path, capsys, argv, co pytest.param("agent", {**_GOOD["agent"], "num_turns": "3"}, id="num-turns-a-string"), pytest.param("agent", {**_GOOD["agent"], "input_tokens": True}, id="input-tokens-a-bool"), pytest.param("skills", [], id="skills-not-a-mapping"), + pytest.param("host_fingerprint", {}, id="host-fingerprint-an-object"), + pytest.param("host_fingerprint", [], id="host-fingerprint-a-list"), + pytest.param("host_fingerprint", 5, id="host-fingerprint-a-number"), + pytest.param("host_match", [], id="host-match-a-list"), + pytest.param("host_match", {"met": False}, id="host-match-no-fields"), + pytest.param("host_match", {"met": "no", "fields": []}, id="host-match-met-a-string"), + pytest.param("host_match", {"met": False, "fields": [{"field": 42, "met": False}]}, id="host-match-field-a-number"), + pytest.param("host_match", {"met": False, "fields": [{"field": "min_cpus", "met": "no"}]}, + id="host-match-field-met-a-string"), pytest.param("skills", {"measured": True, "expected_skill": "s"}, id="skills-no-skill-invoked"), pytest.param("skills", {"measured": True}, id="skills-measured-no-skill-invoked"), pytest.param("skills", {"measured": True, "expected_skill": "s", "skill_invoked": True},