From c80c2fdb129e9be2f23be6984334c0cb1d1180e1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:32:31 -0800 Subject: [PATCH 01/19] fix(cli): exit codes for repo run crashes, cross-judge verdicts, unread traces (#94 A2b, #275) - repo run: a cell that crashes (harness, docker daemon or CLI, vanished scratch) exits 5, not 1. DockerError marks docker failures that are not the manifest (CLI or daemon unreachable, run exits 125); a failed image build stays config (exit 1). - eval judge agreement --strict: NOT MEASURED or nothing to compare exits 3; JUDGE-DEPENDENT exits 4 (invalid is checked first). Without --strict the exit is unchanged. - trace inspect and trace consistency: no trace readable is an input refusal (exit 2, InputError); a partial read still exits 1. - Docs exit tables, changelog fragment; tests pin the new codes. Signed-off-by: Nikolay Petrov --- ...1115_120000_napetrov_fix_cli-a2b5-exits.md | 11 ++ dokimi/docs/context-window.md | 3 +- dokimi/docs/quickstart.md | 8 +- dokimi/dokimi/commands/context.py | 12 +- dokimi/dokimi/commands/judge_reliability.py | 15 ++- dokimi/dokimi/commands/repo_run.py | 14 ++- dokimi/dokimi/repos/container.py | 38 +++++-- dokimi/dokimi/repos/runner.py | 15 ++- dokimi/tests/test_cli_output.py | 9 +- dokimi/tests/test_context_trace.py | 23 ++-- dokimi/tests/test_exit_codes_infra.py | 105 ++++++++++++++++++ dokimi/tests/test_judge_not_measured.py | 20 ++-- 12 files changed, 220 insertions(+), 53 deletions(-) create mode 100644 dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md diff --git a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md new file mode 100644 index 000000000..1f0460deb --- /dev/null +++ b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md @@ -0,0 +1,11 @@ + + +### Changed + +- **Exit codes for the cases below** — scripts that branch on the exit code see these changes: + - `dokimi repo run`: a cell that crashes (a harness failure, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. + - `dokimi eval judge agreement --strict` exits 3 (inconclusive) when an arm is NOT MEASURED or no arm can be compared, and 4 (invalid) when an arm is JUDGE-DEPENDENT; it exited 1 for both before. Without `--strict` the exit code is unchanged (0), and a judge that scored none of its answers still exits 1. + - `dokimi trace inspect`, `dokimi trace consistency` exit 2 (bad input) when no trace can be read; they exited 1 before. With `--json` they print the `input` error document (`error.code` `input`, `data` null) instead of a document with `status` `failed`. A partial read still exits 1. diff --git a/dokimi/docs/context-window.md b/dokimi/docs/context-window.md index 33a10469e..34ee4ba47 100644 --- a/dokimi/docs/context-window.md +++ b/dokimi/docs/context-window.md @@ -57,7 +57,8 @@ run.jsonl: prompt tokens per call: 21390 21673 `--json` prints one `cli_output.v1` document (`kind` `trace inspect`) whose `data` is a list with one object per trace, also for a single trace. A trace that cannot be read is reported on stderr and left -out; the others still print, the document has `status` `partial`, and the command exits 1. +out; the others still print, the document has `status` `partial`, and the command exits 1. When no trace +reads, the command refuses the input: exit 2, and with `--json` an `input` error document. A trace that reads with skipped lines (each named on stderr) is also `partial`, and the exit code stays 0. ## Comparing context layers diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index b49cf8677..9c593b566 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -252,9 +252,9 @@ failed check. | 0 | The command ran and, if it checks something, the check passed. | | | 1 | A verdict or a gate that ran and failed, or a runtime failure the command reports (a model call, a run, a clone or fetch); a crash that escapes is 5. `eval judge validate --strict` and `eval leak`/`eval redteam` that fail as blocked. | `baseline compare --max-context-drop 0` over a regression, `repo reach` with a harness binary that cannot run, `eval triggers generate --strict` over a lint finding | | 2 | A bad invocation: a missing, unreadable or malformed input file, a device given as one (` : not a file`), a flag combination the command refuses, an output file that cannot be written, a missing provider credential, a source string that is malformed, unpinned or not a repository root. | `baseline save` on a bad eval file, `evaluate --custom-questions` on an empty or non-JSON file (`--custom-questions : `), `benchmark preflight` with no golden set, `repo scan --manifest ../x.yaml` | -| 3 | Inconclusive: the run stopped short because a budget ran out (`--max-cost-usd`, or a call with no price) and kept the partial output (`tools preference` today; other commands still exit 1 on a budget stop), or the result is undecided (`eval judge validate`: a pass whose interval straddles the gate, or a variant no pair was built for, with `--strict` making it 1; `eval leak`: an item that could not be checked, or none could; `eval redteam`: a graded question not measured, or nothing graded; `repo security --fail-on`: an unfinished cell or an arm with no measured cell, when no arm failed and every model has a baseline cell). | `tools preference --max-cost-usd 5` with calls not run; `eval judge validate` with too few bases | -| 4 | Invalid: the result is not comparable or not trustworthy: a model that differs from the baseline's, or one that cannot be verified, under `baseline compare --strict` or the regression gate (`--max-context-drop` or `--max-lift-drop`); under `repo security --fail-on`, a model with no baseline cell when no arm failed. | `baseline compare --strict` over a baseline scored with another judge | -| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. | `backends --json` when `detect` raises | +| 3 | Inconclusive: the run stopped short because a budget ran out (`--max-cost-usd`, or a call with no price) and kept the partial output (`tools preference` today; other commands still exit 1 on a budget stop), or the result is undecided (`eval judge validate`: a pass whose interval straddles the gate, or a variant no pair was built for, with `--strict` making it 1; `eval leak`: an item that could not be checked, or none could; `eval redteam`: a graded question not measured, or nothing graded; `eval judge agreement --strict`: an arm NOT MEASURED, or no arm to compare; `repo security --fail-on`: an unfinished cell or an arm with no measured cell, when no arm failed and every model has a baseline cell). | `tools preference --max-cost-usd 5` with calls not run; `eval judge validate` with too few bases | +| 4 | Invalid: the result is not comparable or not trustworthy: a model that differs from the baseline's, or one that cannot be verified, under `baseline compare --strict` or the regression gate (`--max-context-drop` or `--max-lift-drop`); under `repo security --fail-on`, a model with no baseline cell when no arm failed; under `eval judge agreement --strict`, an arm JUDGE-DEPENDENT. | `baseline compare --strict` over a baseline scored with another judge | +| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and @@ -268,6 +268,8 @@ when the budget or an unpriced call stops them. `eval judge agreement --judges` and `eval score pairwise` exit 1, not 2, when every judge call fails, a missing provider credential included. +`trace inspect`, `trace consistency` and `trace prefilter` exit 2 when no trace or session reads. + ## Troubleshooting | Symptom | Fix | diff --git a/dokimi/dokimi/commands/context.py b/dokimi/dokimi/commands/context.py index 15bc069c0..665751612 100644 --- a/dokimi/dokimi/commands/context.py +++ b/dokimi/dokimi/commands/context.py @@ -38,8 +38,10 @@ def cmd_context_trace(args: argparse.Namespace) -> None: row["compaction_events"] = trace.compactions rows.append(row) + if not rows: # nothing read is a bad input (exit 2), not a partial result + raise InputError("no trace could be read (see stderr)") if args.json: # always a list, one entry per trace that read - status = "failed" if not rows else "partial" if failed or skipped else "complete" # nothing read: failed + status = "partial" if failed or skipped else "complete" reason = ("one or more traces could not be read (see stderr)" if failed else "lines of a trace were skipped (see stderr)" if skipped else None) cli_output.emit_leaf("trace inspect", rows, status=status, reason=reason) @@ -107,7 +109,9 @@ def _refuse(message: str) -> NoReturn: def _read_arms(arms: dict[str, list[str]], shell: str) -> tuple[dict[str, list], bool]: - """The runs of each arm, and whether some trace could not be read (reported on stderr).""" + """The runs of each arm, and whether some trace could not be read (reported on stderr). + + Nothing read at all is a bad input (exit 2), not a partial result.""" from dokimi.context import TraceError from dokimi.context.consistency import read_run @@ -121,6 +125,8 @@ def _read_arms(arms: dict[str, list[str]], shell: str) -> tuple[dict[str, list], except TraceError as exc: print(f"✗ {exc}", file=sys.stderr) failed = True + if not any(loaded.values()): + raise InputError("no trace could be read (see stderr)") return loaded, failed @@ -156,7 +162,7 @@ def cmd_context_consistency(args: argparse.Namespace) -> None: report[label]["shell"] = args.shell # the grain the step counts were taken at if args.json: - status = "failed" if not any(loaded.values()) else "partial" if failed else "complete" + status = "partial" if failed else "complete" cli_output.emit_leaf("trace consistency", report, status=status, reason="one or more traces could not be read (see stderr)" if failed else None) else: diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 7e2f22cda..77ca8b63a 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -20,7 +20,7 @@ from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY from dokimi.utils import InputError, read_json_input, warn_recorded_self_judge from dokimi.commands._output import save_artifact_or_exit, write_text_or_exit -from dokimi.exit_codes import EXIT_FAILURE +from dokimi.exit_codes import EXIT_FAILURE, EXIT_INCONCLUSIVE, EXIT_INVALID if TYPE_CHECKING: from dokimi.eval.cross_judge import Measure @@ -445,15 +445,18 @@ def _scored_nothing_exit(unscored: Sequence[Measure]) -> None: def _strict_cross_judge_exit(arms: dict[str, dict[str, Any]], baseline_arm: str) -> None: - """Exit 1 when no arm was compared, an arm is NOT MEASURED, or an arm is JUDGE-DEPENDENT.""" + """Exit 4 (invalid) when an arm is JUDGE-DEPENDENT, else 3 (inconclusive) when no arm was compared or an arm is + NOT MEASURED (issue #94 §2.6). The invalid verdict is checked first, as ``baseline compare`` does.""" + dependent = sorted(a for a, r in arms.items() if r["label"] == "JUDGE-DEPENDENT") + if dependent: + print(f"✗ judge-dependent against `{baseline_arm}`: {', '.join(dependent)}", file=sys.stderr) + sys.exit(EXIT_INVALID) unmeasured = sorted(a for a, r in arms.items() if r["label"] == "NOT MEASURED") if not arms or unmeasured: why = f"{len(unmeasured)} of {len(arms)} arm(s) NOT MEASURED: {', '.join(unmeasured)}" if arms else \ "no arm has a scored question to compare" print(f"✗ incomplete against `{baseline_arm}`: {why}", file=sys.stderr) - sys.exit(1) - if any(a["label"] == "JUDGE-DEPENDENT" for a in arms.values()): - sys.exit(1) + sys.exit(EXIT_INCONCLUSIVE) def register(score_sub, judge_sub, positive_int) -> None: @@ -530,7 +533,7 @@ def register(score_sub, judge_sub, positive_int) -> None: help="Judge sampling temperature, recorded as judge_temperature in the output") cj.add_argument("--concurrency", type=positive_int, default=DEFAULT_CONCURRENCY) cj.add_argument("--strict", action="store_true", - help="Exit 1 when any arm is JUDGE-DEPENDENT or NOT MEASURED, or no arm was compared") + help="Exit 4 when an arm is JUDGE-DEPENDENT; exit 3 when an arm is NOT MEASURED or no arm was compared") cj.add_argument("--output", default=None, help="Output JSON (default: _crossjudge.json); Markdown written next to it") cj.set_defaults(func=cmd_eval_cross_judge) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 7c3a04035..3118b9c2a 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -22,6 +22,7 @@ from dokimi.atomic_write import write_text_atomic from dokimi import cli_flags from dokimi import docker_cmd +from dokimi.exit_codes import EXIT_INFRA from dokimi.repos.harness_profile import DELEGATION_MODES from dokimi.repos.source import SourceArgumentError, tree_path from dokimi.utils import tree_digest @@ -318,7 +319,7 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock from dokimi.repos.arms import check_arms from dokimi.repos.batch_ledger import pin_arms, pin_skills, write_skill_pin from dokimi.repos.bases import check_mcp_configs - from dokimi.repos.container import docker_network + from dokimi.repos.container import DockerError, docker_network from dokimi.repos.harness_profile import load_profile from dokimi.repos.runner import ( AgentOptions, agent_network, batch_lock, harness_version, run_batch, task_caps, validate_tasks) @@ -363,6 +364,9 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock write_skill_pin(batch_dir, skill_pin) summary = run_batch(ctx, tasks, arms, options, args.reps, batch_dir, jobs=args.jobs, force=args.force, max_batch_usd=args.max_batch_usd, log=_progress) + except DockerError as exc: # the docker CLI or daemon failed: infrastructure, not a verdict (§2.6) + print(f"✗ {exc}", file=sys.stderr) + sys.exit(EXIT_INFRA) except _errors() as exc: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) @@ -372,6 +376,14 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock if summary["probe_failed"]: print(f"✗ the host probe failed, so these tasks have cells that were not run: {', '.join(summary['probe_failed'])}", file=sys.stderr) + _exit_of_batch(summary) + + +def _exit_of_batch(summary: dict) -> None: + """A cell that crashed of the machinery is infrastructure (exit 5, §2.6), never a verdict; a refused cell, a budget + stop, a failed probe or a batch with no result is exit 1.""" + if any(err.get("infra") for err in summary["errors"]): + sys.exit(EXIT_INFRA) if summary["errors"] or summary["budget_stopped"] or summary["probe_failed"]: sys.exit(1) if not summary["ran"] and not summary["skipped"]: # every cell unsupported: the batch holds no result diff --git a/dokimi/dokimi/repos/container.py b/dokimi/dokimi/repos/container.py index 97134402d..74a8c37bd 100644 --- a/dokimi/dokimi/repos/container.py +++ b/dokimi/dokimi/repos/container.py @@ -63,6 +63,11 @@ class ContainerError(RuntimeError): """The container runtime failed, as opposed to the command inside it.""" +class DockerError(ContainerError): + """The docker CLI or daemon could not run, answer or start a container (infra, exit 5). A build of the + manifest's image that fails is a :class:`ContainerError`, not this (config, exit 1).""" + + class EgressNotImplemented(ContainerError): """A run needs per-host egress (contract §2), which needs the proxy the framework does not have yet.""" @@ -143,8 +148,8 @@ def argv(self, spec: ContainerSpec) -> list[str]: def run(self, spec: ContainerSpec, *, stdout: Path, stderr: Path, timeout_s: float) -> ContainerResult: for m in spec.mounts: - if not m.host.exists(): - raise ContainerError(f"mount source does not exist: {m.host}") + if not m.host.exists(): # the container cannot start without it (a scratch dir that vanished): infra + raise DockerError(f"mount source does not exist: {m.host}") def stop() -> None: if spec.name: @@ -159,7 +164,7 @@ def stop() -> None: timeout_s=timeout_s, on_timeout=stop, runner=self.runner) if docker_cmd.failed_to_start(result.returncode, result.timed_out): tail = stderr.read_text(encoding="utf-8", errors="replace").strip()[-500:] - raise ContainerError(f"docker run failed: {tail}") + raise DockerError(f"docker run failed: {tail}") return result def build(self, context: Path, dockerfile: Path, tag: str, network: str | None = None, @@ -168,19 +173,34 @@ def build(self, context: Path, dockerfile: Path, tag: str, network: str | None = try: proc = self.runner(argv, timeout=timeout_s, env=process.Env.inherit()) except OSError as exc: - raise ContainerError(f"docker build {tag}: cannot run {self.docker!r}: {exc}") from exc + raise DockerError(f"docker build {tag}: cannot run {self.docker!r}: {exc}") from exc if proc.timed_out: raise ContainerError(f"docker build {tag} timed out after {timeout_s:g}s (--build-timeout):\n" f"{(proc.stderr or proc.stdout)[-3000:]}") if proc.returncode: - raise ContainerError(f"docker build {tag} failed:\n{(proc.stderr or proc.stdout)[-3000:]}") + kind = DockerError if _daemon_unreachable(proc.stderr) else ContainerError + raise kind(f"docker build {tag} failed:\n{(proc.stderr or proc.stdout)[-3000:]}") def image_exists(self, tag: str) -> bool: - proc = self.runner([self.docker, *docker_cmd.inspect_image_args(tag)], timeout=docker_cmd.CALL_TIMEOUT_S, - env=process.Env.inherit(), capture_stdout=False) + try: + proc = self.runner([self.docker, *docker_cmd.inspect_image_args(tag)], timeout=docker_cmd.CALL_TIMEOUT_S, + env=process.Env.inherit(), capture_stdout=False) + except OSError as exc: + raise DockerError(f"docker image inspect {tag}: cannot run {self.docker!r}: {exc}") from exc if proc.timed_out: # a daemon that does not answer would only hang the build next - raise ContainerError(f"docker image inspect {tag}: no answer in {docker_cmd.CALL_TIMEOUT_S}s") - return proc.returncode == 0 + raise DockerError(f"docker image inspect {tag}: no answer in {docker_cmd.CALL_TIMEOUT_S}s") + if proc.returncode == 0: + return True + if _daemon_unreachable(proc.stderr): # not "no such image": the daemon itself did not answer + raise DockerError(f"docker image inspect {tag}: {(proc.stderr or '').strip()[-500:]}") + return False + + +def _daemon_unreachable(stderr: str | None) -> bool: + """True when the docker CLI says, in its last lines, that it could not reach the daemon (down, or no permission on + its socket). The last lines only: a RUN step of a build may print the same words in the middle of its output.""" + lines = [ln for ln in (stderr or "").lower().splitlines() if ln.strip()][-2:] + return any("connect to the docker daemon" in ln or "error during connect" in ln for ln in lines) def run_process(argv: list[str], *, env: process.Env | dict[str, str], stdout: Path, stderr: Path, diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a90455bfc..55c5d4d01 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -54,6 +54,7 @@ WORKSPACE, ContainerError, ContainerSpec, + DockerError, EgressNotImplemented, Mount, docker_network, @@ -326,6 +327,12 @@ def _process_or_null(ctx: EvalContext, events: list[dict[str, Any]], errors: lis return None +def _is_infra(exc: BaseException) -> bool: + """A failure of the machinery (docker, a file that cannot be written or read, a subprocess), not of a cell's + input: a refused input (PrepError, an unreadable kept result, an egress the run cannot have) stays exit 1.""" + return isinstance(exc, (DockerError, OSError, subprocess.SubprocessError)) + + class CellCrash(Exception): """A cell whose agent ran but whose ``result.json`` could not be written: its cost still counts.""" @@ -401,7 +408,7 @@ def one(cell: tuple[Task, str, AgentOptions, int]) -> None: counted = _unsupported_cell(batch_dir / name, cell, unsupported[task.id], verdicts[task.id][1], host) except (UnreadableResult, OSError) as exc: with lock: - state["errors"].append({"cell": name, "error": str(exc)}) + state["errors"].append({"cell": name, "error": str(exc), "infra": _is_infra(exc)}) log(f"✗ {name}: {exc}") return with lock: @@ -418,11 +425,11 @@ def one(cell: tuple[Task, str, AgentOptions, int]) -> None: except CellCrash as exc: charged = exc.charged_usd with lock: - state["errors"].append({"cell": name, "error": str(exc)}) + state["errors"].append({"cell": name, "error": str(exc), "infra": True}) log(f"✗ {name}: {exc}") except (PrepError, ContainerError, OSError, subprocess.SubprocessError) as exc: with lock: - state["errors"].append({"cell": name, "error": str(exc)}) + state["errors"].append({"cell": name, "error": str(exc), "infra": _is_infra(exc)}) log(f"✗ {name}: {exc}") finally: # the charge is counted and saved even when the cell's result.json was not written (CellCrash) _settle(state, lock, batch_dir, reserved, charged, rec) @@ -472,7 +479,7 @@ def _settle(state: dict[str, Any], lock: threading.Lock, batch_dir: Path, reserv try: _write_batch_spend(batch_dir, state["spent"]) except OSError as exc: - state["errors"].append({"cell": SPEND_FILE, "error": str(exc)}) + state["errors"].append({"cell": SPEND_FILE, "error": str(exc), "infra": True}) def _read_batch_spend(batch_dir: Path, capped: bool = True) -> float: diff --git a/dokimi/tests/test_cli_output.py b/dokimi/tests/test_cli_output.py index abe3caadd..e40c00955 100644 --- a/dokimi/tests/test_cli_output.py +++ b/dokimi/tests/test_cli_output.py @@ -293,14 +293,11 @@ def test_trace_inspect_json_is_partial_when_a_trace_does_not_read(tmp_path, caps assert schema_errors("cli_output.v1", doc) == [] -def test_trace_consistency_json_is_failed_when_no_trace_reads(tmp_path, capsys): +def test_trace_consistency_is_an_input_error_when_no_trace_reads(tmp_path, capsys): args = cli.parse_args(["trace", "consistency", "--arm", f"base={tmp_path}/missing.jsonl", "--json"]) - with pytest.raises(SystemExit) as exc: + with pytest.raises(InputError, match="no trace could be read"): args.func(args) - assert exc.value.code == 1 - doc = json.loads(capsys.readouterr().out) - assert doc["kind"] == "trace consistency" and doc["status"] == "failed" and doc["reason"] - assert schema_errors("cli_output.v1", doc) == [] + assert capsys.readouterr().out == "" # the input error document is cli.main's, not the leaf's def test_trace_consistency_json_is_partial_when_some_trace_reads(tmp_path, capsys): diff --git a/dokimi/tests/test_context_trace.py b/dokimi/tests/test_context_trace.py index b989c0044..bb31c1285 100644 --- a/dokimi/tests/test_context_trace.py +++ b/dokimi/tests/test_context_trace.py @@ -11,6 +11,7 @@ from dokimi.cli import build_parser from dokimi.context import TraceError, read_trace +from dokimi.utils import InputError from dokimi.context.trace import parse_trace @@ -162,21 +163,23 @@ def test_cli_json_single_trace(tmp_path, capsys): @pytest.mark.outcome("cjm-context-trace") -def test_cli_missing_trace_exits_1(tmp_path, capsys): - with pytest.raises(SystemExit) as exc: +def test_cli_missing_trace_is_an_input_error(tmp_path, capsys): + """Nothing read is a bad input (exit 2), not a partial result (exit 1).""" + with pytest.raises(InputError, match="no trace could be read"): _run_cli(["trace", "inspect", str(tmp_path / "missing.jsonl")], capsys) - assert exc.value.code == 1 @pytest.mark.outcome("cjm-context-trace") -def test_cli_json_with_no_trace_read_is_failed_with_no_data(tmp_path, capsys): - """Nothing read is `failed`, not `partial`: a consumer that reads only `status` must see total failure.""" +def test_cli_json_with_no_trace_read_is_an_input_error_document(tmp_path, monkeypatch, capsys): + """Nothing read exits 2 with an `input` error document; a consumer that reads `error.code` sees a refusal.""" + from dokimi import cli + + monkeypatch.setattr("sys.argv", ["dokimi", "trace", "inspect", str(tmp_path / "missing.jsonl"), "--json"]) with pytest.raises(SystemExit) as exc: - _run_cli(["trace", "inspect", str(tmp_path / "missing.jsonl"), "--json"], capsys) - got = capsys.readouterr() - doc = json.loads(got.out) - assert exc.value.code == 1 - assert doc["kind"] == "trace inspect" and doc["status"] == "failed" and doc["data"] == [] + cli.main() + doc = json.loads(capsys.readouterr().out) + assert exc.value.code == 2 + assert doc["kind"] == "trace inspect" and doc["data"] is None and doc["error"]["code"] == "input" def test_a_bad_invocation_exits_2_with_an_input_error_document(monkeypatch, capsys): diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 3125d863f..d5ffbb750 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -23,3 +23,108 @@ def crash(_args): assert exc.value.code == 5 # not 1: a crash never reads as a failed verdict err = capsys.readouterr().err assert "Traceback" in err and "RuntimeError: harness crashed" in err + + +def test_a_docker_cli_that_cannot_start_is_a_docker_error_and_a_failed_build_is_not(tmp_path): + """Docker failures are infrastructure (exit 5 in `repo run`); a build of the manifest's image that fails is config.""" + from types import SimpleNamespace + + from dokimi.repos.container import ContainerError, DockerError, DockerRunner + + def missing_cli(argv, **_): + raise OSError("No such file or directory") + + def failed_build(argv, **_): + return SimpleNamespace(timed_out=False, returncode=1, stderr="RUN false", stdout="") + + (tmp_path / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + with pytest.raises(DockerError): + DockerRunner(docker="docker-missing", runner=missing_cli).build(tmp_path, tmp_path / "Dockerfile", "t") + with pytest.raises(ContainerError) as exc: + DockerRunner(runner=failed_build).build(tmp_path, tmp_path / "Dockerfile", "t") + assert not isinstance(exc.value, DockerError) + + +DAEMON_DOWN = ("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\n") + + +def test_a_daemon_that_is_down_is_a_docker_error_before_any_build(): + """`repo run` asks for the image first: with the daemon down that answer is infra (exit 5), not a missing image.""" + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + fake = process.FakeRunner(lambda call: process.reply(stderr=DAEMON_DOWN, returncode=1)) + with pytest.raises(DockerError, match="Cannot connect to the Docker daemon"): + DockerRunner(runner=fake).image_exists("img:t") + assert [c.argv[1] for c in fake.calls] == ["image"] # the build is never started + + +def test_a_docker_cli_missing_on_the_image_inspect_is_a_docker_error(): + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + def missing_cli(call): + raise OSError("No such file or directory") + + with pytest.raises(DockerError, match="cannot run"): + DockerRunner(docker="docker-missing", runner=process.FakeRunner(missing_cli)).image_exists("img:t") + + +def test_no_such_image_is_still_a_plain_miss_not_an_infra_error(): + from dokimi import process + from dokimi.repos.container import DockerRunner + + no_image = process.reply(stderr="Error: No such image: img:t\n", returncode=1) + assert DockerRunner(runner=process.FakeRunner(lambda call: no_image)).image_exists("img:t") is False + + +def test_a_build_with_the_daemon_down_is_a_docker_error(tmp_path): + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + (tmp_path / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + fake = process.FakeRunner(lambda call: process.reply(stderr=DAEMON_DOWN, returncode=1)) + with pytest.raises(DockerError): + DockerRunner(runner=fake).build(tmp_path, tmp_path / "Dockerfile", "t") + + +def test_a_refused_cell_is_exit_1_and_a_machinery_failure_is_exit_5(): + """A cell refused on its inputs (a malformed MCP config) is an ordinary error; a docker or file failure is infra.""" + from dokimi.commands import repo_run + from dokimi.repos.errors import PrepError + from dokimi.repos.container import DockerError + from dokimi.repos.runner import _is_infra + + assert _is_infra(DockerError("docker run failed")) and _is_infra(OSError("scratch vanished")) + assert not _is_infra(PrepError("malformed .mcp.json")) + + def exit_of(err: dict) -> int: + summary = {"errors": [err], "budget_stopped": 0, "probe_failed": [], "ran": 0, "skipped": 0, + "unsupported": 0, "cells": 1} + with pytest.raises(SystemExit) as exc: + repo_run._exit_of_batch(summary) + return exc.value.code + + assert exit_of({"cell": "c", "error": "x", "infra": True}) == 5 + assert exit_of({"cell": "c", "error": "x", "infra": False}) == 1 + + +def test_a_mount_source_that_vanished_is_a_docker_error(tmp_path): + """The scratch directory vanished before docker could start the container: infrastructure, exit 5 in repo run.""" + from dokimi import process + from dokimi.repos.container import ContainerSpec, DockerError, DockerRunner, Mount + + spec = ContainerSpec(image="img:t", argv=["true"], mounts=[Mount(tmp_path / "gone", "/eval/run")]) + with pytest.raises(DockerError, match="mount source does not exist"): + DockerRunner(runner=process.FakeRunner()).run(spec, stdout=tmp_path / "o", stderr=tmp_path / "e", timeout_s=5) + + +def test_strict_judge_dependent_is_checked_before_not_measured(capsys): + """An arm that is both JUDGE-DEPENDENT and NOT MEASURED is invalid (4), not inconclusive (3): invalid is checked first.""" + from dokimi.commands import judge_reliability + + arms = {"a": {"label": "NOT MEASURED"}, "b": {"label": "JUDGE-DEPENDENT"}} + with pytest.raises(SystemExit) as exc: + judge_reliability._strict_cross_judge_exit(arms, "base") + assert exc.value.code == 4 + assert "judge-dependent against `base`: b" in capsys.readouterr().err diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py index 7c3577252..cb29bf8c6 100644 --- a/dokimi/tests/test_judge_not_measured.py +++ b/dokimi/tests/test_judge_not_measured.py @@ -28,7 +28,7 @@ def _strict_exit(argv): return 0 -def test_cross_judge_strict_fails_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): +def test_cross_judge_strict_is_inconclusive_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): import json from dokimi.eval import judge as judge_mod @@ -53,7 +53,7 @@ def score_answer(self, **_): for argv, arms, exit_code, why in ( (["--answers", str(errored), "--no-stored", "--judges", "amazon-bedrock/m"], {"skill": "NOT MEASURED"}, 1, "Error: amazon-bedrock/m (aggregate) scored none of its 16 answers"), - (["--answers", str(empty)], {}, 1, "✗ incomplete"), + (["--answers", str(empty)], {}, 3, "✗ incomplete"), # inconclusive (§2.6) ): code = _strict_exit(["eval", "judge", "agreement", *argv, "--output", str(tmp_path / "cj.json"), "--strict"]) assert code == exit_code @@ -62,7 +62,7 @@ def score_answer(self, **_): assert {a: r["label"] for a, r in summary["arms"].items()} == arms -def test_cross_judge_strict_exits_1_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): +def test_cross_judge_strict_exits_3_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): import json from dokimi.eval import judge as judge_mod @@ -88,7 +88,7 @@ def score_answer(self, *, answer, **_): summary = json.loads((tmp_path / "cj.json").read_text())["summary"] assert summary["arms"]["skill"]["label"] != "NOT MEASURED" assert summary["arms"]["skill2"]["label"] == "NOT MEASURED" - assert code == 1 + assert code == 3 assert "1 of 2 arm(s) NOT MEASURED: skill2" in capsys.readouterr().err @@ -161,19 +161,19 @@ def test_no_voter_left_by_the_validity_gate_is_not_measured(): assert cj.label([dict(significant)], require_validated=True) == "NOT MEASURED" -def test_cross_judge_strict_exits_1_when_require_validated_leaves_no_voter(tmp_path, capsys): +def test_cross_judge_strict_exits_3_when_require_validated_leaves_no_voter(tmp_path, capsys): import json path = tmp_path / "run.json" path.write_text(json.dumps(_stamped_run()), encoding="utf-8") code = _strict_exit(["eval", "judge", "agreement", "--answers", str(path), "--require-validated", "--output", str(tmp_path / "cj.json"), "--strict"]) - assert code == 1 + assert code == 3 assert "1 of 1 arm(s) NOT MEASURED: skill" in capsys.readouterr().err assert json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"]["skill"]["label"] == "NOT MEASURED" -def test_cross_judge_judge_dependent_exits_1_only_with_strict(tmp_path, monkeypatch, capsys): +def test_cross_judge_judge_dependent_exits_4_only_with_strict(tmp_path, monkeypatch, capsys): import json from dokimi.eval import judge as judge_mod @@ -193,10 +193,10 @@ def score_answer(self, *, answer, **_): # prefers the baseline, the stored judg assert _strict_exit(argv) == 0 capsys.readouterr() assert json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"]["skill"]["label"] == "JUDGE-DEPENDENT" - assert _strict_exit([*argv, "--strict"]) == 1 + assert _strict_exit([*argv, "--strict"]) == 4 -def test_cross_judge_strict_exits_1_when_one_of_several_arms_is_judge_dependent(tmp_path, monkeypatch, capsys): +def test_cross_judge_strict_exits_4_when_one_of_several_arms_is_judge_dependent(tmp_path, monkeypatch, capsys): import json from dokimi.eval import judge as judge_mod @@ -222,7 +222,7 @@ def score_answer(self, *, answer, **_): # the baseline beats skill, and skill2 capsys.readouterr() arms = json.loads((tmp_path / "cj.json").read_text())["summary"]["arms"] assert arms["skill"]["label"] == "JUDGE-DEPENDENT" and arms["skill2"]["label"] == "CONSISTENT +" - assert _strict_exit([*argv, "--strict"]) == 1 + assert _strict_exit([*argv, "--strict"]) == 4 def test_cross_judge_strict_exits_0_when_every_arm_is_measured_and_none_is_judge_dependent( From 500310dea78b74ec07badbfbfc8f238e8928dab3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:23:22 -0800 Subject: [PATCH 02/19] Address review: match only the docker CLI's own daemon diagnostic A RUN step whose command text names the daemon message is a failed manifest build (exit 1), not a daemon that is down (exit 5). BuildKit repeats the command in its last line, so the substring check matched it. Thread: PRRT_kwDOVBg7g863zd3V Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/container.py | 12 +++++++++--- dokimi/tests/test_exit_codes_infra.py | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/container.py b/dokimi/dokimi/repos/container.py index 74a8c37bd..95fc0987d 100644 --- a/dokimi/dokimi/repos/container.py +++ b/dokimi/dokimi/repos/container.py @@ -36,6 +36,7 @@ from __future__ import annotations import os +import re from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -196,11 +197,16 @@ def image_exists(self, tag: str) -> bool: return False +_DAEMON_DIAGNOSTIC = re.compile(r"^(?:(?:error|docker):\s+)?(?:cannot connect to the docker daemon|error during connect|" + r"permission denied while trying to connect to the docker daemon)") + + def _daemon_unreachable(stderr: str | None) -> bool: """True when the docker CLI says, in its last lines, that it could not reach the daemon (down, or no permission on - its socket). The last lines only: a RUN step of a build may print the same words in the middle of its output.""" - lines = [ln for ln in (stderr or "").lower().splitlines() if ln.strip()][-2:] - return any("connect to the docker daemon" in ln or "error during connect" in ln for ln in lines) + its socket). Only a line that opens with the CLI's own diagnostic counts: a RUN step prints its command text, and + BuildKit repeats that command in its final `failed to solve` line, so the words can sit in the last lines too.""" + lines = [ln.strip().lower() for ln in (stderr or "").splitlines() if ln.strip()][-2:] + return any(_DAEMON_DIAGNOSTIC.match(ln) for ln in lines) def run_process(argv: list[str], *, env: process.Env | dict[str, str], stdout: Path, stderr: Path, diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index d5ffbb750..859d8ed9f 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -59,6 +59,30 @@ def test_a_daemon_that_is_down_is_a_docker_error_before_any_build(): assert [c.argv[1] for c in fake.calls] == ["image"] # the build is never started +BUILDKIT_QUOTED_MESSAGE = ( + "#5 [3/4] RUN echo 'Cannot connect to the Docker daemon' && false\n" + "#5 0.1 Cannot connect to the Docker daemon\n" + "ERROR: failed to solve: process \"/bin/sh -c echo 'Cannot connect to the Docker daemon' && false\" " + "did not complete successfully: exit code: 1\n" +) + + +def test_a_build_step_that_quotes_the_daemon_message_is_a_container_error_not_infra(tmp_path): + """A RUN step whose command text names the daemon message is a failed manifest build (exit 1), not a daemon + that is down (exit 5): only the CLI's own top-level diagnostic counts.""" + from dokimi import process + from dokimi.repos.container import ContainerError, DockerError, DockerRunner + + (tmp_path / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + quoted = process.FakeRunner(lambda call: process.reply(stderr=BUILDKIT_QUOTED_MESSAGE, returncode=1)) + with pytest.raises(ContainerError) as exc: + DockerRunner(runner=quoted).build(tmp_path, tmp_path / "Dockerfile", "t") + assert not isinstance(exc.value, DockerError) + down = process.FakeRunner(lambda call: process.reply(stderr=DAEMON_DOWN, returncode=1)) + with pytest.raises(DockerError): + DockerRunner(runner=down).build(tmp_path, tmp_path / "Dockerfile", "t") + + def test_a_docker_cli_missing_on_the_image_inspect_is_a_docker_error(): from dokimi import process from dokimi.repos.container import DockerError, DockerRunner From 3cc2398782f605eb3a07619d3c7921d2a7f99020 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:23:22 -0800 Subject: [PATCH 03/19] Address review: pin the infra exit of an unwritable cell result and spend file Flipping the infra flag of a CellCrash, or of a failed spend write, in run_batch passed every repo run test. Each case now fails under that flip. Thread: PRRT_kwDOVBg7g8630B4b Review: PRR_kwDOVBg7g88AAAABYKDFnA Signed-off-by: Nikolay Petrov --- .../tests/test_repo_run_unreadable_result.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/dokimi/tests/test_repo_run_unreadable_result.py b/dokimi/tests/test_repo_run_unreadable_result.py index 2fc304afb..4da08ca8c 100644 --- a/dokimi/tests/test_repo_run_unreadable_result.py +++ b/dokimi/tests/test_repo_run_unreadable_result.py @@ -78,3 +78,38 @@ def test_an_unreadable_kept_result_of_a_task_this_host_cannot_run_is_that_cells_ assert f"{fix_add}, 2 crashed, 0 not started (batch budget), 0 unsupported" in logged assert [listed.read_bytes(), cut.read_bytes()] == before # neither rerun nor marked unsupported assert not list((out / "b1").glob("*/unsupported.json")) + + +def test_a_cell_whose_result_cannot_be_written_exits_5_and_the_other_cells_still_run(setup, monkeypatch, capsys): + """A CellCrash is infra (exit 5): flipping its `infra` flag must fail this test, not pass unnoticed.""" + from dokimi.repos import runner + + write_json = runner._write_json + + def refuse_r1(path, data): + if path.name == "result.json" and path.parent.name.endswith("__r1"): + raise OSError("disk full") + write_json(path, data) + + monkeypatch.setattr(runner, "_write_json", refuse_r1) + with pytest.raises(SystemExit) as exit_: + _run(setup) + assert exit_.value.code == 5 + out = capsys.readouterr().out + assert "✗ fix_add__guidance-raw__m-1__r1: cannot write result.json: disk full" in out + assert "2 ran" in out + + +def test_a_batch_spend_file_that_cannot_be_written_exits_5(setup, monkeypatch, capsys): + """The spend file is the batch cap's record: a write failure is infra (exit 5), not a cell's input error.""" + from dokimi.repos import runner + + def refuse(batch_dir, spent): + raise OSError("disk full") + + monkeypatch.setattr(runner, "_write_batch_spend", refuse) + with pytest.raises(SystemExit) as exit_: + _run(setup) + assert exit_.value.code == 5 + captured = capsys.readouterr() + assert "3 ran, 0 already done, 3 crashed" in captured.out # one spend error per charged cell From 77cb2ee912f6823bd955a051ede2c8f7d363cd69 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:27:04 -0800 Subject: [PATCH 04/19] Address review: accept the older CLI's "Got permission denied" socket error The anchored match missed `Got permission denied while trying to connect to the Docker daemon socket`, which the old substring check caught, so a host with no socket permission exited 1 instead of 5. Adds that stderr as a regression test (red before this change). Thread: PRRT_kwDOVBg7g863zd3V Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/container.py | 7 +++---- dokimi/tests/test_exit_codes_infra.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dokimi/dokimi/repos/container.py b/dokimi/dokimi/repos/container.py index 95fc0987d..ef663905f 100644 --- a/dokimi/dokimi/repos/container.py +++ b/dokimi/dokimi/repos/container.py @@ -197,14 +197,13 @@ def image_exists(self, tag: str) -> bool: return False -_DAEMON_DIAGNOSTIC = re.compile(r"^(?:(?:error|docker):\s+)?(?:cannot connect to the docker daemon|error during connect|" - r"permission denied while trying to connect to the docker daemon)") +_DAEMON_DIAGNOSTIC = re.compile(r"^(?:(?:error|docker):\s+)?(?:got\s+)?(?:cannot connect to the docker daemon|" + r"error during connect|permission denied while trying to connect to the docker daemon)") def _daemon_unreachable(stderr: str | None) -> bool: """True when the docker CLI says, in its last lines, that it could not reach the daemon (down, or no permission on - its socket). Only a line that opens with the CLI's own diagnostic counts: a RUN step prints its command text, and - BuildKit repeats that command in its final `failed to solve` line, so the words can sit in the last lines too.""" + its socket). Only a line opening with the CLI's own diagnostic counts: BuildKit repeats a RUN command in its last line.""" lines = [ln.strip().lower() for ln in (stderr or "").splitlines() if ln.strip()][-2:] return any(_DAEMON_DIAGNOSTIC.match(ln) for ln in lines) diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 859d8ed9f..101657ba5 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -59,6 +59,19 @@ def test_a_daemon_that_is_down_is_a_docker_error_before_any_build(): assert [c.argv[1] for c in fake.calls] == ["image"] # the build is never started +def test_an_older_cli_got_permission_denied_on_the_socket_is_a_docker_error(tmp_path): + """Older docker CLIs open the socket error with `Got permission denied`: no permission on the socket is infra.""" + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + got = ("Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: " + "Post \"http://%2Fvar%2Frun%2Fdocker.sock/v1.24/build\": dial unix /var/run/docker.sock: permission denied\n") + (tmp_path / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + fake = process.FakeRunner(lambda call: process.reply(stderr=got, returncode=1)) + with pytest.raises(DockerError): + DockerRunner(runner=fake).build(tmp_path, tmp_path / "Dockerfile", "t") + + BUILDKIT_QUOTED_MESSAGE = ( "#5 [3/4] RUN echo 'Cannot connect to the Docker daemon' && false\n" "#5 0.1 Cannot connect to the Docker daemon\n" From d6675880efd59cb7e35aab495627278627c333de Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:27:04 -0800 Subject: [PATCH 05/19] Address review: match the full summary line of the unwritable-result test Thread: PRRT_kwDOVBg7g8630B4b Review: PRR_kwDOVBg7g88AAAABYKDFnA Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_run_unreadable_result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_run_unreadable_result.py b/dokimi/tests/test_repo_run_unreadable_result.py index 4da08ca8c..0e8f502f3 100644 --- a/dokimi/tests/test_repo_run_unreadable_result.py +++ b/dokimi/tests/test_repo_run_unreadable_result.py @@ -97,7 +97,7 @@ def refuse_r1(path, data): assert exit_.value.code == 5 out = capsys.readouterr().out assert "✗ fix_add__guidance-raw__m-1__r1: cannot write result.json: disk full" in out - assert "2 ran" in out + assert "2 ran, 0 already done, 1 crashed" in out def test_a_batch_spend_file_that_cannot_be_written_exits_5(setup, monkeypatch, capsys): From eacee60292929806756ad15ece61ac52843e61af Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:43:43 -0800 Subject: [PATCH 06/19] Mark the A2b test renames as narrowed on purpose The renamed tests pin the exit codes of the A2b contract (2 for an input error, 3 and 4 for --strict). Each carries a Test-narrowed comment with its old name; the pass/fail intent of each case is unchanged. Test-narrowed: renamed to the A2b exit codes (2 input, 3 inconclusive, 4 invalid); old names in each comment Signed-off-by: Nikolay Petrov --- dokimi/tests/test_cli_output.py | 1 + dokimi/tests/test_context_trace.py | 2 ++ dokimi/tests/test_judge_not_measured.py | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/dokimi/tests/test_cli_output.py b/dokimi/tests/test_cli_output.py index e40c00955..898c484fe 100644 --- a/dokimi/tests/test_cli_output.py +++ b/dokimi/tests/test_cli_output.py @@ -294,6 +294,7 @@ def test_trace_inspect_json_is_partial_when_a_trace_does_not_read(tmp_path, caps def test_trace_consistency_is_an_input_error_when_no_trace_reads(tmp_path, capsys): + # Test-narrowed: renamed from ..._json_is_failed_when_no_trace_reads; no-trace reads exit 2 (input), not 1 (A2b) args = cli.parse_args(["trace", "consistency", "--arm", f"base={tmp_path}/missing.jsonl", "--json"]) with pytest.raises(InputError, match="no trace could be read"): args.func(args) diff --git a/dokimi/tests/test_context_trace.py b/dokimi/tests/test_context_trace.py index bb31c1285..7fd46a677 100644 --- a/dokimi/tests/test_context_trace.py +++ b/dokimi/tests/test_context_trace.py @@ -164,6 +164,7 @@ def test_cli_json_single_trace(tmp_path, capsys): @pytest.mark.outcome("cjm-context-trace") def test_cli_missing_trace_is_an_input_error(tmp_path, capsys): + # Test-narrowed: renamed from test_cli_missing_trace_exits_1; the exit it pins is 2 (input), not 1 (A2b) """Nothing read is a bad input (exit 2), not a partial result (exit 1).""" with pytest.raises(InputError, match="no trace could be read"): _run_cli(["trace", "inspect", str(tmp_path / "missing.jsonl")], capsys) @@ -171,6 +172,7 @@ def test_cli_missing_trace_is_an_input_error(tmp_path, capsys): @pytest.mark.outcome("cjm-context-trace") def test_cli_json_with_no_trace_read_is_an_input_error_document(tmp_path, monkeypatch, capsys): + # Test-narrowed: renamed from ..._is_failed_with_no_data; --json prints the input error document (A2b) """Nothing read exits 2 with an `input` error document; a consumer that reads `error.code` sees a refusal.""" from dokimi import cli diff --git a/dokimi/tests/test_judge_not_measured.py b/dokimi/tests/test_judge_not_measured.py index cb29bf8c6..068967b98 100644 --- a/dokimi/tests/test_judge_not_measured.py +++ b/dokimi/tests/test_judge_not_measured.py @@ -29,6 +29,7 @@ def _strict_exit(argv): def test_cross_judge_strict_is_inconclusive_when_no_arm_was_measured(tmp_path, monkeypatch, capsys): + # Test-narrowed: renamed from ..._fails_when_no_arm_was_measured; --strict exits 3 (inconclusive) (A2b) import json from dokimi.eval import judge as judge_mod @@ -63,6 +64,7 @@ def score_answer(self, **_): def test_cross_judge_strict_exits_3_when_only_one_of_several_arms_was_measured(tmp_path, monkeypatch, capsys): + # Test-narrowed: renamed from ..._exits_1_...; NOT MEASURED exits 3 under --strict (A2b) import json from dokimi.eval import judge as judge_mod @@ -162,6 +164,7 @@ def test_no_voter_left_by_the_validity_gate_is_not_measured(): def test_cross_judge_strict_exits_3_when_require_validated_leaves_no_voter(tmp_path, capsys): + # Test-narrowed: renamed from ..._exits_1_...; no voter exits 3 under --strict (A2b) import json path = tmp_path / "run.json" @@ -174,6 +177,7 @@ def test_cross_judge_strict_exits_3_when_require_validated_leaves_no_voter(tmp_p def test_cross_judge_judge_dependent_exits_4_only_with_strict(tmp_path, monkeypatch, capsys): + # Test-narrowed: renamed from ..._exits_1_only_with_strict; JUDGE-DEPENDENT exits 4 (A2b) import json from dokimi.eval import judge as judge_mod @@ -197,6 +201,7 @@ def score_answer(self, *, answer, **_): # prefers the baseline, the stored judg def test_cross_judge_strict_exits_4_when_one_of_several_arms_is_judge_dependent(tmp_path, monkeypatch, capsys): + # Test-narrowed: renamed from ..._exits_1_...; JUDGE-DEPENDENT arm exits 4 under --strict (A2b) import json from dokimi.eval import judge as judge_mod From 614dbc245ae041bbf9cc7f5824b02aaf7e36b9f0 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:46:25 -0800 Subject: [PATCH 07/19] Address review: a harness that exits nonzero before any trace is a crash A harness that exits nonzero before its first trace line is not a verdict on the cell. The cell raises CellCrash, so the batch exits 5, and it writes no result.json, so the next run runs it again. A harness that exits 0 with no trace stays a fail cell. The changelog, quickstart, contract and the behavioural-security note follow the exit rule. Thread: PRRT_kwDOVBg7g8630B4h Signed-off-by: Nikolay Petrov --- ...1115_120000_napetrov_fix_cli-a2b5-exits.md | 2 +- dokimi/docs/behavioural-security.md | 3 ++- dokimi/docs/quickstart.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 +++- dokimi/dokimi/repos/runner.py | 10 ++++++++- dokimi/tests/repo_eval_fixture.py | 21 +++++++++++++++++++ dokimi/tests/test_repo_run.py | 4 ++-- .../tests/test_repo_run_unreadable_result.py | 13 ++++++++++++ dokimi/tests/test_repo_skill_arms.py | 10 ++++----- dokimi/tests/test_repo_uptake.py | 4 ++-- 10 files changed, 58 insertions(+), 15 deletions(-) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md index 1f0460deb..51f26701d 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md @@ -6,6 +6,6 @@ SPDX-License-Identifier: Apache-2.0 ### Changed - **Exit codes for the cases below** — scripts that branch on the exit code see these changes: - - `dokimi repo run`: a cell that crashes (a harness failure, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. + - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. - `dokimi eval judge agreement --strict` exits 3 (inconclusive) when an arm is NOT MEASURED or no arm can be compared, and 4 (invalid) when an arm is JUDGE-DEPENDENT; it exited 1 for both before. Without `--strict` the exit code is unchanged (0), and a judge that scored none of its answers still exits 1. - `dokimi trace inspect`, `dokimi trace consistency` exit 2 (bad input) when no trace can be read; they exited 1 before. With `--json` they print the `input` error document (`error.code` `input`, `data` null) instead of a document with `status` `failed`. A partial read still exits 1. diff --git a/dokimi/docs/behavioural-security.md b/dokimi/docs/behavioural-security.md index 9955945d3..4f6701ad1 100644 --- a/dokimi/docs/behavioural-security.md +++ b/dokimi/docs/behavioural-security.md @@ -58,7 +58,8 @@ of cell that are in no rate: - `not_started`: the cell finished, but nothing shows the agent ran (no trace event, a null `process` in `result.json`, no finding; an empty `events.jsonl` is what the runner - writes when the agent never started, even though its `rc` is set, unless the runner also + writes when the agent never started with `rc` 0; a nonzero `rc` with no trace is a crash + (exit 5) that writes no `result.json`, so it is in no rate. Unless the runner also recorded `trace_errors`, see `unfinished`). Counting it as clean would make an arm whose skill crashes the agent look safer. Each arm entry gives its `not_started` count; its `cells` (the denominator of every rate) are the measured ones. An arm, the baseline diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index d6b112385..3573795d0 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -261,7 +261,7 @@ failed check. | 2 | A bad invocation: a missing, unreadable or malformed input file, a device given as one (` : not a file`), a flag combination the command refuses, an output file that cannot be written, a missing provider credential, a source string that is malformed, unpinned or not a repository root. | `baseline save` on a bad eval file, `evaluate --custom-questions` on an empty or non-JSON file (`--custom-questions : `), `benchmark preflight` with no golden set, `repo scan --manifest ../x.yaml` | | 3 | Inconclusive: the run stopped short because a budget ran out (`--max-cost-usd`, or a call with no price) and kept the partial output (`tools preference` today; other commands still exit 1 on a budget stop), or the result is undecided (`eval judge validate`: a pass whose interval straddles the gate, or a variant no pair was built for, with `--strict` making it 1; `eval leak`: an item that could not be checked, or none could; `eval redteam`: a graded question not measured, or nothing graded; `eval judge agreement --strict`: an arm NOT MEASURED, or no arm to compare; `repo security --fail-on`: an unfinished cell or an arm with no measured cell, when no arm failed and every model has a baseline cell). | `tools preference --max-cost-usd 5` with calls not run; `eval judge validate` with too few bases | | 4 | Invalid: the result is not comparable or not trustworthy: a model that differs from the baseline's, or one that cannot be verified, under `baseline compare --strict` or the regression gate (`--max-context-drop` or `--max-lift-drop`); under `repo security --fail-on`, a model with no baseline cell when no arm failed; under `eval judge agreement --strict`, an arm JUDGE-DEPENDENT. | `baseline compare --strict` over a baseline scored with another judge | -| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | +| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness that exits nonzero before its first trace line, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 16574ed34..f4beb5b87 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -696,7 +696,9 @@ failed still do. The probed facts are recorded in `check.json` `host`, and for each holds an `unsupported.json` with the reason, and `repo report` lists them apart, neither failed nor missing. If the probe itself fails, a task with `host` requirements is not run either, and `repo run` exits 1 when one of its -cells is left unrun; a cell that already has a `result.json` keeps it. +cells is left unrun; a cell that already has a `result.json` keeps it. A harness that exits nonzero +before its first trace line is not a verdict on the cell: the cell is a crash, the batch exits 5, and it writes +no `result.json`, so the next run runs it again. Both commands exit 2, before any build, on a source that does not parse (a local path that is not a directory, an unpinned remote) and on a pinned local path inside a repository that is not its root; a diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 65ada2309..65a8d1bb5 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -274,6 +274,11 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta["model_resolved"] = transcript.model_resolved() # an alias can map `model` to another one _write_json(rd / "meta.json", meta) started = bool(transcript.events) or transcript.result_seen # else the agent never started + no_trace = not started and not transcript.is_error and not transcript.errors # nothing the harness wrote + if no_trace and result.returncode != 0 and not result.timed_out: + # the harness failed before its first line: a crash of the machinery, not a verdict on the cell + raise CellCrash(f"the harness exited {result.returncode} before its first trace line", + float(agent["charged_usd"])) if started: events = transcript.events useful_events = None if transcript.dropped_lines else events @@ -296,6 +301,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, skills = skill_usage(transcript.events, [s["name"] for s in meta["skills"]], task.expected_skill) graded = run_entry(ctx, "grade", task, rd, grade_network, container_name(rd.name, "grade")) grade = outcome(task, rd, graded) + except CellCrash: # the harness failed before its trace: the cell is not graded, it has no result + raise except Exception as exc: # noqa: BLE001 # the grade run failed to start, or the host failed around it # What the agent left behind (a read-only .git, a hostile trace) can break any host step after it. # Excluding the run would favour arms that break workspaces (contract §4.2): it fails the task. @@ -340,7 +347,8 @@ def _is_infra(exc: BaseException) -> bool: class CellCrash(Exception): - """A cell whose agent ran but whose ``result.json`` could not be written: its cost still counts.""" + """A cell whose agent ran but whose ``result.json`` could not be written, or whose harness exited nonzero before + any trace line (no result is written, so the next run runs the cell again): its cost still counts.""" def __init__(self, message: str, charged_usd: float): super().__init__(message) diff --git a/dokimi/tests/repo_eval_fixture.py b/dokimi/tests/repo_eval_fixture.py index 4925cfa20..b424b5286 100644 --- a/dokimi/tests/repo_eval_fixture.py +++ b/dokimi/tests/repo_eval_fixture.py @@ -18,6 +18,7 @@ from __future__ import annotations +import dataclasses import json import os import re @@ -292,6 +293,11 @@ def host(text: str) -> str: stdout.write_text("".join(f"{s}\n" for s in seen)) stderr.write_text("") return ContainerResult(returncode=1 if seen else 0, timed_out=False, wall_s=0.0) + if os.environ.get("FAKE_ACTION") == "quiet" and "-p" in argv: + # the agent ends before its first line with exit 0 (the fake script is left as it is: its digest is pinned) + stdout.write_text("") + stderr.write_text("") + return ContainerResult(returncode=0, timed_out=False, wall_s=0.0) if argv[0] == "python3": argv[0] = sys.executable # FAKE_CONTAINER_CWD: the fake harness reports paths as a harness in the container would @@ -304,3 +310,18 @@ def host(text: str) -> str: if not cwd.is_dir(): cwd = stdout.parent return run_process(argv, env=env, stdout=stdout, stderr=stderr, timeout_s=timeout_s, cwd=cwd) + + +def config_runner(tmp_path, monkeypatch): + """A FakeRunner whose agent gets FAKE_CONFIG_DIR: the built-in claude-code profile does not set it, and the fake + agent exits 1 without it (a crash, exit 5, now that a nonzero exit with no trace is one).""" + runner = FakeRunner() + config = str(tmp_path / "config") + + def run(spec, **kwargs): + env = {**spec.env, "FAKE_CONFIG_DIR": config} + return FakeRunner.run(runner, dataclasses.replace(spec, env=env), **kwargs) + + runner.run = run + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + return runner diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 78c0f3f58..7ad39a182 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -124,8 +124,8 @@ def refuse(*args, **kwargs): def test_an_agent_that_left_no_trace_is_not_given_a_process_metrics_error(setup, monkeypatch): - # No events and no turns: the metrics are not run on the missing trace, so they record no failure. - monkeypatch.setenv("FAKE_ACTION", "silent") + # No events and no turns (exit 0: a nonzero exit is a crash): the metrics are not run on the missing trace. + monkeypatch.setenv("FAKE_ACTION", "quiet") _run(setup, "--arm", "guidance:raw", "--task", "fix_add") rec = json.loads((setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").read_text()) assert rec["process"] is None diff --git a/dokimi/tests/test_repo_run_unreadable_result.py b/dokimi/tests/test_repo_run_unreadable_result.py index 0e8f502f3..8bef08dac 100644 --- a/dokimi/tests/test_repo_run_unreadable_result.py +++ b/dokimi/tests/test_repo_run_unreadable_result.py @@ -113,3 +113,16 @@ def refuse(batch_dir, spent): assert exit_.value.code == 5 captured = capsys.readouterr() assert "3 ran, 0 already done, 3 crashed" in captured.out # one spend error per charged cell + + +def test_a_harness_that_exits_nonzero_before_any_trace_is_a_crash_exit_5(setup, monkeypatch, capsys): + """A nonzero exit with no trace is the harness failing, not a verdict on the agent: exit 5, no result.json, so + the next run executes the cell again (flipping the CellCrash for it must fail this test).""" + monkeypatch.setenv("FAKE_ACTION", "silent") + with pytest.raises(SystemExit) as exit_: + _run(setup) + assert exit_.value.code == 5 + assert not list((setup["out"] / "b1").glob("fix_add__*/result.json")) + out = capsys.readouterr().out + assert "0 ran, 0 already done, 3 crashed" in out # the three reps of the cell + assert "the harness exited 1 before its first trace line" in out diff --git a/dokimi/tests/test_repo_skill_arms.py b/dokimi/tests/test_repo_skill_arms.py index ea72b6be4..c92b03585 100644 --- a/dokimi/tests/test_repo_skill_arms.py +++ b/dokimi/tests/test_repo_skill_arms.py @@ -20,7 +20,7 @@ from dokimi.repos.events import convert from dokimi.repos.report import format_report, summarize from dokimi.repos.workspace import PrepError, Task, _install_skills, parse_arm -from repo_eval_fixture import FakeRunner, fake_harness, git, make_repo, write +from repo_eval_fixture import FakeRunner, config_runner, fake_harness, git, make_repo, write def _skill(root: Path, rel: str, name: str) -> None: @@ -451,7 +451,7 @@ def cell(name, skills): def test_a_harness_that_never_started_leaves_skill_usage_unmeasured(setup, monkeypatch): - monkeypatch.setenv("FAKE_ACTION", "startfail") # exit 1, no trace: not "the agent ignored the skill" + monkeypatch.setenv("FAKE_ACTION", "quiet") # exit 0, no trace: not "the agent ignored the skill" (exit 1 is a crash) _run(setup, "--arm", "guidance:raw+skill:skills/lint", "--task", "fix_lint", "--reps", "2") monkeypatch.setenv("FAKE_ACTION", "fix") # ran, used no skill: measured _run(setup, "--batch", "b2", "--arm", "guidance:raw+skill:skills/lint", "--task", "fix_lint") @@ -514,8 +514,7 @@ def test_a_resumed_batch_refuses_other_skills_under_one_arm_name(setup, tmp_path def test_claude_code_gets_mcp_config_only_when_the_workspace_has_the_file(tmp_path, monkeypatch): - runner = FakeRunner() - monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + runner = config_runner(tmp_path, monkeypatch) monkeypatch.setenv("FAKE_ACTION", "fix") profile = Path(__file__).resolve().parents[1] / "dokimi/repos/profiles/claude-code.yaml" exe, _ = fake_harness(tmp_path) @@ -815,8 +814,7 @@ def test_a_read_of_a_skills_directory_is_not_a_use_of_a_skill(): def _claude_run(tmp_path, monkeypatch, mcp_json, *extra, patch=None): """`repo run` with the built-in claude-code profile on a base that ships `mcp_json`; the agent specs.""" - runner = FakeRunner() - monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + runner = config_runner(tmp_path, monkeypatch) monkeypatch.setenv("FAKE_ACTION", "fix") profile = Path(__file__).resolve().parents[1] / "dokimi/repos/profiles/claude-code.yaml" exe, _ = fake_harness(tmp_path) diff --git a/dokimi/tests/test_repo_uptake.py b/dokimi/tests/test_repo_uptake.py index 9d0c6f761..10681877c 100644 --- a/dokimi/tests/test_repo_uptake.py +++ b/dokimi/tests/test_repo_uptake.py @@ -231,10 +231,10 @@ def test_an_agent_that_left_no_trace_is_unmeasured_one_that_called_no_tool_is_me patch = setup["tmp"] / "kit.diff" patch.write_text(KIT_PATCH) arm = str(parse_arm(f"patch:{patch}")) - for action in ("silent", "idle"): + for action in ("quiet", "idle"): # exit 0 with no trace: a nonzero exit with no trace is a crash (exit 5) monkeypatch.setenv("FAKE_ACTION", action) _run(setup, action, "guidance:raw", f"patch:{patch}") - silent, idle = setup["out"] / "silent", setup["out"] / "idle" + silent, idle = setup["out"] / "quiet", setup["out"] / "idle" assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["uptake"]["measured"] is False # an agent that left no trace has no process counts; a run that started and made no call has zeros assert json.loads(next(silent.glob("*patch*__r1/result.json")).read_text())["process"] is None From ff47728a936e3ad5ffe7d3ab823c590af95ad29b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:59:14 -0800 Subject: [PATCH 08/19] test, docs: pin the retry of a crashed cell and the timed-out no-trace cell A crashed cell writes no result.json, so the next run executes it again. A nonzero harness exit that the cap killed before its first trace line is a fail cell, not a crash: the docs now say so where they name the crash. Thread: PRRT_kwDOVBg7g8630B4h Signed-off-by: Nikolay Petrov --- ...1115_120000_napetrov_fix_cli-a2b5-exits.md | 2 +- dokimi/docs/behavioural-security.md | 6 ++--- dokimi/docs/quickstart.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 3 ++- .../tests/test_repo_run_unreadable_result.py | 24 +++++++++++++++++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md index 51f26701d..c7e71f95e 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md @@ -6,6 +6,6 @@ SPDX-License-Identifier: Apache-2.0 ### Changed - **Exit codes for the cases below** — scripts that branch on the exit code see these changes: - - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. + - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. - `dokimi eval judge agreement --strict` exits 3 (inconclusive) when an arm is NOT MEASURED or no arm can be compared, and 4 (invalid) when an arm is JUDGE-DEPENDENT; it exited 1 for both before. Without `--strict` the exit code is unchanged (0), and a judge that scored none of its answers still exits 1. - `dokimi trace inspect`, `dokimi trace consistency` exit 2 (bad input) when no trace can be read; they exited 1 before. With `--json` they print the `input` error document (`error.code` `input`, `data` null) instead of a document with `status` `failed`. A partial read still exits 1. diff --git a/dokimi/docs/behavioural-security.md b/dokimi/docs/behavioural-security.md index 4f6701ad1..9208a7c2f 100644 --- a/dokimi/docs/behavioural-security.md +++ b/dokimi/docs/behavioural-security.md @@ -58,9 +58,9 @@ of cell that are in no rate: - `not_started`: the cell finished, but nothing shows the agent ran (no trace event, a null `process` in `result.json`, no finding; an empty `events.jsonl` is what the runner - writes when the agent never started with `rc` 0; a nonzero `rc` with no trace is a crash - (exit 5) that writes no `result.json`, so it is in no rate. Unless the runner also - recorded `trace_errors`, see `unfinished`). Counting it as clean would make an arm + writes when the agent never started, even though its `rc` is 0 or the cap killed it, unless the runner also + recorded `trace_errors`, see `unfinished`). A nonzero `rc` with no trace, unless the cap + killed the harness (`timed_out`), is not a cell at all: it is a crash (exit 5) that writes no `result.json`. Counting it as clean would make an arm whose skill crashes the agent look safer. Each arm entry gives its `not_started` count; its `cells` (the denominator of every rate) are the measured ones. An arm, the baseline included, whose every cell is `not_started` has no rate at all: the report shows it as diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index 3573795d0..0afd8d969 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -261,7 +261,7 @@ failed check. | 2 | A bad invocation: a missing, unreadable or malformed input file, a device given as one (` : not a file`), a flag combination the command refuses, an output file that cannot be written, a missing provider credential, a source string that is malformed, unpinned or not a repository root. | `baseline save` on a bad eval file, `evaluate --custom-questions` on an empty or non-JSON file (`--custom-questions : `), `benchmark preflight` with no golden set, `repo scan --manifest ../x.yaml` | | 3 | Inconclusive: the run stopped short because a budget ran out (`--max-cost-usd`, or a call with no price) and kept the partial output (`tools preference` today; other commands still exit 1 on a budget stop), or the result is undecided (`eval judge validate`: a pass whose interval straddles the gate, or a variant no pair was built for, with `--strict` making it 1; `eval leak`: an item that could not be checked, or none could; `eval redteam`: a graded question not measured, or nothing graded; `eval judge agreement --strict`: an arm NOT MEASURED, or no arm to compare; `repo security --fail-on`: an unfinished cell or an arm with no measured cell, when no arm failed and every model has a baseline cell). | `tools preference --max-cost-usd 5` with calls not run; `eval judge validate` with too few bases | | 4 | Invalid: the result is not comparable or not trustworthy: a model that differs from the baseline's, or one that cannot be verified, under `baseline compare --strict` or the regression gate (`--max-context-drop` or `--max-lift-drop`); under `repo security --fail-on`, a model with no baseline cell when no arm failed; under `eval judge agreement --strict`, an arm JUDGE-DEPENDENT. | `baseline compare --strict` over a baseline scored with another judge | -| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness that exits nonzero before its first trace line, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | +| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index f4beb5b87..550972ea5 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -698,7 +698,8 @@ apart, neither failed nor missing. If the probe itself fails, a task with `host` requirements is not run either, and `repo run` exits 1 when one of its cells is left unrun; a cell that already has a `result.json` keeps it. A harness that exits nonzero before its first trace line is not a verdict on the cell: the cell is a crash, the batch exits 5, and it writes -no `result.json`, so the next run runs it again. +no `result.json`, so the next run runs it again. A harness the cap killed before its first trace line is a fail +cell, as before. Both commands exit 2, before any build, on a source that does not parse (a local path that is not a directory, an unpinned remote) and on a pinned local path inside a repository that is not its root; a diff --git a/dokimi/tests/test_repo_run_unreadable_result.py b/dokimi/tests/test_repo_run_unreadable_result.py index 8bef08dac..766ddc007 100644 --- a/dokimi/tests/test_repo_run_unreadable_result.py +++ b/dokimi/tests/test_repo_run_unreadable_result.py @@ -5,6 +5,8 @@ from __future__ import annotations +import json + import pytest from dokimi.cli import build_parser @@ -126,3 +128,25 @@ def test_a_harness_that_exits_nonzero_before_any_trace_is_a_crash_exit_5(setup, out = capsys.readouterr().out assert "0 ran, 0 already done, 3 crashed" in out # the three reps of the cell assert "the harness exited 1 before its first trace line" in out + setup["runner"].specs.clear() + with pytest.raises(SystemExit) as again: # the next run executes the cells again: none is kept + _run(setup) + assert again.value.code == 5 and "0 ran, 0 already done, 3 crashed" in capsys.readouterr().out + assert len([s for s in setup["runner"].specs if "-p" in s.argv]) == 3 + + +def test_a_nonzero_timed_out_harness_with_no_trace_is_a_fail_cell_not_a_crash(setup, monkeypatch): + """The cap ran out before the agent wrote anything: a verdict on the cell (a fail), not a crash of the harness.""" + from dokimi.repos.container import ContainerResult + + fake = setup["runner"].run + + def time_out(spec, **kwargs): + res = fake(spec, **kwargs) + return ContainerResult(returncode=res.returncode, timed_out=True, wall_s=res.wall_s) if "-p" in spec.argv else res + + monkeypatch.setattr(setup["runner"], "run", time_out) + monkeypatch.setenv("FAKE_ACTION", "silent") + _run(setup) # no SystemExit: the batch holds results + rec = json.loads(next((setup["out"] / "b1").glob("fix_add__*__r1/result.json")).read_text(encoding="utf-8")) + assert rec["agent"]["timed_out"] is True and rec["grade"]["success"] is False From 5aa80f3cdd63f369d6e52d7d2a26755120495456 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 12:54:22 -0800 Subject: [PATCH 09/19] Address review: a harness that prints a trace line before its crash keeps its cell A text trace has no events and a start-up record has no event either, so both read as no trace and exited 5. Their stdout or start-up record is the trace line; a crash after it is a cell with a result. Thread: PRRT_kwDOVBg7g864HKTL Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/runner.py | 3 ++- dokimi/tests/test_repo_run.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index 65a8d1bb5..d04f95152 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -274,7 +274,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, meta["model_resolved"] = transcript.model_resolved() # an alias can map `model` to another one _write_json(rd / "meta.json", meta) started = bool(transcript.events) or transcript.result_seen # else the agent never started - no_trace = not started and not transcript.is_error and not transcript.errors # nothing the harness wrote + traced = started or bool(transcript.start_record) or (opts.profile.trace == "text" and bool(raw.strip())) + no_trace = not traced and not transcript.is_error and not transcript.errors # nothing the harness wrote if no_trace and result.returncode != 0 and not result.timed_out: # the harness failed before its first line: a crash of the machinery, not a verdict on the cell raise CellCrash(f"the harness exited {result.returncode} before its first trace line", diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index 7ad39a182..d2765bead 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -132,6 +132,35 @@ def test_an_agent_that_left_no_trace_is_not_given_a_process_metrics_error(setup, assert not any("process metrics failed" in e for e in rec["agent"]["trace_errors"]) +def _stub_harness(tmp_path, body): + """A harness that answers --version, then runs ``body`` (the fixture binary is pinned by a golden digest).""" + exe = tmp_path / "stub-agent" + exe.write_text('#!/usr/bin/env python3\nimport json, sys\n' + 'if sys.argv[1:] == ["--version"]:\n print("1.2.3 (Stub Agent)")\n sys.exit(0)\n' + body, + encoding="utf-8") + exe.chmod(0o755) + return exe + + +def test_a_harness_that_crashes_after_its_start_record_has_a_trace_and_is_not_a_crash(setup, tmp_path): + # The start-up record is a trace line: the agent started, so the cell keeps its result (no exit 5). + body = 'print(json.dumps({"type": "system", "subtype": "init", "model": "m-1"}))\nsys.exit(1)\n' + _run({**setup, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", "--task", "fix_add") + assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() + + +def test_a_text_harness_whose_stdout_is_its_trace_is_not_a_crash(setup, tmp_path): + # trace: text has no events, so its stdout is the only trace line: a crash after an answer keeps the cell. + prof = yaml.safe_load(setup["profile"].read_text(encoding="utf-8")) + prof["trace"] = "text" + text = tmp_path / "text-profile.yaml" + text.write_text(yaml.safe_dump(prof), encoding="utf-8") + body = 'print("The answer is 4.")\nsys.exit(1)\n' + _run({**setup, "profile": text, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", + "--task", "fix_add") + assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() + + def test_a_process_metrics_error_leaves_process_null_and_keeps_the_cost_and_the_grade(setup, monkeypatch): # The metric is best-effort: a counter that raises must not fail the task or drop the cost the harness reported. def refuse(*args, **kwargs): From 75d8d0a80e572f89a9d24fba5cc7c96094438446 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 12:54:22 -0800 Subject: [PATCH 10/19] Address review: qualify the docker-failure exit 5 promise A grader-phase docker failure is recorded in result.json, not a crash, so exit 5 is narrowed to a docker failure that stops the agent. Thread: PRRT_kwDOVBg7g864HKTJ Signed-off-by: Nikolay Petrov --- .../changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md | 2 +- dokimi/docs/quickstart.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md index c7e71f95e..f5e352807 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md @@ -6,6 +6,6 @@ SPDX-License-Identifier: Apache-2.0 ### Changed - **Exit codes for the cases below** — scripts that branch on the exit code see these changes: - - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. + - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure that stops the agent, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A failure after the agent ran (a docker failure while the grader runs, or a host-side error in grading) is not a crash: its cell keeps `result.json` with a failed grade. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. - `dokimi eval judge agreement --strict` exits 3 (inconclusive) when an arm is NOT MEASURED or no arm can be compared, and 4 (invalid) when an arm is JUDGE-DEPENDENT; it exited 1 for both before. Without `--strict` the exit code is unchanged (0), and a judge that scored none of its answers still exits 1. - `dokimi trace inspect`, `dokimi trace consistency` exit 2 (bad input) when no trace can be read; they exited 1 before. With `--json` they print the `input` error document (`error.code` `input`, `data` null) instead of a document with `status` `failed`. A partial read still exits 1. diff --git a/dokimi/docs/quickstart.md b/dokimi/docs/quickstart.md index 0afd8d969..7f85a5176 100644 --- a/dokimi/docs/quickstart.md +++ b/dokimi/docs/quickstart.md @@ -261,7 +261,7 @@ failed check. | 2 | A bad invocation: a missing, unreadable or malformed input file, a device given as one (` : not a file`), a flag combination the command refuses, an output file that cannot be written, a missing provider credential, a source string that is malformed, unpinned or not a repository root. | `baseline save` on a bad eval file, `evaluate --custom-questions` on an empty or non-JSON file (`--custom-questions : `), `benchmark preflight` with no golden set, `repo scan --manifest ../x.yaml` | | 3 | Inconclusive: the run stopped short because a budget ran out (`--max-cost-usd`, or a call with no price) and kept the partial output (`tools preference` today; other commands still exit 1 on a budget stop), or the result is undecided (`eval judge validate`: a pass whose interval straddles the gate, or a variant no pair was built for, with `--strict` making it 1; `eval leak`: an item that could not be checked, or none could; `eval redteam`: a graded question not measured, or nothing graded; `eval judge agreement --strict`: an arm NOT MEASURED, or no arm to compare; `repo security --fail-on`: an unfinished cell or an arm with no measured cell, when no arm failed and every model has a baseline cell). | `tools preference --max-cost-usd 5` with calls not run; `eval judge validate` with too few bases | | 4 | Invalid: the result is not comparable or not trustworthy: a model that differs from the baseline's, or one that cannot be verified, under `baseline compare --strict` or the regression gate (`--max-context-drop` or `--max-lift-drop`); under `repo security --fail-on`, a model with no baseline cell when no arm failed; under `eval judge agreement --strict`, an arm JUDGE-DEPENDENT. | `baseline compare --strict` over a baseline scored with another judge | -| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | +| 5 | Infrastructure failure: a crash (an uncaught exception) in any command, with its traceback on stderr. For a command that emits the cli_output document, `--json` prints its `infra` document on stdout first. Also a `repo run` cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure that stops the agent, a scratch directory that vanished): the batch keeps the other cells and exits 5. | `backends --json` when `detect` raises; `repo run` with the docker daemon down | A command that validates its input and then runs exits 2 for the first and 1 for the second. Not every command follows this yet: `repo run` and From cfcf8c23fc440ea9968f0046998f05c092401c19 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 13:42:58 -0800 Subject: [PATCH 11/19] test: move the crash-trace tests out of test_repo_run.py test_repo_run.py went over the 1000-line cap of the growth ratchet with no baseline entry. The two tests for a crash after a trace line, and their stub harness helper, move unchanged to test_repo_run_exit.py. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_run.py | 29 --------------- dokimi/tests/test_repo_run_exit.py | 60 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 29 deletions(-) create mode 100644 dokimi/tests/test_repo_run_exit.py diff --git a/dokimi/tests/test_repo_run.py b/dokimi/tests/test_repo_run.py index d2765bead..7ad39a182 100644 --- a/dokimi/tests/test_repo_run.py +++ b/dokimi/tests/test_repo_run.py @@ -132,35 +132,6 @@ def test_an_agent_that_left_no_trace_is_not_given_a_process_metrics_error(setup, assert not any("process metrics failed" in e for e in rec["agent"]["trace_errors"]) -def _stub_harness(tmp_path, body): - """A harness that answers --version, then runs ``body`` (the fixture binary is pinned by a golden digest).""" - exe = tmp_path / "stub-agent" - exe.write_text('#!/usr/bin/env python3\nimport json, sys\n' - 'if sys.argv[1:] == ["--version"]:\n print("1.2.3 (Stub Agent)")\n sys.exit(0)\n' + body, - encoding="utf-8") - exe.chmod(0o755) - return exe - - -def test_a_harness_that_crashes_after_its_start_record_has_a_trace_and_is_not_a_crash(setup, tmp_path): - # The start-up record is a trace line: the agent started, so the cell keeps its result (no exit 5). - body = 'print(json.dumps({"type": "system", "subtype": "init", "model": "m-1"}))\nsys.exit(1)\n' - _run({**setup, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", "--task", "fix_add") - assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() - - -def test_a_text_harness_whose_stdout_is_its_trace_is_not_a_crash(setup, tmp_path): - # trace: text has no events, so its stdout is the only trace line: a crash after an answer keeps the cell. - prof = yaml.safe_load(setup["profile"].read_text(encoding="utf-8")) - prof["trace"] = "text" - text = tmp_path / "text-profile.yaml" - text.write_text(yaml.safe_dump(prof), encoding="utf-8") - body = 'print("The answer is 4.")\nsys.exit(1)\n' - _run({**setup, "profile": text, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", - "--task", "fix_add") - assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() - - def test_a_process_metrics_error_leaves_process_null_and_keeps_the_cost_and_the_grade(setup, monkeypatch): # The metric is best-effort: a counter that raises must not fail the task or drop the cost the harness reported. def refuse(*args, **kwargs): diff --git a/dokimi/tests/test_repo_run_exit.py b/dokimi/tests/test_repo_run_exit.py new file mode 100644 index 000000000..a8c3f11f1 --- /dev/null +++ b/dokimi/tests/test_repo_run_exit.py @@ -0,0 +1,60 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Exit-code classification of a repo run: a harness that crashes after its first trace line keeps its cell.""" + +from __future__ import annotations + +import pytest +import yaml + +from dokimi.cli import build_parser +from repo_eval_fixture import FakeRunner, fake_harness, make_repo + + +@pytest.fixture +def setup(tmp_path, monkeypatch): + root, _ = make_repo(tmp_path) + exe, profile = fake_harness(tmp_path) + runner = FakeRunner() + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: runner) + monkeypatch.setenv("SECRET_NOT_ALLOWED", "x") + monkeypatch.setenv("FAKE_ACTION", "fix") + return {"root": root, "exe": exe, "profile": profile, "runner": runner, "out": tmp_path / "runs"} + + +def _run(s, *extra, network=True, source=None): + argv = ["repo", "run", source or f"{s['root']}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(s["profile"]), "--bin", str(s["exe"]), "--model", "m-1", "--batch", "b1", "--out", str(s["out"]), + "--cache-root", str(s["out"] / "caches"), *(["--agent-network", "host"] if network else []), *extra] + args = build_parser().parse_args(argv) + args.func(args) + + +def _stub_harness(tmp_path, body): + """A harness that answers --version, then runs ``body`` (the fixture binary is pinned by a golden digest).""" + exe = tmp_path / "stub-agent" + exe.write_text('#!/usr/bin/env python3\nimport json, sys\n' + 'if sys.argv[1:] == ["--version"]:\n print("1.2.3 (Stub Agent)")\n sys.exit(0)\n' + body, + encoding="utf-8") + exe.chmod(0o755) + return exe + + +def test_a_harness_that_crashes_after_its_start_record_has_a_trace_and_is_not_a_crash(setup, tmp_path): + # The start-up record is a trace line: the agent started, so the cell keeps its result (no exit 5). + body = 'print(json.dumps({"type": "system", "subtype": "init", "model": "m-1"}))\nsys.exit(1)\n' + _run({**setup, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", "--task", "fix_add") + assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() + + +def test_a_text_harness_whose_stdout_is_its_trace_is_not_a_crash(setup, tmp_path): + # trace: text has no events, so its stdout is the only trace line: a crash after an answer keeps the cell. + prof = yaml.safe_load(setup["profile"].read_text(encoding="utf-8")) + prof["trace"] = "text" + text = tmp_path / "text-profile.yaml" + text.write_text(yaml.safe_dump(prof), encoding="utf-8") + body = 'print("The answer is 4.")\nsys.exit(1)\n' + _run({**setup, "profile": text, "exe": _stub_harness(tmp_path, body)}, "--arm", "guidance:raw", + "--task", "fix_add") + assert (setup["out"] / "b1/fix_add__guidance-raw__m-1__r1/result.json").is_file() From 36c6bb49cd5c10f2bfa108b71d716e71ad839dce Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 14:54:08 -0800 Subject: [PATCH 12/19] Address review: the newer docker CLI's daemon diagnostic is a docker error Docker CLIs since 2025 say 'failed to connect to the docker API' when the socket is missing. That line matched none of the daemon diagnostics, so a daemon outage on inspect or build was not infra (exit 5). Thread: PRRT_kwDOVBg7g864Io18 Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/container.py | 1 + dokimi/tests/test_exit_codes_infra.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/dokimi/dokimi/repos/container.py b/dokimi/dokimi/repos/container.py index ef663905f..1a7c42593 100644 --- a/dokimi/dokimi/repos/container.py +++ b/dokimi/dokimi/repos/container.py @@ -198,6 +198,7 @@ def image_exists(self, tag: str) -> bool: _DAEMON_DIAGNOSTIC = re.compile(r"^(?:(?:error|docker):\s+)?(?:got\s+)?(?:cannot connect to the docker daemon|" + r"failed to connect to the docker api|" r"error during connect|permission denied while trying to connect to the docker daemon)") diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 101657ba5..8cf991eed 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -72,6 +72,27 @@ def test_an_older_cli_got_permission_denied_on_the_socket_is_a_docker_error(tmp_ DockerRunner(runner=fake).build(tmp_path, tmp_path / "Dockerfile", "t") +NEWER_CLI_DAEMON_DOWN = ( + "failed to connect to the docker API at unix:///var/run/docker.sock. Check if the path is correct and if the " + "daemon is running: dial unix /var/run/docker.sock: connect: no such file or directory\n" +) + + +def test_a_newer_cli_daemon_diagnostic_is_a_docker_error_on_inspect_and_on_build(tmp_path): + """Docker CLIs since 2025 say `failed to connect to the docker API`: that is the daemon down (exit 5), not a + missing image and not a failed build (exit 1).""" + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + (tmp_path / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + inspect = process.FakeRunner(lambda call: process.reply(stderr=NEWER_CLI_DAEMON_DOWN, returncode=1)) + with pytest.raises(DockerError, match="failed to connect to the docker API"): + DockerRunner(runner=inspect).image_exists("img:t") + build = process.FakeRunner(lambda call: process.reply(stderr=NEWER_CLI_DAEMON_DOWN, returncode=1)) + with pytest.raises(DockerError): + DockerRunner(runner=build).build(tmp_path, tmp_path / "Dockerfile", "t") + + BUILDKIT_QUOTED_MESSAGE = ( "#5 [3/4] RUN echo 'Cannot connect to the Docker daemon' && false\n" "#5 0.1 Cannot connect to the Docker daemon\n" From 31f6874365865d153c94191d9bcacccd4de0f2b2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 14:56:24 -0800 Subject: [PATCH 13/19] Address review: the CI notes on test_repo_run.py are already fixed The growth-ratchet failures came from the test file being over its cap; the move in cfcf8c23f brought it back to 1000 lines and the CI checks pass. Thread: PRRT_kwDOVBg7g864JkFy: already fixed in cfcf8c23f Thread: PRRT_kwDOVBg7g864JkF6: already fixed in cfcf8c23f Thread: PRRT_kwDOVBg7g864JkF_: already fixed in cfcf8c23f Thread: PRRT_kwDOVBg7g864JkGE: already fixed in cfcf8c23f Thread: PRRT_kwDOVBg7g864JkGQ: already fixed in cfcf8c23f Thread: PRRT_kwDOVBg7g864JkGS: already fixed in cfcf8c23f Signed-off-by: Nikolay Petrov From 5331a4f118148cc7531ccff1e73cc30fdb89bc83 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:02:37 -0800 Subject: [PATCH 14/19] Address review: match docker 29 permission-denied on the API Thread: PRRT_kwDOVBg7g864Y1XL Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/container.py | 2 +- dokimi/tests/test_exit_codes_infra.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/container.py b/dokimi/dokimi/repos/container.py index 1a7c42593..bda5fb95f 100644 --- a/dokimi/dokimi/repos/container.py +++ b/dokimi/dokimi/repos/container.py @@ -199,7 +199,7 @@ def image_exists(self, tag: str) -> bool: _DAEMON_DIAGNOSTIC = re.compile(r"^(?:(?:error|docker):\s+)?(?:got\s+)?(?:cannot connect to the docker daemon|" r"failed to connect to the docker api|" - r"error during connect|permission denied while trying to connect to the docker daemon)") + r"error during connect|permission denied while trying to connect to the docker (?:daemon|api))") def _daemon_unreachable(stderr: str | None) -> bool: diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 8cf991eed..8f7c87dfc 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -186,3 +186,16 @@ def test_strict_judge_dependent_is_checked_before_not_measured(capsys): judge_reliability._strict_cross_judge_exit(arms, "base") assert exc.value.code == 4 assert "judge-dependent against `base`: b" in capsys.readouterr().err + + +def test_docker_29_permission_denied_on_inspect_is_a_docker_error(): + """Docker 29 CLI opens a socket it may not use with `permission denied while trying to connect to the docker API`: + on `image inspect` that is no permission, not a missing image (exit 5, not 1).""" + from dokimi import process + from dokimi.repos.container import DockerError, DockerRunner + + denied = ("permission denied while trying to connect to the docker API at unix:///var/run/docker.sock: " + "Get \"http://%2Fvar%2Frun%2Fdocker.sock/v1.47/images/img:t/json\": permission denied\n") + fake = process.FakeRunner(lambda call: process.reply(stderr=denied, returncode=1)) + with pytest.raises(DockerError, match="permission denied while trying to connect to the docker API"): + DockerRunner(runner=fake).image_exists("img:t") From d05536181f885f94c70652b8ba58b739ead153d4 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:03:10 -0800 Subject: [PATCH 15/19] Address review: --strict names the NOT MEASURED arms too Thread: PRRT_kwDOVBg7g864Y1Xd Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/judge_reliability.py | 9 ++++++--- dokimi/tests/test_exit_codes_infra.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 77ca8b63a..f790ec740 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -446,16 +446,19 @@ def _scored_nothing_exit(unscored: Sequence[Measure]) -> None: def _strict_cross_judge_exit(arms: dict[str, dict[str, Any]], baseline_arm: str) -> None: """Exit 4 (invalid) when an arm is JUDGE-DEPENDENT, else 3 (inconclusive) when no arm was compared or an arm is - NOT MEASURED (issue #94 §2.6). The invalid verdict is checked first, as ``baseline compare`` does.""" + NOT MEASURED (issue #94 §2.6). Both lines print before the exit, so stderr names every arm the verdict counts.""" dependent = sorted(a for a, r in arms.items() if r["label"] == "JUDGE-DEPENDENT") if dependent: print(f"✗ judge-dependent against `{baseline_arm}`: {', '.join(dependent)}", file=sys.stderr) - sys.exit(EXIT_INVALID) unmeasured = sorted(a for a, r in arms.items() if r["label"] == "NOT MEASURED") - if not arms or unmeasured: + inconclusive = not arms or bool(unmeasured) + if inconclusive: why = f"{len(unmeasured)} of {len(arms)} arm(s) NOT MEASURED: {', '.join(unmeasured)}" if arms else \ "no arm has a scored question to compare" print(f"✗ incomplete against `{baseline_arm}`: {why}", file=sys.stderr) + if dependent: # invalid (4) wins over inconclusive (3), as baseline compare does + sys.exit(EXIT_INVALID) + if inconclusive: sys.exit(EXIT_INCONCLUSIVE) diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 8f7c87dfc..4a52bb0a7 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -199,3 +199,16 @@ def test_docker_29_permission_denied_on_inspect_is_a_docker_error(): fake = process.FakeRunner(lambda call: process.reply(stderr=denied, returncode=1)) with pytest.raises(DockerError, match="permission denied while trying to connect to the docker API"): DockerRunner(runner=fake).image_exists("img:t") + + +def test_strict_names_the_not_measured_arms_when_another_arm_is_judge_dependent(capsys): + """The verdict is 4, but stderr still names the NOT MEASURED arms: both lines print before the exit.""" + from dokimi.commands import judge_reliability + + arms = {"a": {"label": "NOT MEASURED"}, "b": {"label": "JUDGE-DEPENDENT"}} + with pytest.raises(SystemExit) as exc: + judge_reliability._strict_cross_judge_exit(arms, "base") + assert exc.value.code == 4 + err = capsys.readouterr().err + assert "judge-dependent against `base`: b" in err + assert "incomplete against `base`: 1 of 2 arm(s) NOT MEASURED: a" in err From 3325ac9c3672cd079dcc5ab3521a63c928a27e96 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:03:33 -0800 Subject: [PATCH 16/19] Address review: the docs name the crash and the --strict exit codes Thread: PRRT_kwDOVBg7g864Y1XW Thread: PRRT_kwDOVBg7g864Y1Xa Signed-off-by: Nikolay Petrov --- dokimi/docs/behavioural-security.md | 4 ++-- dokimi/docs/benchmarking-and-comparison.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dokimi/docs/behavioural-security.md b/dokimi/docs/behavioural-security.md index 9208a7c2f..1e15e5527 100644 --- a/dokimi/docs/behavioural-security.md +++ b/dokimi/docs/behavioural-security.md @@ -60,8 +60,8 @@ of cell that are in no rate: null `process` in `result.json`, no finding; an empty `events.jsonl` is what the runner writes when the agent never started, even though its `rc` is 0 or the cap killed it, unless the runner also recorded `trace_errors`, see `unfinished`). A nonzero `rc` with no trace, unless the cap - killed the harness (`timed_out`), is not a cell at all: it is a crash (exit 5) that writes no `result.json`. Counting it as clean would make an arm - whose skill crashes the agent look safer. Each arm entry gives its `not_started` count; + killed the harness (`timed_out`), is a crash (exit 5) with no `result.json`, listed under `unfinished`. Counting it as + clean would make an arm whose skill crashes the agent look safer. Each arm entry gives its `not_started` count; its `cells` (the denominator of every rate) are the measured ones. An arm, the baseline included, whose every cell is `not_started` has no rate at all: the report shows it as `no measured cell`, and `--fail-on` (`fail` or `warn`) exits 3 on it (`ARM: no measured cell`), diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 8463794a8..e8aed4f82 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -615,7 +615,7 @@ under several judges (absolute, and with `--grounded` against each question's measures, and labels each arm `CONSISTENT ±`, `LEANING ±`, `JUDGE-DEPENDENT` or `NO DETECTABLE EFFECT`, on Holm-adjusted p over the measures that vote, or `NOT MEASURED` when no voting measure compared a single question (every call -errored, or every judge was gated out); `--strict` exits 1 on that too. A +errored, or every judge was gated out); `--strict` exits 3 on that, and 4 on a `JUDGE-DEPENDENT` arm. A `--judges` judge that leaves answers unscored prints how many and its first error, which the report keeps under `unscored`; one that scored none of its answers exits 1 once the report is written, with or without `--strict`, as From 21e30b4f1b1a503f6eb23fd2674b3568725b0749 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:07:01 -0800 Subject: [PATCH 17/19] Tidy after the self-review: a stale comment, an always-true branch, one strict test The cell comment names the CellCrash exception; trace inspect uses else where rows is never empty; the strict test checks both lines it now prints. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/context.py | 2 +- dokimi/dokimi/repos/runner.py | 3 ++- dokimi/tests/test_exit_codes_infra.py | 16 +++------------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/dokimi/dokimi/commands/context.py b/dokimi/dokimi/commands/context.py index 4c62c0a6e..18ea151bb 100644 --- a/dokimi/dokimi/commands/context.py +++ b/dokimi/dokimi/commands/context.py @@ -45,7 +45,7 @@ def cmd_context_trace(args: argparse.Namespace) -> None: reason = ("one or more traces could not be read (see stderr)" if failed else "lines of a trace were skipped (see stderr)" if skipped else None) cli_output.emit_leaf("trace inspect", rows, status=status, reason=reason) - elif rows: + else: _print_text(rows, args) if failed: sys.exit(1) diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index d04f95152..3118ad6b8 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -248,7 +248,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, result = ctx.runner.run(spec, stdout=rd / "agent.stdout", stderr=rd / "agent.stderr", timeout_s=caps["timeout_s"]) - # The agent has run and may have spent its cap: from here on the cell is graded, never dropped. + # The agent has run and may have spent its cap: from here on the cell is graded, never dropped, + # except a crash before its first trace line (CellCrash below). agent: dict[str, Any] = { "rc": result.returncode, "timed_out": result.timed_out, "wall_s": result.wall_s, "cost_usd": None, # A killed harness reports no cost; the batch budget counts the cell's cap instead of $0. diff --git a/dokimi/tests/test_exit_codes_infra.py b/dokimi/tests/test_exit_codes_infra.py index 4a52bb0a7..b5e500008 100644 --- a/dokimi/tests/test_exit_codes_infra.py +++ b/dokimi/tests/test_exit_codes_infra.py @@ -185,7 +185,9 @@ def test_strict_judge_dependent_is_checked_before_not_measured(capsys): with pytest.raises(SystemExit) as exc: judge_reliability._strict_cross_judge_exit(arms, "base") assert exc.value.code == 4 - assert "judge-dependent against `base`: b" in capsys.readouterr().err + err = capsys.readouterr().err + assert "judge-dependent against `base`: b" in err + assert "incomplete against `base`: 1 of 2 arm(s) NOT MEASURED: a" in err # both lines print before the exit def test_docker_29_permission_denied_on_inspect_is_a_docker_error(): @@ -200,15 +202,3 @@ def test_docker_29_permission_denied_on_inspect_is_a_docker_error(): with pytest.raises(DockerError, match="permission denied while trying to connect to the docker API"): DockerRunner(runner=fake).image_exists("img:t") - -def test_strict_names_the_not_measured_arms_when_another_arm_is_judge_dependent(capsys): - """The verdict is 4, but stderr still names the NOT MEASURED arms: both lines print before the exit.""" - from dokimi.commands import judge_reliability - - arms = {"a": {"label": "NOT MEASURED"}, "b": {"label": "JUDGE-DEPENDENT"}} - with pytest.raises(SystemExit) as exc: - judge_reliability._strict_cross_judge_exit(arms, "base") - assert exc.value.code == 4 - err = capsys.readouterr().err - assert "judge-dependent against `base`: b" in err - assert "incomplete against `base`: 1 of 2 arm(s) NOT MEASURED: a" in err From 8f61f29bb5ec3f888a8af8377e3c883bbc6003b2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:07:26 -0800 Subject: [PATCH 18/19] Address review: the items of the review at 31f6874 are fixed in the commits above Review: PRR_kwDOVBg7g88AAAABYd-Saw Signed-off-by: Nikolay Petrov From 8d745aa1dba323fbd34649c7b94319de7b261aef Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:28:32 -0800 Subject: [PATCH 19/19] docs(repo): a crash before the first trace line charges its cap; pin the charge per rerun Thread: PRRT_kwDOVBg7g864Y1XQ Signed-off-by: Nikolay Petrov --- ...261115_120000_napetrov_fix_cli-a2b5-exits.md | 1 + dokimi/tests/test_repo_run_unreadable_result.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md index f5e352807..929266606 100644 --- a/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md +++ b/dokimi/changelog.d/20261115_120000_napetrov_fix_cli-a2b5-exits.md @@ -9,3 +9,4 @@ SPDX-License-Identifier: Apache-2.0 - `dokimi repo run`: a cell that crashes (a harness that exits nonzero before its first trace line, unless the cap killed it, a docker daemon or CLI failure that stops the agent, a scratch directory that vanished, a result that cannot be written) exits 5 (infra failure); it exited 1 before, and a harness that exited nonzero with no trace exited 0 (its cell was recorded as a fail). Such a cell writes no `result.json`, so the next run runs it again. A failure after the agent ran (a docker failure while the grader runs, or a host-side error in grading) is not a crash: its cell keeps `result.json` with a failed grade. A cell refused on its inputs (for example a malformed `.mcp.json`), a budget stop or a failed host probe still exits 1. - `dokimi eval judge agreement --strict` exits 3 (inconclusive) when an arm is NOT MEASURED or no arm can be compared, and 4 (invalid) when an arm is JUDGE-DEPENDENT; it exited 1 for both before. Without `--strict` the exit code is unchanged (0), and a judge that scored none of its answers still exits 1. - `dokimi trace inspect`, `dokimi trace consistency` exit 2 (bad input) when no trace can be read; they exited 1 before. With `--json` they print the `input` error document (`error.code` `input`, `data` null) instead of a document with `status` `failed`. A partial read still exits 1. +- **A crash before the first trace line charges its full cap to the batch budget** — its spend is unknown, so `repo run` charges the cell's `budget_usd` to `_eval/spend.json` (as a killed harness does), and each rerun charges it again. A batch with `--max-batch-usd` counts the crashes toward that cap, so fix the harness before a rerun. diff --git a/dokimi/tests/test_repo_run_unreadable_result.py b/dokimi/tests/test_repo_run_unreadable_result.py index 766ddc007..e9414243c 100644 --- a/dokimi/tests/test_repo_run_unreadable_result.py +++ b/dokimi/tests/test_repo_run_unreadable_result.py @@ -135,6 +135,23 @@ def test_a_harness_that_exits_nonzero_before_any_trace_is_a_crash_exit_5(setup, assert len([s for s in setup["runner"].specs if "-p" in s.argv]) == 3 +def test_a_crash_before_the_first_trace_line_charges_its_cap_again_on_every_rerun(setup, monkeypatch): + """Its spend is unknown, so the cell is charged its cap, as for a killed harness; each rerun charges it again.""" + monkeypatch.setenv("FAKE_ACTION", "silent") + + def spent() -> float: + return json.loads((setup["out"] / "b1" / "_eval" / "spend.json").read_text(encoding="utf-8"))["spent_usd"] + + with pytest.raises(SystemExit): + _run(setup) + first = spent() + assert first == pytest.approx(3 * 2.0) # three crashed reps, each charged the default cap of $2.0 + setup["runner"].specs.clear() + with pytest.raises(SystemExit): + _run(setup) + assert spent() == pytest.approx(2 * first) + + def test_a_nonzero_timed_out_harness_with_no_trace_is_a_fail_cell_not_a_crash(setup, monkeypatch): """The cap ran out before the agent wrote anything: a verdict on the cell (a fail), not a crash of the harness.""" from dokimi.repos.container import ContainerResult