From eb893066c0706fe48f0fe19302b61bde64383f50 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 14 Nov 2026 22:07:43 -0800 Subject: [PATCH 01/14] feat(repo): suite manifest for tasks whose base is in another repository A suite_manifest.v1 file in the eval tree lists tasks whose base is a commit of another repository (a shared scaffold), each repository referenced as @. `repo run --suite` resolves each listed task's base in the repository it names. A repository that cannot be opened, or a base that is not one of its commits, leaves those cells unsupported (unsupported.json), never failed or missing. Revisions must be full SHAs; a branch, tag or ref needs --allow-floating-suite-revs and is pinned in the batch (key `suite`). Tests use local git repositories only. Contract section 2.1. Signed-off-by: Nikolay Petrov --- ...20261114_120000_napetrov_suite-manifest.md | 8 + dokimi/data/suites/example-suite.yaml | 12 ++ dokimi/docs/reference/cli-interface.json | 4 + dokimi/docs/repo-evaluation-contract.md | 45 +++- dokimi/dokimi/cli_flags.py | 7 + dokimi/dokimi/commands/repo_run.py | 58 +++++- dokimi/dokimi/repos/grading.py | 15 +- dokimi/dokimi/repos/runner.py | 7 +- dokimi/dokimi/repos/selfcheck.py | 12 +- dokimi/dokimi/repos/suite.py | 146 +++++++++++++ dokimi/dokimi/schemas/suite_manifest.v1.json | 32 +++ .../fixtures/artifacts/suite_manifest.v1.json | 12 ++ dokimi/tests/test_repo_suite.py | 192 ++++++++++++++++++ dokimi/tests/test_repo_typecheck_grader.py | 4 +- 14 files changed, 533 insertions(+), 21 deletions(-) create mode 100644 dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md create mode 100644 dokimi/data/suites/example-suite.yaml create mode 100644 dokimi/dokimi/repos/suite.py create mode 100644 dokimi/dokimi/schemas/suite_manifest.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/suite_manifest.v1.json create mode 100644 dokimi/tests/test_repo_suite.py diff --git a/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md new file mode 100644 index 000000000..860462a8c --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md @@ -0,0 +1,8 @@ + + +### Added + +- **Suite manifests for tasks whose base is in another repository** — `suite_manifest.v1` lists eval-tree tasks whose base is a commit of a scaffold repository, referenced as `@`. `repo run --suite ` resolves each base there; a repository that cannot be opened leaves its cells unsupported, not failed. Floating refs need `--allow-floating-suite-revs`. See `docs/repo-evaluation-contract.md` §2.1. diff --git a/dokimi/data/suites/example-suite.yaml b/dokimi/data/suites/example-suite.yaml new file mode 100644 index 000000000..6ebd9b6ae --- /dev/null +++ b/dokimi/data/suites/example-suite.yaml @@ -0,0 +1,12 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Example suite_manifest.v1 (contract §2.1): `repos` pins each scaffold, `tasks` names the tasks whose base is in one. +schema: suite_manifest.v1 +id: linreg-cpu-solver +repos: + sklearn-scaffold: https://github.com/example-org/sklearn-scaffold@9f8e7d6c5b4a39281706f5e4d3c2b1a098765432 + numpy-scaffold: https://github.com/example-org/numpy-scaffold@0a1b2c3d4e5f60718293a4b5c6d7e8f901234567 +tasks: + - {id: solver-pick-cholesky, repo: sklearn-scaffold} + - {id: solver-pick-lstsq, repo: numpy-scaffold} diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a87a843cc..b4e60ebcb 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -563,6 +563,7 @@ "allow_abbrev": true, "params": [ {"key": "source", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, + {"key": "--allow-floating-suite-revs", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--allow-unsupported", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--arm", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--build-network", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -580,6 +581,7 @@ {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-check", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--suite", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--task", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--warm", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null} ], @@ -1371,6 +1373,7 @@ "params": [ {"key": "source", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, {"key": "--agent-network", "action": "_StoreAction", "choices": ["'bridge'", "'host'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--allow-floating-suite-revs", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--arm", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--batch", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--bin", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -1397,6 +1400,7 @@ {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--suite", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--task", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--timeout", "action": "_StoreAction", "choices": null, "const": null, "default": 2700, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--tools", "action": "_StoreAction", "choices": null, "const": null, "default": "all", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 5c1c0de52..a349fa536 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -20,8 +20,8 @@ SPDX-License-Identifier: Apache-2.0 > Implemented so far (slice S1): the `repo_manifest.v1` schema, validation of > the manifest and of every `task.json` against one repository snapshot > (with git history, every task base must resolve and the guidance globs are -> evaluated at each base), the `repo_static.v1` schema (§4.4), and the `repo` -> subject kind: +> evaluated at each base), the `repo_static.v1` schema (§4.4), the suite +> manifest for bases in other repositories (§2.1), and the `repo` subject kind: > > ```bash > python cli.py repo validate --manifest [--json] [--out FILE] [--fail-on-warning] @@ -312,6 +312,47 @@ Rules: harness short-lived credentials scoped to the model endpoint. Every connection through the proxy is logged with the run. +### 2.1 Suite manifest (`suite_manifest.v1`) + +A task's `base` is normally a commit of the eval tree's source (§1.1). A task +whose base is in another repository, such as a scaffold that a benchmark shares +or a library at a fixed revision, is listed in a suite file kept in the eval +tree: + +```yaml +schema: suite_manifest.v1 +id: linreg-cpu-solver +repos: + sklearn-scaffold: https://github.com/example/scikit-learn-scaffold@<40-hex sha> + numpy-scaffold: /path/to/numpy-scaffold@<40-hex sha> +tasks: + - id: solver-pick-cholesky + repo: sklearn-scaffold +``` + +- `repo run --suite ` reads the file from the eval tree; the path is + relative to its root and may not leave it. Each listed task takes its base + checkout from the repository it names; its `task.json` keeps the rest of the + task. A task the suite does not list keeps its eval-tree base. Every listed + task must be a task of `tasks` (§2), or the run stops before any cell. +- Each repository is `@` or a local `@`, as in §1.1. The + revision must be a full commit SHA. A branch, tag or other ref is refused + unless the run passes `--allow-floating-suite-revs`; it is then resolved once + when the run starts, and the batch pin (`_eval/batch.json`, key `suite`) + records the commits, so a resumed batch reuses them. +- A repository that cannot be opened, or a base that is not a commit of it, + leaves that task's cells unsupported (§9): `unsupported.json` gives the reason, + and `repo report` counts the cells neither as failed nor as missing. A cell + that runs records `base_repo` (`{"repo", "source"}`) in `meta.json`. +- `repo tasks check` reports these tasks as not checked when their repository is + missing and fails only on the other bad tasks. +- Not covered yet: `repo validate` checks the bases of the eval tree only; a suite + task's `{source_dir}` is still the eval-tree source; and a `guidance:rev:` + arm is pinned against the eval-tree source. + +The schema is `dokimi/schemas/suite_manifest.v1.json`; an example is +`data/suites/example-suite.yaml`. + ## 3. `task.json` The framework interprets the **core** and **workspace** keys. Every other key diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index ab3985679..55f1cc9ea 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -225,6 +225,13 @@ class Concept: canonical=(Flag("--network", "[PHASE=]MODE[:hosts]", "text"),), banned=("--agent-network", "--build-network"), ), + Concept( + name="suite", + help="a suite manifest: tasks whose base is in another repository (contract §2.1)", + canonical=(Flag("--suite", "PATH", "text", meaning="the suite file, in the eval tree"), + Flag("--allow-floating-suite-revs", kind="flag", + meaning="resolve a branch or tag of a suite repository when the run starts")), + ), Concept( name="online", help="network checks of a catalog's skills, opt in", diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 3f9368856..02469ef32 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -14,7 +14,7 @@ import shutil import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from dokimi.repos.manifest import RepoManifest @@ -40,12 +40,19 @@ def _refuse_typed(source, args: argparse.Namespace) -> None: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) if args.manifest and not args.manifest_file: - if os.path.splitdrive(args.manifest)[0]: # `D:/x`, `D:x`, `//host/share/x`: no `/` check sees a drive - raise InputError(f"path escapes the repository: {args.manifest}") - try: # as the host reads it: on Windows `..\x` is `../x`, also for a pinned tree, which is read with `/` - tree_path(os.path.normpath(args.manifest).replace(os.path.sep, "/")) - except SourceArgumentError as exc: - raise InputError(f"path escapes the repository: {args.manifest}") from exc + _refuse_escape(args.manifest) + if getattr(args, "suite", None): + _refuse_escape(args.suite) + + +def _refuse_escape(path: str) -> None: + """Exit 2 for a path that leaves the tree, as the host reads it.""" + if os.path.splitdrive(path)[0]: # `D:/x`, `D:x`, `//host/share/x`: no `/` check sees a drive + raise InputError(f"path escapes the repository: {path}") + try: # as the host reads it: on Windows `..\x` is `../x`, also for a pinned tree, which is read with `/` + tree_path(os.path.normpath(path).replace(os.path.sep, "/")) + except SourceArgumentError as exc: + raise InputError(f"path escapes the repository: {path}") from exc def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): @@ -79,6 +86,7 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): # a pinned batch keeps its content until the pin is checked staged = export_eval_dir(tree, manifest, content.with_name("content.next") if pin else content) _require_grade(staged, manifest, args.task) # before the pin and the image build + suite = _suite_bases(args, tree, staged, manifest, _cache_dir(args)) # its pins go in the batch pin runner = DockerRunner(args.docker) image, ctx_dir = args.image, None if not image: @@ -92,7 +100,8 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): elif not runner.image_exists(image): raise ValueError(f"--image {image} does not exist") if pin: - stamp = {"tree": tree.label, "eval_dir": tree_digest(staged), "image": image} + stamp: dict[str, Any] = {"tree": tree.label, "eval_dir": tree_digest(staged), "image": image} + stamp.update(_suite_pin(suite)) if _uses_builtin_graders(staged, manifest): # else no score depends on them: an upgrade keeps the batch stamp["graders"] = graders_digest() _pin_batch(out, stamp) @@ -117,10 +126,30 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): uses_source = any("{source_dir}" in v for v in manifest.entry_points.values()) source_dir = Path(args.source_dir).resolve() if args.source_dir else (git_dir if uses_source else None) ctx = EvalContext(manifest=manifest, git_dir=git_dir, content_dir=content, image=image, runner=runner, - caches=caches, source_dir=source_dir, entry_timeout_s=args.grade_timeout) + caches=caches, source_dir=source_dir, entry_timeout_s=args.grade_timeout, suite=suite) return ctx, tree +def _suite_bases(args: argparse.Namespace, tree, content: Path, manifest: RepoManifest, cache_dir: Path | None): + """The base checkouts of ``--suite`` (contract §2.1), or None: every base is then in the eval source.""" + if not args.suite: + return None + from dokimi.repos.suite import SuiteError, load_suite, resolve_suite + from dokimi.repos.workspace import load_tasks + + suite = load_suite(tree, args.suite, args.allow_floating_suite_revs) + found = {t.id: t for t in load_tasks(content, manifest)} + absent = sorted(set(suite.tasks) - set(found)) + if absent: + raise SuiteError(suite.path, [f"task {t} is not under {manifest.tasks}" for t in absent]) + return resolve_suite(suite, {t: str(found[t].raw.get("base", "")) for t in suite.tasks}, cache_dir) + + +def _suite_pin(suite) -> dict[str, Any]: + """The commit each suite repository resolved to, for the batch pin: a resume must reuse them.""" + return {} if suite is None else {"suite": suite.pins} + + def _require_grade(content: Path, manifest: RepoManifest, ids: list[str] | None) -> None: """Refuse the selected tasks when one needs ``entry_points.grade`` and the manifest has none.""" from dokimi.repos.manifest import ManifestError, TaskSummary, grade_problems @@ -216,11 +245,14 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: sys.exit(1) bad = [r for r in results if r["verdict"] != "ok"] print(f"{len(results) - len(bad)}/{len(results)} ok; wrote {out / 'check' / 'check.json'}") + unavailable = [r["task"] for r in bad if r.get("unsupported_by") == "suite"] + if unavailable: # the base's repository of --suite is missing: not checked, and not a failure + print(f"{len(unavailable)} not checked, the repository of their base is missing: " + ", ".join(unavailable)) allowed = [r for r in bad if args.allow_unsupported and r.get("unsupported_by") == "host"] if allowed: print(f"{len(allowed)} unsupported on this host, allowed by --allow-unsupported: " + ", ".join(r["task"] for r in allowed)) - if len(bad) > len(allowed): + if len(bad) > len(allowed) + len(unavailable): sys.exit(1) if len(bad) == len(results): # --allow-unsupported passed every task by: none was checked print(f"✗ incomplete: no task passed the self-check (0 ok, {len(allowed)} unsupported)", file=sys.stderr) @@ -459,6 +491,12 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit p.add_argument("source", help=source_help) p.add_argument("--manifest", default=".agent-eval/repo-eval.yaml", help="Manifest path relative to the repository root") + p.add_argument("--suite", default=None, + help="Suite manifest (suite_manifest.v1) path relative to the repository root: tasks whose " + "base is a commit of another repository, named as @ (contract §2.1)") + p.add_argument("--allow-floating-suite-revs", action="store_true", dest="allow_floating_suite_revs", + help="Let --suite name a branch or tag: it is resolved once, when the run starts, and the " + "batch pins the commit it resolved to") p.add_argument("--manifest-file", default=None, dest="manifest_file", help="Read the manifest from this local file instead of from the tree") p.add_argument("--task", action="append", default=None, help="Only this task id; repeatable (default: all)") diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py index 1067d96a1..9b6111da4 100644 --- a/dokimi/dokimi/repos/grading.py +++ b/dokimi/dokimi/repos/grading.py @@ -30,7 +30,7 @@ import stat from dataclasses import dataclass, field from pathlib import Path, PurePosixPath -from typing import Any +from typing import TYPE_CHECKING, Any from dokimi.artifacts import schema_errors from dokimi.measure import SCORE_BOUND, as_float, is_finite @@ -74,6 +74,10 @@ _PYTHON3_NOT_FOUND_SIG = "exec python3 failed" # tini's message under --init, which every container runs with +if TYPE_CHECKING: + from .suite import SuiteBases + + @dataclass class EvalContext: """What every container of one evaluation needs: the trees, the image and the caches.""" @@ -86,6 +90,15 @@ class EvalContext: caches: dict[str, Path] = field(default_factory=dict) # cache name -> host directory source_dir: Path | None = None # host side of {source_dir} entry_timeout_s: float = 3600 + suite: SuiteBases | None = None # base checkouts of a suite run (suite.py); None: every base is in git_dir + + def git_dir_for(self, task_id: str) -> Path: + """The git dir a task's base is checked out from: its suite repository, else ``git_dir``.""" + return self.suite.git_dirs.get(task_id, self.git_dir) if self.suite else self.git_dir + + def base_gaps(self) -> dict[str, str]: + """Why each suite task has no base checkout; its cells are unsupported, not run.""" + return dict(self.suite.gaps) if self.suite else {} def cache_mounts(self) -> list[Mount]: image = self.manifest.image diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a5809f29a..b65b6de09 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -218,9 +218,11 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, if rd.exists(): _rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) + meta = prepare_workspace(ctx.git_dir_for(task.id), ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) + if ctx.suite is not None and task.id in ctx.suite.meta: # additive: only a suite task's base is elsewhere + meta["base_repo"] = ctx.suite.meta[task.id] present = _workspace_args(task, opts, rd, meta) _write_json(rd / "meta.json", meta) skills: dict[str, Any] | None = None @@ -375,7 +377,8 @@ def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: lis check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells} if force else None) host = probe_host(ctx, batch_dir / "_eval") _write_json(batch_dir / "_eval" / "host.json", host) - verdicts = {t.id: unmet(t, host) for t in tasks} + gaps = ctx.base_gaps() # a task whose base repository is missing is unsupported, as a host miss is + verdicts = {t.id: (gaps[t.id], "suite") if t.id in gaps else unmet(t, host) for t in tasks} unsupported = {task_id: why for task_id, (why, _) in verdicts.items() if why} if force and unsupported: # --force keeps their finished cells, so those must match this run too check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index f4b10cdaf..2f99f6074 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -63,9 +63,9 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, guidance_names: tuple[str, ...] = ANCESTOR_NAMES, host: dict[str, Any] | None = None) -> dict[str, Any]: """Null, oracle and isolation checks for one task; ``verdict`` is ok, broken, flaky or unsupported. - An unsupported result says why in ``unsupported_by``: ``egress``, ``host`` (a hardware miss, which - ``--allow-unsupported`` may pass) or ``host-probe`` (the probe could not run or gave no CPU count, - which it never passes). + An unsupported result says why in ``unsupported_by``: ``egress``, ``suite`` (the base's repository + of a suite is missing: not a failure), ``host`` (a hardware miss, which ``--allow-unsupported`` may + pass) or ``host-probe`` (the probe could not run or gave no CPU count, which it never passes). A task the host cannot show still has its workspace prepared and isolation probed, and its oracle run when only CPUs are short; only the null check, which depends on the hardware, is skipped. ``--warm`` @@ -80,6 +80,10 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, except ContainerError as exc: res.update(verdict="unsupported", unsupported_by="egress", problems=[str(exc)]) return res + gap = ctx.base_gaps().get(task.id) # its base is in a suite repository that is missing + if gap: + res.update(verdict="unsupported", unsupported_by="suite", problems=[gap]) + return res # The probe sees what every container of this runner sees (affinity, cgroup quota), with or without # --warm's network: an unmet host requirement is unsupported either way, never a passing null run. reason, unsupported_by, facts = _host_reason(ctx, task, work_dir, host) @@ -167,7 +171,7 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: if rd.exists(): shutil.rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, "guidance:raw", rd) + meta = prepare_workspace(ctx.git_dir_for(task.id), ctx.manifest, task, "guidance:raw", rd) meta.update(harness="none", harness_version=None, model="none", rep=0) for name, data in (("meta.json", meta), ("task.json", task.raw)): (rd / name).write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8") diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py new file mode 100644 index 000000000..e88ba0382 --- /dev/null +++ b/dokimi/dokimi/repos/suite.py @@ -0,0 +1,146 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Suite manifests (``suite_manifest.v1``, contract §2.1): tasks whose base is in another repository. + +A suite names the scaffold repositories, each as ``@``, and the tasks of the eval tree that +take their base checkout from one of them. The task itself (prompt, grader, ``task.json``) stays in the +eval tree; only its ``base`` is looked up in the named repository. A repository that cannot be opened, +or a base that is not a commit of it, leaves those tasks' cells unsupported: not run, not failed. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +from dokimi.artifacts import schema_errors + +from .errors import PrepError +from .source import RepoSource, RepoTree, SourceError, open_source, resolve_rev +from .workspace import source_git_dir + +SCHEMA_VERSION = "suite_manifest.v1" +_SHA_RE = re.compile(r"^[0-9a-f]{7,40}$") # a commit: a branch, tag or other ref is not + + +class SuiteError(ValueError): + """The suite manifest is malformed; ``errors`` lists every problem found.""" + + def __init__(self, path: str, errors: list[str]): + self.path = path + self.errors = errors + super().__init__(f"{path}: invalid {SCHEMA_VERSION}:\n" + "\n".join(f" - {e}" for e in errors)) + + +@dataclass(frozen=True) +class SuiteManifest: + """A validated suite: ``repos`` maps each name to its parsed ``@``, ``tasks`` each task id to a name.""" + + path: str + id: str + repos: dict[str, RepoSource] + tasks: dict[str, str] + + +@dataclass +class SuiteBases: + """Where each task of a suite run takes its base from, and why a task has no base. + + ``git_dirs`` and ``meta`` (``{"repo", "source"}``, for meta.json) cover the tasks whose base + resolved; ``gaps`` gives the reason for each task that did not; ``pins`` names the commit each + repository resolved to, which a pinned batch records. + """ + + git_dirs: dict[str, Path] = field(default_factory=dict) + gaps: dict[str, str] = field(default_factory=dict) + meta: dict[str, dict[str, str]] = field(default_factory=dict) + pins: dict[str, str] = field(default_factory=dict) + + +def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) -> SuiteManifest: + """Validate an already-parsed suite mapping. A branch, tag or other ref is refused as a pin unless + ``allow_floating``, in which case it is resolved when the run starts and the batch pins the result.""" + errors = schema_errors(SCHEMA_VERSION, raw) + if errors: + raise SuiteError(path, errors) + problems: list[str] = [] + repos: dict[str, RepoSource] = {} + for name, spec in raw["repos"].items(): + try: + source = RepoSource.parse(spec) + except SourceError as exc: + problems.append(f"repos.{name}: {exc}") + continue + if source.rev is None: + problems.append(f"repos.{name}: {spec!r} is not pinned; give @") + elif not _SHA_RE.match(source.rev) and not allow_floating: + problems.append(f"repos.{name}: {source.rev!r} is a branch, tag or ref, not a commit; pin its SHA, " + "or pass --allow-floating-suite-revs to resolve it when the run starts") + else: + repos[name] = source + tasks: dict[str, str] = {} + for item in raw["tasks"]: + task_id, repo = item["id"], item["repo"] + if task_id in tasks: + problems.append(f"task {task_id} is listed more than once") + elif repo not in raw["repos"]: + problems.append(f"task {task_id}: repo {repo!r} is not declared in repos") + else: + tasks[task_id] = repo + if problems: + raise SuiteError(path, problems) + return SuiteManifest(path=path, id=raw["id"], repos=repos, tasks=tasks) + + +def load_suite(tree: RepoTree, path: str, allow_floating: bool = False) -> SuiteManifest: + """Read and validate the suite manifest at ``path`` inside the eval tree.""" + if not tree.is_file(path): + raise SuiteError(path, [f"suite manifest not found in {tree.label}"]) + try: + raw = yaml.safe_load(tree.read_text(path)) + except yaml.YAMLError as exc: + raise SuiteError(path, [f"not valid YAML: {exc}"]) from exc + return parse_suite(raw, path, allow_floating) + + +def _checkout(source: RepoSource, cache_dir: Path | None) -> tuple[Path, str] | str: + """``(git dir, commit)`` of a repository of the suite, or the reason it cannot be opened.""" + try: + tree = open_source(source, cache_dir) + git_dir = source_git_dir(source, tree, cache_dir) + except (ValueError, OSError, PrepError) as exc: # SourceError is a ValueError + return f"{source} is unavailable: {exc}" + sha = getattr(tree, "sha", None) + return (git_dir, sha) if isinstance(sha, str) else f"{source} did not resolve to a commit" + + +def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Path | None = None) -> SuiteBases: + """Open every repository of the suite and find each task's base in the repository it names. + + ``bases`` maps each suite task to the ``base`` of its task.json. Repositories are opened once + each, all of them, so the pins do not depend on which tasks a run selects. + """ + checkouts = {name: _checkout(source, cache_dir) for name, source in suite.repos.items()} + out = SuiteBases(pins={name: found[1] for name, found in checkouts.items() if not isinstance(found, str)}) + for task_id, repo in suite.tasks.items(): + found = checkouts[repo] + if isinstance(found, str): + out.gaps[task_id] = f"base repository {repo}: {found}" + continue + git_dir, sha = found + base = bases.get(task_id, "") + try: + resolve_rev(git_dir, base) + except SourceError as exc: + out.gaps[task_id] = (f"base {base!r} is not a commit of {repo} " + f"({suite.repos[repo].location}@{sha[:12]}): {exc}") + continue + out.git_dirs[task_id] = git_dir + out.meta[task_id] = {"repo": repo, "source": f"{suite.repos[repo].location}@{sha}"} + return out diff --git a/dokimi/dokimi/schemas/suite_manifest.v1.json b/dokimi/dokimi/schemas/suite_manifest.v1.json new file mode 100644 index 000000000..1e0a65bd9 --- /dev/null +++ b/dokimi/dokimi/schemas/suite_manifest.v1.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/suite_manifest.v1.json", + "title": "Cross-repository suite manifest v1", + "description": "suite.yaml: tasks of the eval tree whose base is a commit of another repository (docs/repo-evaluation-contract.md §2.1). `repos` names the scaffold repositories, each referenced as @ (a local pinned path works too); `tasks` lists the tasks of the eval tree that take their base checkout from one of them. This is an input the benchmark repository owns, so it carries `schema`, not `schema_version`. Semantic rules (every task is a task of the tree, every repo name is declared, revisions are full commits unless the run allows floating ones) are checked in dokimi/repos/suite.py, which says them in friendlier words.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "repos", "tasks"], + "properties": { + "schema": {"const": "suite_manifest.v1"}, + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}, + "repos": { + "type": "object", + "minProperties": 1, + "propertyNames": {"pattern": "^[a-z0-9][a-z0-9._-]*$"}, + "additionalProperties": {"type": "string", "minLength": 1} + }, + "tasks": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "repo"], + "properties": { + "id": {"type": "string", "pattern": "^[^\\s,]+$"}, + "repo": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"} + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json b/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json new file mode 100644 index 000000000..dff4ecdd1 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json @@ -0,0 +1,12 @@ +{ + "schema": "suite_manifest.v1", + "id": "linreg-cpu-solver", + "repos": { + "sklearn-scaffold": "https://github.com/example-org/sklearn-scaffold@9f8e7d6c5b4a39281706f5e4d3c2b1a098765432", + "numpy-scaffold": "https://github.com/example-org/numpy-scaffold@0a1b2c3d4e5f60718293a4b5c6d7e8f901234567" + }, + "tasks": [ + {"id": "solver-pick-cholesky", "repo": "sklearn-scaffold"}, + {"id": "solver-pick-lstsq", "repo": "numpy-scaffold"} + ] +} diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py new file mode 100644 index 000000000..9a892052a --- /dev/null +++ b/dokimi/tests/test_repo_suite.py @@ -0,0 +1,192 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Suite manifests (suite_manifest.v1, contract §2.1): the schema, the pins, the base of each task +and the run. Every repository here is a local git repository; nothing reaches the network.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from dokimi.cli import build_parser +from dokimi.repos.source import RepoSource, open_source +from dokimi.repos.suite import SuiteError, load_suite, parse_suite, resolve_suite +from dokimi.utils import InputError +from repo_eval_fixture import FakeRunner, fake_harness, git, make_repo, write + +EXAMPLE = Path(__file__).resolve().parent.parent / "data" / "suites" / "example-suite.yaml" +PINNED = "https://example.invalid/scaffold@" + "a" * 40 +CALC = "def add(a, b):\n return a + b\n" + + +def _raw(**over): + raw = {"schema": "suite_manifest.v1", "id": "s", "repos": {"sc": PINNED}, "tasks": [{"id": "t1", "repo": "sc"}]} + raw.update(over) + return raw + + +def test_the_example_suite_parses_and_every_repo_is_pinned(): + suite = parse_suite(yaml.safe_load(EXAMPLE.read_text(encoding="utf-8")), str(EXAMPLE)) + assert suite.tasks == {"solver-pick-cholesky": "sklearn-scaffold", "solver-pick-lstsq": "numpy-scaffold"} + assert all(len(src.rev or "") == 40 for src in suite.repos.values()) + + +@pytest.mark.parametrize("rev", ["main", "v1.0", "HEAD~1"]) +def test_a_branch_tag_or_ref_is_refused_as_a_pin(rev): + with pytest.raises(SuiteError, match="not a commit; pin its SHA"): + parse_suite(_raw(repos={"sc": f"https://example.invalid/scaffold@{rev}"})) + + +def test_a_floating_rev_is_kept_only_when_the_run_allows_it(): + suite = parse_suite(_raw(repos={"sc": "https://example.invalid/scaffold@main"}), allow_floating=True) + assert suite.repos["sc"].rev == "main" + + +def test_a_repo_without_a_revision_is_refused_even_when_floating_is_allowed(tmp_path): + with pytest.raises(SuiteError, match="not pinned"): + parse_suite(_raw(repos={"sc": str(tmp_path)}), allow_floating=True) + with pytest.raises(SuiteError, match="must be pinned"): + parse_suite(_raw(repos={"sc": "https://example.invalid/scaffold"})) + + +def test_a_task_must_name_a_declared_repo_and_appear_once(): + with pytest.raises(SuiteError, match="repo 'nope' is not declared"): + parse_suite(_raw(tasks=[{"id": "t1", "repo": "nope"}])) + with pytest.raises(SuiteError, match="listed more than once"): + parse_suite(_raw(tasks=[{"id": "t1", "repo": "sc"}, {"id": "t1", "repo": "sc"}])) + + +@pytest.mark.parametrize("change", [ + {"schema": "suite_manifest.v2"}, + {"unexpected": True}, + {"repos": {}}, + {"tasks": [{"id": "a,b", "repo": "sc"}]}, + {"tasks": [{"id": "t1"}]}, +]) +def test_the_schema_refuses_a_malformed_suite(change): + with pytest.raises(SuiteError): + parse_suite(_raw(**change)) + + +def test_a_suite_file_missing_from_the_tree_is_refused(tmp_path): + root, _ = make_repo(tmp_path) + tree = open_source(RepoSource.parse(str(root))) + with pytest.raises(SuiteError, match="suite manifest not found"): + load_suite(tree, ".agent-eval/suite.yaml") + + +def _scaffold(tmp_path: Path, name: str = "scaffold") -> tuple[Path, str]: + root = tmp_path / name + write(root, "src/calc.py", CALC) + git(root, "init", "-q", "-b", "main") + git(root, "add", "-A") + git(root, "commit", "-qm", "scaffold") + return root, git(root, "rev-parse", "HEAD") + + +def test_each_task_takes_its_base_from_the_repo_it_names_and_a_gap_is_recorded(tmp_path): + good, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"file://{tmp_path}/missing@{'b' * 40}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "absent", "repo": "good"}, + {"id": "lost", "repo": "gone"}])) + bases = {"ok": sha, "absent": "1" * 40, "lost": sha} + found = resolve_suite(suite, bases, tmp_path / "cache") + assert set(found.git_dirs) == {"ok"} and found.git_dirs["ok"].is_dir() + assert found.meta == {"ok": {"repo": "good", "source": f"{good}@{sha}"}} + assert found.pins == {"good": sha} + assert "is not a commit of good" in found.gaps["absent"] + assert found.gaps["lost"].startswith("base repository gone:") and "unavailable" in found.gaps["lost"] + + +def test_a_floating_branch_resolves_once_to_the_commit_it_names(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"{root}@main"}, tasks=[{"id": "t1", "repo": "sc"}]), + allow_floating=True) + found = resolve_suite(suite, {"t1": sha}, tmp_path / "cache") + assert found.pins == {"sc": sha} and found.gaps == {} + + +def test_the_suite_file_is_read_from_the_eval_tree(tmp_path): + root, _ = make_repo(tmp_path) + write(root, ".agent-eval/suite.yaml", yaml.safe_dump(_raw())) + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + tree = open_source(RepoSource.parse(str(root))) + assert load_suite(tree, ".agent-eval/suite.yaml").tasks == {"t1": "sc"} + + +@pytest.fixture +def run(tmp_path, monkeypatch): + root, _ = make_repo(tmp_path) + exe, profile = fake_harness(tmp_path) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + monkeypatch.setenv("FAKE_ACTION", "fix") + state = {"root": root, "out": tmp_path / "runs", "tmp": tmp_path} + + def invoke(*extra: str) -> None: + argv = ["repo", "run", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(profile), "--bin", str(exe), "--model", "m-1", "--batch", "b1", "--out", str(state["out"]), + "--cache-root", str(state["out"] / "caches"), "--agent-network", "host", *extra] + args = build_parser().parse_args(argv) + args.func(args) + + state["invoke"] = invoke + return state + + +def _commit_suite(root: Path, base_of_fix: str | None, suite: dict) -> None: + """Point fix_add at ``base_of_fix`` (when given) and commit the suite file into the eval tree.""" + if base_of_fix is not None: + task = root / ".agent-eval/tasks/fix_add/task.json" + write(root, ".agent-eval/tasks/fix_add/task.json", + json.dumps({**json.loads(task.read_text(encoding="utf-8")), "base": base_of_fix})) + write(root, ".agent-eval/suite.yaml", yaml.safe_dump(suite)) + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + + +def test_a_run_takes_the_base_of_a_suite_task_from_its_repo(run): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + fix = json.loads((batch / "fix_add__guidance-none__m-1__r1/meta.json").read_text(encoding="utf-8")) + assert fix["base_sha"] == sha and fix["base_repo"] == {"repo": "sc", "source": f"{scaffold}@{sha}"} + assert (batch / "fix_add__guidance-none__m-1__r1/result.json").is_file() + review = json.loads((batch / "review_add__guidance-none__m-1__r1/meta.json").read_text(encoding="utf-8")) + assert "base_repo" not in review # a task the suite does not name keeps its eval-tree base + pin = json.loads((batch / "_eval/batch.json").read_text(encoding="utf-8")) + assert pin["suite"] == {"sc": sha} + + +def test_a_missing_suite_repo_leaves_its_cells_unsupported_and_not_failed(run, capsys): + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(run["root"], None, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + assert not (batch / "fix_add__guidance-none__m-1__r1/result.json").exists() + unsupported = json.loads((batch / "fix_add__guidance-none__m-1__r1/unsupported.json").read_text(encoding="utf-8")) + assert unsupported["reason"].startswith("base repository sc:") and unsupported["task"] == "fix_add" + assert (batch / "review_add__guidance-none__m-1__r1/result.json").is_file() + assert "fix_add: UNSUPPORTED, its cells are not run: base repository sc:" in capsys.readouterr().out + + +def test_a_suite_task_the_tree_lacks_is_refused_before_any_cell(run, capsys): + _commit_suite(run["root"], None, _raw(tasks=[{"id": "nope", "repo": "sc"}], + repos={"sc": f"{run['root']}@{'d' * 40}"})) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--arm", "guidance:none") + assert exc.value.code == 1 + assert "task nope is not under" in capsys.readouterr().err + assert not (run["out"] / "b1" / "nope__guidance-none__m-1__r1").exists() + + +def test_a_suite_path_that_leaves_the_tree_is_a_usage_error(run): + with pytest.raises(InputError, match="path escapes the repository"): + run["invoke"]("--suite", "../suite.yaml", "--arm", "guidance:none") diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 7508d0c72..1ef5cf2a8 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -435,7 +435,7 @@ def test_check_task_runs_no_oracle_step_for_a_typecheck_task_without_reference(t monkeypatch.setattr(selfcheck, "_check_step", lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) - ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None), base_gaps=dict) # no suite: no gaps res = selfcheck.check_task(ctx, Task("t", raw, bare), tmp_path / "work") assert ran == [("null", True), ("oracle", False)] assert res["verdict"] == "ok" @@ -456,7 +456,7 @@ def test_check_task_runs_the_oracle_step_for_a_typecheck_task_with_reference(tmp monkeypatch.setattr(selfcheck, "_check_step", lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) - ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None), base_gaps=dict) # no suite: no gaps selfcheck.check_task(ctx, Task("t", raw, with_reference), tmp_path / "work") assert ran == [("null", True), ("oracle", True)] From c8732661c25962aad1da5a7e1697b52a303ad1c4 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:46:02 -0600 Subject: [PATCH 02/14] fix(repo): pin and resolve suite bases strictly Fixes on the suite manifest: - a pin must be a full 40 or 64 hex commit; a short SHA or a hex-named branch is no longer taken as one - a pinned local path that does not exist leaves its tasks unsupported instead of refusing the whole suite - a base missing from a warm remote cache is fetched once before it is recorded as a gap - the batch pin records the suite file's path, the task-to-repository mapping and the repository locations; resuming with another suite file, or without --suite, is refused - only the suite's tasks are loaded, so an unrelated malformed task.json does not stop the run - the guidance revision of an arm is read from the eval tree while the base is checked out from the scaffold - check_arms and the .mcp.json check skip suite tasks, whose base is not in the eval tree, so a gap no longer aborts the batch - repo tasks check exits 0 when every selected task is a suite gap, and the gap is judged before the egress requirement - a suite file that is not UTF-8 is a SuiteError Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/docs/repo-evaluation-contract.md | 10 +- dokimi/dokimi/commands/repo_run.py | 31 +++-- dokimi/dokimi/repos/arms.py | 11 +- dokimi/dokimi/repos/grading.py | 4 + dokimi/dokimi/repos/runner.py | 3 +- dokimi/dokimi/repos/selfcheck.py | 10 +- dokimi/dokimi/repos/suite.py | 40 ++++++- dokimi/dokimi/repos/workspace.py | 15 ++- dokimi/tests/test_repo_host.py | 4 +- dokimi/tests/test_repo_suite.py | 149 +++++++++++++++++++++++- 10 files changed, 237 insertions(+), 40 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index a349fa536..25ca39f49 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -339,13 +339,17 @@ tasks: revision must be a full commit SHA. A branch, tag or other ref is refused unless the run passes `--allow-floating-suite-revs`; it is then resolved once when the run starts, and the batch pin (`_eval/batch.json`, key `suite`) - records the commits, so a resumed batch reuses them. + records the commits, so a resumed batch reuses them. The pin also records the + suite file's path, the task-to-repository mapping and the repository locations (key `suite_plan`): a + resume with another suite file, or without `--suite`, is refused. - A repository that cannot be opened, or a base that is not a commit of it, leaves that task's cells unsupported (§9): `unsupported.json` gives the reason, and `repo report` counts the cells neither as failed nor as missing. A cell - that runs records `base_repo` (`{"repo", "source"}`) in `meta.json`. + that runs records `base_repo` (`{"repo", "source"}`) in `meta.json`. A base that + the cached clone of a remote lacks is fetched once before it counts as missing. - `repo tasks check` reports these tasks as not checked when their repository is - missing and fails only on the other bad tasks. + missing and fails only on the other bad tasks; when every selected task is + not checked it exits 0. - Not covered yet: `repo validate` checks the bases of the eval tree only; a suite task's `{source_dir}` is still the eval-tree source; and a `guidance:rev:` arm is pinned against the eval-tree source. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 02469ef32..97f0e3f69 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -135,19 +135,20 @@ def _suite_bases(args: argparse.Namespace, tree, content: Path, manifest: RepoMa if not args.suite: return None from dokimi.repos.suite import SuiteError, load_suite, resolve_suite - from dokimi.repos.workspace import load_tasks + from dokimi.repos.workspace import PrepError, load_tasks suite = load_suite(tree, args.suite, args.allow_floating_suite_revs) - found = {t.id: t for t in load_tasks(content, manifest)} - absent = sorted(set(suite.tasks) - set(found)) - if absent: - raise SuiteError(suite.path, [f"task {t} is not under {manifest.tasks}" for t in absent]) + try: # only the listed tasks are read: an unrelated malformed task.json is not this run's + found = {t.id: t for t in load_tasks(content, manifest, list(suite.tasks))} + except PrepError as exc: + raise SuiteError(suite.path, [str(exc)]) from exc return resolve_suite(suite, {t: str(found[t].raw.get("base", "")) for t in suite.tasks}, cache_dir) def _suite_pin(suite) -> dict[str, Any]: - """The commit each suite repository resolved to, for the batch pin: a resume must reuse them.""" - return {} if suite is None else {"suite": suite.pins} + """The commit each suite repository resolved to and the task-to-repository mapping, for the batch pin: + a resume with another suite file, or with none, is refused (``_pin_batch``).""" + return {} if suite is None else {"suite": suite.pins, "suite_plan": suite.plan} def _require_grade(content: Path, manifest: RepoManifest, ids: list[str] | None) -> None: @@ -177,6 +178,9 @@ def _uses_builtin_graders(content: Path, manifest: RepoManifest) -> bool: return any(isinstance(t.raw, dict) and builtin_name(t.raw.get("grader")) is not None for t in tasks) +_SUITE_KEYS = ("suite", "suite_plan") + + def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None: path = batch_dir / "_eval" / "batch.json" if path.exists(): @@ -187,9 +191,10 @@ def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None: if not isinstance(old, dict): raise ValueError(f"batch {batch_dir.name} has an unreadable pin ({path}); use another --batch") changed = [k for k in pin if old.get(k) != pin[k]] + changed += [k for k in _SUITE_KEYS if k in old and k not in pin] # a suite batch resumed without --suite if changed: was = ", ".join(f"{k} {old.get(k)}" for k in changed) - now = ", ".join(f"{k} {pin[k]}" for k in changed) + now = ", ".join(f"{k} {pin.get(k, 'none')}" for k in changed) raise ValueError(f"batch {batch_dir.name} was run from {was}, not {now}; use another --batch") return path.parent.mkdir(parents=True, exist_ok=True) @@ -234,7 +239,8 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: ctx, _ = _context(args, out, source) tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) arms = [a for spec in args.arm or () for a in spec.split(",") if a] - check_arms([parse_arm(a) for a in arms], ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id) # skill paths exist, catalogs hold each expected_skill + check_arms([parse_arm(a) for a in arms], ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, + ctx.manifest.id, ctx.suite_tasks()) # skill paths exist, catalogs hold each expected_skill if arms: print(f"{len(arms)} arm(s) ok: {', '.join(arms)}", file=sys.stderr) print(f"{len(tasks)} task(s), image {ctx.image}{' (warm: with network)' if args.warm else ''}", @@ -254,7 +260,7 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: + ", ".join(r["task"] for r in allowed)) if len(bad) > len(allowed) + len(unavailable): sys.exit(1) - if len(bad) == len(results): # --allow-unsupported passed every task by: none was checked + if allowed and len(allowed) == len(results) - len(unavailable): # every task that could be checked was passed by print(f"✗ incomplete: no task passed the self-check (0 ok, {len(allowed)} unsupported)", file=sys.stderr) sys.exit(1) @@ -370,8 +376,9 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock lock.enter_context(batch_lock(batch_dir)) # before _context, which rewrites _eval/content ctx, _ = _context(args, batch_dir, source, pin=True) tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) - check_arms(arms, ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id) # skill paths exist, catalogs hold each expected_skill - check_mcp_configs(ctx.git_dir, tasks, profile, args.tools) # a malformed .mcp.json fails here, not in every cell + check_arms(arms, ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id, + ctx.suite_tasks()) # skill paths exist, catalogs hold each expected_skill + check_mcp_configs(ctx.git_dir, [t for t in tasks if t.id not in ctx.suite_tasks()], profile, args.tools) # a malformed .mcp.json fails here, not in every cell skill_pin = pin_skills(batch_dir, arms, ctx.content_dir) # a resume may not swap an arm's skills validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, diff --git a/dokimi/dokimi/repos/arms.py b/dokimi/dokimi/repos/arms.py index 23974f242..6aee65632 100644 --- a/dokimi/dokimi/repos/arms.py +++ b/dokimi/dokimi/repos/arms.py @@ -21,6 +21,7 @@ import hashlib import os import re +from collections.abc import Collection from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -191,7 +192,8 @@ def _base_skill_files(git_dir: Path, base: str) -> tuple[list[str], list[str]]: def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Path | None = None, - guidance_globs: tuple[str, ...] | list[str] = (), repo_id: str | None = None) -> None: + guidance_globs: tuple[str, ...] | list[str] = (), repo_id: str | None = None, + elsewhere: Collection[str] = ()) -> None: """Find the skills of every parsed arm before anything runs; a catalog must hold each task's ``expected_skill``. Two arms that would share a name or a result directory are refused, and so is a skill the base of a task (in ``git_dir``) already ships, which preparing the workspace would refuse cell by cell. @@ -201,7 +203,8 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa arm writes its own skill, ``.claude/skills/-guidance``: a skill of the arm with that name, or a file the base tracks under it, is refused here (``repo_id`` given). The files of a ``patch:`` arm's patch and of a task's ``setup_patches`` are not read: a skill directory they add is refused only in - preparation.""" + preparation. The bases of the tasks in ``elsewhere`` (a suite's) are not in ``git_dir`` and are not read.""" + own = [t for t in tasks if t.id not in elsewhere] # the bases read from git_dir cells: dict[str, str] = {} # cell-name slug of an arm -> the arm spec that owns it shipped: dict[str, tuple[list[str], list[str]]] = {} # base -> (tracked files under .claude/skills, its links) for arm in arms: @@ -212,12 +215,12 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa names = {s.name for s in skills} generated = None # the skill preparation installs beside the catalog's; its usage is scored apart if repo_id is not None and arm.patch is None and arm.base == "guidance:as-skill": - _check_generated_skill(arm, skills, repo_id, tasks, git_dir, shipped) + _check_generated_skill(arm, skills, repo_id, own, git_dir, shipped) generated = guidance_skill_name(repo_id) drops = arm.patch is None and (arm.base in ("guidance:none", "guidance:as-skill") or arm.rev is not None) rev = (arm.rev_sha or arm.rev) if arm.patch is None else None if skills and git_dir is not None: - for task in tasks: + for task in own: _check_shipped(arm, skills, task, git_dir, shipped, (drops, rev, guidance_globs)) if any(kind == "skill-catalog" for kind, _ in arm.skills): _check_expected_skills(arm, tasks, names, generated) diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py index 9b6111da4..546994980 100644 --- a/dokimi/dokimi/repos/grading.py +++ b/dokimi/dokimi/repos/grading.py @@ -96,6 +96,10 @@ def git_dir_for(self, task_id: str) -> Path: """The git dir a task's base is checked out from: its suite repository, else ``git_dir``.""" return self.suite.git_dirs.get(task_id, self.git_dir) if self.suite else self.git_dir + def suite_tasks(self) -> set[str]: + """The tasks of the suite: their base is not in ``git_dir``, so nothing reads it there.""" + return set(self.suite.git_dirs) | set(self.suite.gaps) if self.suite else set() + def base_gaps(self) -> dict[str, str]: """Why each suite task has no base checkout; its cells are unsupported, not run.""" return dict(self.suite.gaps) if self.suite else {} diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index b65b6de09..8f11b6cc1 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -218,7 +218,8 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, if rd.exists(): _rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir_for(task.id), ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) + meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir, + base_git_dir=ctx.git_dir_for(task.id)) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) if ctx.suite is not None and task.id in ctx.suite.meta: # additive: only a suite task's base is elsewhere diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index 2f99f6074..2f7dbb764 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -75,15 +75,15 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, ``host`` requirements probes for itself.""" network = ctx.manifest.image.network if ctx.manifest.image else "none" res: dict[str, Any] = {"task": task.id, "family": task.family, "warm": warm, "problems": []} + gap = ctx.base_gaps().get(task.id) # its base is in a suite repository that is missing: before anything else + if gap: + res.update(verdict="unsupported", unsupported_by="suite", problems=[gap]) + return res try: net = "bridge" if warm else docker_network(network, task.egress) except ContainerError as exc: res.update(verdict="unsupported", unsupported_by="egress", problems=[str(exc)]) return res - gap = ctx.base_gaps().get(task.id) # its base is in a suite repository that is missing - if gap: - res.update(verdict="unsupported", unsupported_by="suite", problems=[gap]) - return res # The probe sees what every container of this runner sees (affinity, cgroup quota), with or without # --warm's network: an unmet host requirement is unsupported either way, never a passing null run. reason, unsupported_by, facts = _host_reason(ctx, task, work_dir, host) @@ -171,7 +171,7 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: if rd.exists(): shutil.rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir_for(task.id), ctx.manifest, task, "guidance:raw", rd) + meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, "guidance:raw", rd, base_git_dir=ctx.git_dir_for(task.id)) meta.update(harness="none", harness_version=None, model="none", rep=0) for name, data in (("meta.json", meta), ("task.json", task.raw)): (rd / name).write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8") diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index e88ba0382..abeb616e8 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -19,6 +19,7 @@ import yaml +from dokimi import process from dokimi.artifacts import schema_errors from .errors import PrepError @@ -26,7 +27,7 @@ from .workspace import source_git_dir SCHEMA_VERSION = "suite_manifest.v1" -_SHA_RE = re.compile(r"^[0-9a-f]{7,40}$") # a commit: a branch, tag or other ref is not +_SHA_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") # a full commit: a branch, tag, short SHA or other ref is not class SuiteError(ValueError): @@ -54,13 +55,15 @@ class SuiteBases: ``git_dirs`` and ``meta`` (``{"repo", "source"}``, for meta.json) cover the tasks whose base resolved; ``gaps`` gives the reason for each task that did not; ``pins`` names the commit each - repository resolved to, which a pinned batch records. + repository resolved to and ``plan`` the suite file's path, the task-to-repository mapping and the repository + locations, which a pinned batch records. """ git_dirs: dict[str, Path] = field(default_factory=dict) gaps: dict[str, str] = field(default_factory=dict) meta: dict[str, dict[str, str]] = field(default_factory=dict) pins: dict[str, str] = field(default_factory=dict) + plan: dict[str, Any] = field(default_factory=dict) def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) -> SuiteManifest: @@ -73,7 +76,7 @@ def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) - repos: dict[str, RepoSource] = {} for name, spec in raw["repos"].items(): try: - source = RepoSource.parse(spec) + source = _parse_repo(spec) except SourceError as exc: problems.append(f"repos.{name}: {exc}") continue @@ -98,13 +101,25 @@ def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) - return SuiteManifest(path=path, id=raw["id"], repos=repos, tasks=tasks) +def _parse_repo(spec: str) -> RepoSource: + """``RepoSource.parse``, except that a pinned local path that is not a directory is kept: it is + unavailable, which leaves its tasks unsupported (``_checkout``), not a malformed suite.""" + try: + return RepoSource.parse(spec) + except SourceError as exc: + head, sep, tail = spec.strip().rpartition("@") + if "local source is not a directory" not in str(exc) or not (sep and head and tail) or "://" in head: + raise + return RepoSource(location=head, rev=tail, remote=False) + + def load_suite(tree: RepoTree, path: str, allow_floating: bool = False) -> SuiteManifest: """Read and validate the suite manifest at ``path`` inside the eval tree.""" if not tree.is_file(path): raise SuiteError(path, [f"suite manifest not found in {tree.label}"]) try: raw = yaml.safe_load(tree.read_text(path)) - except yaml.YAMLError as exc: + except (yaml.YAMLError, UnicodeDecodeError) as exc: raise SuiteError(path, [f"not valid YAML: {exc}"]) from exc return parse_suite(raw, path, allow_floating) @@ -120,6 +135,17 @@ def _checkout(source: RepoSource, cache_dir: Path | None) -> tuple[Path, str] | return (git_dir, sha) if isinstance(sha, str) else f"{source} did not resolve to a commit" +def _resolve_base(source: RepoSource, git_dir: Path, base: str) -> str: + """``base`` in ``git_dir``. A remote's cached clone may predate it, so a miss fetches it once before it is a gap.""" + try: + return resolve_rev(git_dir, base) + except SourceError: + if not source.remote or not _SHA_RE.match(base): + raise + process.git("-C", str(git_dir), "fetch", "origin", "--", base) # a failure leaves the miss below + return resolve_rev(git_dir, base) + + def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Path | None = None) -> SuiteBases: """Open every repository of the suite and find each task's base in the repository it names. @@ -127,7 +153,9 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat each, all of them, so the pins do not depend on which tasks a run selects. """ checkouts = {name: _checkout(source, cache_dir) for name, source in suite.repos.items()} - out = SuiteBases(pins={name: found[1] for name, found in checkouts.items() if not isinstance(found, str)}) + out = SuiteBases(pins={name: found[1] for name, found in checkouts.items() if not isinstance(found, str)}, + plan={"path": suite.path, "repos": {name: str(source) for name, source in suite.repos.items()}, + "tasks": dict(suite.tasks)}) for task_id, repo in suite.tasks.items(): found = checkouts[repo] if isinstance(found, str): @@ -136,7 +164,7 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat git_dir, sha = found base = bases.get(task_id, "") try: - resolve_rev(git_dir, base) + _resolve_base(suite.repos[repo], git_dir, base) except SourceError as exc: out.gaps[task_id] = (f"base {base!r} is not a commit of {repo} " f"({suite.repos[repo].location}@{sha[:12]}): {exc}") diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index bf8fa5efc..2698fa1e3 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -366,9 +366,13 @@ def _apply_review_patch(ws: Path, task: Task, patches: list[dict[str, str]]) -> def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: str, run_dir: Path, - profile: HarnessProfile | None = None, content_dir: Path | None = None) -> dict[str, Any]: + profile: HarnessProfile | None = None, content_dir: Path | None = None, + base_git_dir: Path | None = None) -> dict[str, Any]: """Steps 1-7 into ``run_dir/workspace``; returns the ``meta.json`` fields prep owns. + The base is checked out from ``base_git_dir`` (default ``git_dir``): a suite task's base is in another + repository, while the arm's revisions and guidance are still read from ``git_dir``. + ``arm`` is an :class:`Arm` (``repo run`` parses and pins each once per batch) or an arm string, parsed and pinned here. ``content_dir`` (the exported ``{eval_dir}``) is where relative skill paths of the arm are found. @@ -389,17 +393,18 @@ def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: st if arm.base == OVERLAY_ARM and overlay not in overlay_names(): names = ", ".join(sorted(overlay_names())) raise PrepError(f"guidance:overlay: overlay {overlay!r} is not a builtin overlay name ({names})") + base_dir = base_git_dir or git_dir try: - base_sha = resolve_rev(git_dir, task.base) + base_sha = resolve_rev(base_dir, task.base) except SourceError as exc: - raise PrepError(f"task {task.id}: base {task.base!r} is not in {git_dir}: {exc}") from exc - base_date = _run(["git", "-C", str(git_dir), "log", "-1", "--format=%cI", base_sha]).strip() + raise PrepError(f"task {task.id}: base {task.base!r} is not in {base_dir}: {exc}") from exc + base_date = _run(["git", "-C", str(base_dir), "log", "-1", "--format=%cI", base_sha]).strip() ws = run_dir / "workspace" if ws.exists(): shutil.rmtree(ws) ws.mkdir(parents=True) - _checkout_tree(git_dir, base_sha, ws) # 1 + _checkout_tree(base_dir, base_sha, ws) # 1 shutil.rmtree(ws / manifest.run_paths()[0], ignore_errors=True) # 2 env = _git_env(SNAPSHOT_AUTHOR) _git(ws, env, "init", "-q") # 3 diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index cab6f3190..a275ae8ba 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -718,10 +718,10 @@ def test_a_prepare_error_on_the_null_step_is_a_problem_and_the_oracle_still_runs root = _repo(tmp_path) real = prepare_workspace - def prepare(git_dir, manifest, task, arm, rd): + def prepare(git_dir, manifest, task, arm, rd, **kwargs): if rd.name.endswith("__null"): raise PrepError("no such revision") - return real(git_dir, manifest, task, arm, rd) + return real(git_dir, manifest, task, arm, rd, **kwargs) monkeypatch.setattr("dokimi.repos.selfcheck.prepare_workspace", prepare) with pytest.raises(SystemExit) as exc: diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 9a892052a..516089818 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -35,12 +35,18 @@ def test_the_example_suite_parses_and_every_repo_is_pinned(): assert all(len(src.rev or "") == 40 for src in suite.repos.values()) -@pytest.mark.parametrize("rev", ["main", "v1.0", "HEAD~1"]) +@pytest.mark.parametrize("rev", ["main", "v1.0", "HEAD~1", "abcdef0", "a" * 39, "a" * 41, "a" * 63]) def test_a_branch_tag_or_ref_is_refused_as_a_pin(rev): with pytest.raises(SuiteError, match="not a commit; pin its SHA"): parse_suite(_raw(repos={"sc": f"https://example.invalid/scaffold@{rev}"})) +@pytest.mark.parametrize("rev", ["a" * 40, "b" * 64]) +def test_a_full_commit_sha_is_a_pin(rev): + suite = parse_suite(_raw(repos={"sc": f"https://example.invalid/scaffold@{rev}"})) + assert suite.repos["sc"].rev == rev + + def test_a_floating_rev_is_kept_only_when_the_run_allows_it(): suite = parse_suite(_raw(repos={"sc": "https://example.invalid/scaffold@main"}), allow_floating=True) assert suite.repos["sc"].rev == "main" @@ -183,10 +189,149 @@ def test_a_suite_task_the_tree_lacks_is_refused_before_any_cell(run, capsys): with pytest.raises(SystemExit) as exc: run["invoke"]("--suite", ".agent-eval/suite.yaml", "--arm", "guidance:none") assert exc.value.code == 1 - assert "task nope is not under" in capsys.readouterr().err + assert "no such task(s) under" in capsys.readouterr().err assert not (run["out"] / "b1" / "nope__guidance-none__m-1__r1").exists() def test_a_suite_path_that_leaves_the_tree_is_a_usage_error(run): with pytest.raises(InputError, match="path escapes the repository"): run["invoke"]("--suite", "../suite.yaml", "--arm", "guidance:none") + + +def test_a_suite_file_that_is_not_utf8_is_a_suite_error(tmp_path): + root, _ = make_repo(tmp_path) + write(root, ".agent-eval/suite.yaml", "") + (root / ".agent-eval/suite.yaml").write_bytes(b"schema: \xff\xfe\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + tree = open_source(RepoSource.parse(str(root))) + with pytest.raises(SuiteError, match="not valid YAML"): + load_suite(tree, ".agent-eval/suite.yaml") + + +@pytest.mark.parametrize("scheme", ["", "file://"]) +def test_a_missing_repository_leaves_its_tasks_unsupported_in_either_spelling(tmp_path, scheme): + good, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"{scheme}{tmp_path}/missing@{'b' * 40}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "lost", "repo": "gone"}])) + found = resolve_suite(suite, {"ok": sha, "lost": sha}, tmp_path / "cache") + assert set(found.git_dirs) == {"ok"} + assert found.gaps["lost"].startswith("base repository gone:") and "unavailable" in found.gaps["lost"] + + +def test_a_local_path_that_is_not_pinned_is_still_refused(tmp_path): + with pytest.raises(SuiteError, match="local source is not a directory"): + parse_suite(_raw(repos={"sc": f"{tmp_path}/missing"}), allow_floating=True) + + +def test_a_base_the_warm_cache_of_a_remote_predates_is_fetched_not_a_gap(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + later = git(root, "rev-parse", "HEAD") + found = resolve_suite(suite, {"t1": later}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + assert resolve_suite(suite, {"t1": "1" * 40}, cache).gaps["t1"].startswith(f"base {'1' * 40!r} is not a commit of sc") + + +def _suite_of(root: Path, tmp: Path, name: str, scaffold: Path, sha: str, *tasks: str) -> str: + """Commit ``.agent-eval/.yaml``: both suites declare ``a`` and ``b`` (the same scaffold), ``tasks`` map to ``b``.""" + repos = {"a": f"{scaffold}@{sha}", "b": f"{scaffold}@{sha}"} + path = f".agent-eval/{name}.yaml" + write(root, path, yaml.safe_dump(_raw(repos=repos, tasks=[{"id": t, "repo": "b"} for t in tasks]))) + git(root, "add", "-A") + git(root, "commit", "-qm", name) + return path + + +def test_a_resume_with_another_suite_file_or_none_is_refused(run, capsys): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw()) # fix_add's base is the scaffold's commit + first = _suite_of(run["root"], run["tmp"], "first", scaffold, sha, "fix_add") + second = _suite_of(run["root"], run["tmp"], "second", scaffold, sha, "review_add") + same = _suite_of(run["root"], run["tmp"], "same", scaffold, sha, "fix_add") # first's content, another file + argv = ["--task", "fix_add", "--arm", "guidance:none"] + run["invoke"]("--suite", first, *argv) + run["invoke"]("--suite", first, *argv) # the same suite resumes + for extra in (["--suite", second], ["--suite", same], []): + with pytest.raises(SystemExit) as exc: + run["invoke"](*extra, *argv) + assert exc.value.code == 1 + assert "use another --batch" in capsys.readouterr().err + + +def test_a_malformed_task_the_suite_and_the_selection_leave_out_does_not_stop_the_run(run): + scaffold, sha = _scaffold(run["tmp"]) + write(run["root"], ".agent-eval/tasks/broken/task.json", "{bad") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", "guidance:none") + assert (run["out"] / "b1/fix_add__guidance-none__m-1__r1/result.json").is_file() + + +def test_a_guidance_revision_arm_reads_the_eval_tree_while_the_base_comes_from_the_scaffold(run): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + rev = git(run["root"], "rev-parse", "HEAD") + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", f"guidance:rev:{rev}") + cell = next((run["out"] / "b1").glob("fix_add__*/meta.json")) + meta = json.loads(cell.read_text(encoding="utf-8")) + assert meta["base_sha"] == sha + assert "AGENTS.md" in meta["arm_files"] # the scaffold has none: it is the eval tree's, at the revision + assert (cell.parent / "workspace/AGENTS.md").read_text(encoding="utf-8") == "# guidance\n" + + +def test_a_suite_gap_does_not_abort_the_batch_on_a_skill_its_eval_base_ships(run): + root = run["root"] + skill = "---\nname: lint\ndescription: x\n---\nLint.\n" + write(root, ".claude/skills/lint/SKILL.md", skill) + git(root, "add", "-A") + git(root, "commit", "-qm", "base ships lint") + base = git(root, "rev-parse", "HEAD") + write(root, ".agent-eval/skills/lint/SKILL.md", skill) + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(root, base, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:raw+skill:skills/lint") + batch = run["out"] / "b1" + assert list(batch.glob("fix_add__*/unsupported.json")) + assert list(batch.glob("review_add__*/result.json")) + + +def _check(tmp: Path, monkeypatch, suite: dict, *extra: str, tasks: dict | None = None) -> tuple[int, dict]: + """``eval check --suite`` over a repository whose suite is ``suite``; returns (exit code, check.json).""" + root, base = make_repo(tmp, tasks) + _commit_suite(root, None, suite) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + out = tmp / "check-out" + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(out), "--cache-root", + str(out / "caches"), "--suite", ".agent-eval/suite.yaml", *extra]) + code = 0 + try: + args.func(args) + except SystemExit as exc: + code = exc.code + return code, json.loads((out / "check/check.json").read_text(encoding="utf-8")) + + +def test_a_self_check_where_every_selected_task_is_a_suite_gap_exits_0(tmp_path, monkeypatch, capsys): + gone = f"file://{tmp_path}/missing@{'c' * 40}" + code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}]), + "--task", "fix_add") + captured = capsys.readouterr() + assert code == 0 and "incomplete" not in captured.err + assert [(r["task"], r["unsupported_by"]) for r in report["tasks"]] == [("fix_add", "suite")] + + +def test_a_suite_gap_is_not_checked_even_when_the_task_also_needs_egress(tmp_path, monkeypatch): + gone = f"file://{tmp_path}/missing@{'c' * 40}" + net = {"net_add": {"family": "review", "grader": "review", "review_patch": "change.patch", "egress": ["example.com"], + "score": {"metric": "recall", "max": 1.0}, "prompt": "Review HEAD."}} + code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "net_add", "repo": "sc"}]), + "--task", "net_add", tasks=net) + assert code == 0 + assert report["tasks"][0]["unsupported_by"] == "suite" From 4b80490e439f88b14f34b8e71c3ac09a6f26f437 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:57:28 -0600 Subject: [PATCH 03/14] fix(repo): check suite bases in their repositories before a run - check_arms and check_mcp_configs read each available suite task's base in its scaffold repository; only suite gaps are skipped - a pin that resolves to another commit (a branch named like a hash) is refused, except under --allow-floating-suite-revs, which resolves it - a repos key that is not a string is a SuiteError, not a TypeError - the non-string name check is a helper, so parse_suite stays under the complexity cap - the scaffold MCP test runs through repo run, so it fails when the suite base dirs are not passed to check_mcp_configs Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- dokimi/dokimi/commands/repo_run.py | 6 +-- dokimi/dokimi/repos/arms.py | 46 ++++++++++++---------- dokimi/dokimi/repos/bases.py | 13 +++++-- dokimi/dokimi/repos/grading.py | 6 +-- dokimi/dokimi/repos/suite.py | 19 +++++++++- dokimi/tests/test_repo_suite.py | 61 +++++++++++++++++++++++++++++- 6 files changed, 119 insertions(+), 32 deletions(-) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 97f0e3f69..09f010450 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -240,7 +240,7 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) arms = [a for spec in args.arm or () for a in spec.split(",") if a] check_arms([parse_arm(a) for a in arms], ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, - ctx.manifest.id, ctx.suite_tasks()) # skill paths exist, catalogs hold each expected_skill + ctx.manifest.id, ctx.base_dirs()) # skill paths exist, catalogs hold each expected_skill if arms: print(f"{len(arms)} arm(s) ok: {', '.join(arms)}", file=sys.stderr) print(f"{len(tasks)} task(s), image {ctx.image}{' (warm: with network)' if args.warm else ''}", @@ -377,8 +377,8 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock ctx, _ = _context(args, batch_dir, source, pin=True) tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) check_arms(arms, ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id, - ctx.suite_tasks()) # skill paths exist, catalogs hold each expected_skill - check_mcp_configs(ctx.git_dir, [t for t in tasks if t.id not in ctx.suite_tasks()], profile, args.tools) # a malformed .mcp.json fails here, not in every cell + ctx.base_dirs()) # skill paths exist, catalogs hold each expected_skill + check_mcp_configs(ctx.git_dir, tasks, profile, args.tools, ctx.base_dirs()) # a malformed .mcp.json fails here, not in every cell skill_pin = pin_skills(batch_dir, arms, ctx.content_dir) # a resume may not swap an arm's skills validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, diff --git a/dokimi/dokimi/repos/arms.py b/dokimi/dokimi/repos/arms.py index 6aee65632..3e26ae63e 100644 --- a/dokimi/dokimi/repos/arms.py +++ b/dokimi/dokimi/repos/arms.py @@ -21,7 +21,7 @@ import hashlib import os import re -from collections.abc import Collection +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -193,7 +193,7 @@ def _base_skill_files(git_dir: Path, base: str) -> tuple[list[str], list[str]]: def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Path | None = None, guidance_globs: tuple[str, ...] | list[str] = (), repo_id: str | None = None, - elsewhere: Collection[str] = ()) -> None: + base_dirs: Mapping[str, Path | None] = {}) -> None: """Find the skills of every parsed arm before anything runs; a catalog must hold each task's ``expected_skill``. Two arms that would share a name or a result directory are refused, and so is a skill the base of a task (in ``git_dir``) already ships, which preparing the workspace would refuse cell by cell. @@ -203,10 +203,11 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa arm writes its own skill, ``.claude/skills/-guidance``: a skill of the arm with that name, or a file the base tracks under it, is refused here (``repo_id`` given). The files of a ``patch:`` arm's patch and of a task's ``setup_patches`` are not read: a skill directory they add is refused only in - preparation. The bases of the tasks in ``elsewhere`` (a suite's) are not in ``git_dir`` and are not read.""" - own = [t for t in tasks if t.id not in elsewhere] # the bases read from git_dir + preparation. ``base_dirs`` maps a suite task to the git dir of its base: that base is read there, and a task + mapped to None (its repository is missing) is not read.""" + own = [(t, base_dirs[t.id] if t.id in base_dirs else git_dir) for t in tasks] # (task, git dir of its base) cells: dict[str, str] = {} # cell-name slug of an arm -> the arm spec that owns it - shipped: dict[str, tuple[list[str], list[str]]] = {} # base -> (tracked files under .claude/skills, its links) + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]] = {} # (git dir, commit) -> (tracked files under .claude/skills, its links) for arm in arms: cell = slug(str(arm).replace(":", "-")) if cells.setdefault(cell, arm.spec) != arm.spec: @@ -220,14 +221,15 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa drops = arm.patch is None and (arm.base in ("guidance:none", "guidance:as-skill") or arm.rev is not None) rev = (arm.rev_sha or arm.rev) if arm.patch is None else None if skills and git_dir is not None: - for task in own: - _check_shipped(arm, skills, task, git_dir, shipped, (drops, rev, guidance_globs)) + for task, base_dir in own: + if base_dir is not None: + _check_shipped(arm, skills, task, (git_dir, base_dir), shipped, (drops, rev, guidance_globs)) if any(kind == "skill-catalog" for kind, _ in arm.skills): _check_expected_skills(arm, tasks, names, generated) -def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: list[Any], git_dir: Path | None, - shipped: dict[str, tuple[list[str], list[str]]]) -> None: +def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: list[tuple[Any, Path | None]], + git_dir: Path | None, shipped: dict[tuple[Path, str], tuple[list[str], list[str]]]) -> None: """Refuse a ``guidance:as-skill`` arm whose generated skill directory is a skill of the arm, or is tracked by the base of a task, or whose base links ``.claude`` or ``.claude/skills`` (preparation installs through both): it would fail cell by cell, not here. A ``shipped`` cache entry holds the base's files and links.""" @@ -237,8 +239,10 @@ def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: l raise ArmError(f"arm {arm.spec!r}: its generated guidance skill {target} is also one of its skills") if git_dir is None: return - for task in tasks: - files, links = _cached_shipped(git_dir, task.base, shipped) + for task, base_dir in tasks: + if base_dir is None: + continue + files, links = _cached_shipped(base_dir, task.base, shipped) for path in files: if path == target or path.startswith(target + "/"): raise ArmError(f"arm {arm.spec!r}: the base of task {task.id} already has {target}") @@ -247,15 +251,17 @@ def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: l f"refusing to install {target} through it") -def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dir: Path, - shipped: dict[str, tuple[list[str], list[str]]], +def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dirs: tuple[Path, Path], + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]], dropping: tuple[bool, str | None, tuple[str, ...] | list[str]]) -> None: """Refuse ``skills`` that the base of ``task`` (or the arm's guidance revision) already ships, or a link that - preparation would refuse to install through; ``shipped`` caches ``_base_skill_files`` per commit.""" + preparation would refuse to install through; ``git_dirs`` is the git dir of the guidance revision and that of + the task's base; ``shipped`` caches ``_base_skill_files`` per git dir and commit.""" from dokimi.repos.manifest import match_glob drops, rev, guidance_globs = dropping - base_files, base_links = _cached_shipped(git_dir, task.base, shipped) + git_dir, base_dir = git_dirs + base_files, base_links = _cached_shipped(base_dir, task.base, shipped) if rev is not None: rev_files = _cached_shipped(git_dir, rev, shipped)[0] left = [p for p in base_files if not (drops and any(match_glob(g, p) for g in guidance_globs))] @@ -270,11 +276,11 @@ def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dir: Path, def _cached_shipped(git_dir: Path, commit: str, - shipped: dict[str, tuple[list[str], list[str]]]) -> tuple[list[str], list[str]]: - """``_base_skill_files`` of a commit, one ``git ls-tree`` per commit for the whole check.""" - if commit not in shipped: - shipped[commit] = _base_skill_files(git_dir, commit) - return shipped[commit] + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]]) -> tuple[list[str], list[str]]: + """``_base_skill_files`` of a commit, one ``git ls-tree`` per git dir and commit for the whole check.""" + if (git_dir, commit) not in shipped: + shipped[git_dir, commit] = _base_skill_files(git_dir, commit) + return shipped[git_dir, commit] def _check_expected_skills(arm: Any, tasks: list[Any], names: set[str], generated: str | None = None) -> None: diff --git a/dokimi/dokimi/repos/bases.py b/dokimi/dokimi/repos/bases.py index c9f20e63d..3ff7d9c77 100644 --- a/dokimi/dokimi/repos/bases.py +++ b/dokimi/dokimi/repos/bases.py @@ -14,6 +14,7 @@ import json import tempfile +from collections.abc import Mapping from pathlib import Path from typing import Any, Callable @@ -58,13 +59,17 @@ def tree_mcp_error(tree: RepoTree) -> str | None: return None -def check_mcp_configs(git_dir: Any, tasks: list[Any], profile: Any, tools: str) -> None: +def check_mcp_configs(git_dir: Any, tasks: list[Any], profile: Any, tools: str, + base_dirs: Mapping[str, Any] = {}) -> None: """Before a run: refuse a task base whose ``.mcp.json`` the profile would pass to the agent (the ``all`` tool - mode) and the agent's CLI would not accept, instead of every cell of the task failing as a task failure.""" + mode) and the agent's CLI would not accept, instead of every cell of the task failing as a task failure. + A task in ``base_dirs`` (a suite's) is read in the git dir it maps to; one mapped to None is not read.""" if not profile.applied_args(tools, frozenset({MCP_CONFIG})): return - for base in dict.fromkeys(t.base for t in tasks): - problem = tree_mcp_error(GitTree(git_dir, base)) + for where, base in dict.fromkeys((base_dirs.get(t.id, git_dir) if base_dirs else git_dir, t.base) for t in tasks): + if where is None: + continue + problem = tree_mcp_error(GitTree(where, base)) if problem: raise ValueError(f"{MCP_CONFIG} at task base {base} is {problem}; `--tools all` passes it to the agent, " "which exits on it before its first turn. Fix or remove the file, or use --tools read/none") diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py index 546994980..d2c4527b4 100644 --- a/dokimi/dokimi/repos/grading.py +++ b/dokimi/dokimi/repos/grading.py @@ -96,9 +96,9 @@ def git_dir_for(self, task_id: str) -> Path: """The git dir a task's base is checked out from: its suite repository, else ``git_dir``.""" return self.suite.git_dirs.get(task_id, self.git_dir) if self.suite else self.git_dir - def suite_tasks(self) -> set[str]: - """The tasks of the suite: their base is not in ``git_dir``, so nothing reads it there.""" - return set(self.suite.git_dirs) | set(self.suite.gaps) if self.suite else set() + def base_dirs(self) -> dict[str, Path | None]: + """The git dir of each suite task's base, None for a task whose repository is missing.""" + return {**dict.fromkeys(self.suite.gaps), **self.suite.git_dirs} if self.suite else {} def base_gaps(self) -> dict[str, str]: """Why each suite task has no base checkout; its cells are unsupported, not run.""" diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index abeb616e8..81c1751ef 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -47,6 +47,7 @@ class SuiteManifest: id: str repos: dict[str, RepoSource] tasks: dict[str, str] + floating: bool = False # the run allows a branch, tag or other ref as a pin @dataclass @@ -69,6 +70,7 @@ class SuiteBases: def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) -> SuiteManifest: """Validate an already-parsed suite mapping. A branch, tag or other ref is refused as a pin unless ``allow_floating``, in which case it is resolved when the run starts and the batch pins the result.""" + _refuse_odd_names(raw, path) errors = schema_errors(SCHEMA_VERSION, raw) if errors: raise SuiteError(path, errors) @@ -98,7 +100,15 @@ def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) - tasks[task_id] = repo if problems: raise SuiteError(path, problems) - return SuiteManifest(path=path, id=raw["id"], repos=repos, tasks=tasks) + return SuiteManifest(path=path, id=raw["id"], repos=repos, tasks=tasks, floating=allow_floating) + + +def _refuse_odd_names(raw: Any, path: str) -> None: + """A ``repos`` name that is not a string is a suite error; ``schema_errors`` cannot sort it among the others.""" + repos = raw.get("repos") if isinstance(raw, dict) else None + odd = [k for k in repos if not isinstance(k, str)] if isinstance(repos, dict) else [] + if odd: + raise SuiteError(path, [f"repos: the name {k!r} is not a string" for k in odd]) def _parse_repo(spec: str) -> RepoSource: @@ -153,6 +163,13 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat each, all of them, so the pins do not depend on which tasks a run selects. """ checkouts = {name: _checkout(source, cache_dir) for name, source in suite.repos.items()} + for name, found in checkouts.items(): # a name shaped like a hash (a branch) is not the commit the pin says + if suite.floating: # the run asked for refs to be resolved: the commit it found is the pin + break + rev = suite.repos[name].rev + if not isinstance(found, str) and rev and _SHA_RE.match(rev) and found[1] != rev: + raise SuiteError(suite.path, [f"repos.{name}: {rev!r} resolves to {found[1]}, not to that commit; " + "pin the full commit SHA"]) out = SuiteBases(pins={name: found[1] for name, found in checkouts.items() if not isinstance(found, str)}, plan={"path": suite.path, "repos": {name: str(source) for name, source in suite.repos.items()}, "tasks": dict(suite.tasks)}) diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 516089818..b87604553 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -16,7 +16,7 @@ from dokimi.repos.source import RepoSource, open_source from dokimi.repos.suite import SuiteError, load_suite, parse_suite, resolve_suite from dokimi.utils import InputError -from repo_eval_fixture import FakeRunner, fake_harness, git, make_repo, write +from repo_eval_fixture import PROFILE, FakeRunner, fake_harness, git, make_repo, write EXAMPLE = Path(__file__).resolve().parent.parent / "data" / "suites" / "example-suite.yaml" PINNED = "https://example.invalid/scaffold@" + "a" * 40 @@ -72,6 +72,7 @@ def test_a_task_must_name_a_declared_repo_and_appear_once(): {"repos": {}}, {"tasks": [{"id": "a,b", "repo": "sc"}]}, {"tasks": [{"id": "t1"}]}, + {"repos": {1: PINNED, "sc": PINNED}}, ]) def test_the_schema_refuses_a_malformed_suite(change): with pytest.raises(SuiteError): @@ -85,6 +86,11 @@ def test_a_suite_file_missing_from_the_tree_is_refused(tmp_path): load_suite(tree, ".agent-eval/suite.yaml") +def test_a_repo_name_that_is_not_a_string_is_a_suite_error_and_names_it(): + with pytest.raises(SuiteError, match="the name 1 is not a string"): + parse_suite(_raw(repos={1: None, "sc": PINNED})) + + def _scaffold(tmp_path: Path, name: str = "scaffold") -> tuple[Path, str]: root = tmp_path / name write(root, "src/calc.py", CALC) @@ -116,6 +122,26 @@ def test_a_floating_branch_resolves_once_to_the_commit_it_names(tmp_path): assert found.pins == {"sc": sha} and found.gaps == {} +def test_a_branch_named_like_a_hash_resolves_when_the_run_allows_floating_revs(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "branch", "b" * 64, sha) + suite = parse_suite(_raw(repos={"sc": f"{root}@{'b' * 64}"}, tasks=[{"id": "t1", "repo": "sc"}]), + allow_floating=True) + assert resolve_suite(suite, {"t1": sha}, tmp_path / "cache").pins == {"sc": sha} + + +def test_a_branch_named_like_a_hash_is_not_the_pinned_commit(tmp_path): + root, sha = _scaffold(tmp_path) + write(root, "src/more.py", "x = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "more") + fake = "b" * 64 # a SHA-1 repository: this is a branch, whatever its spelling + git(root, "branch", fake, sha) + suite = parse_suite(_raw(repos={"sc": f"{root}@{fake}"})) + with pytest.raises(SuiteError, match="resolves to .*not to that commit"): + resolve_suite(suite, {"t1": sha}, tmp_path / "cache") + + def test_the_suite_file_is_read_from_the_eval_tree(tmp_path): root, _ = make_repo(tmp_path) write(root, ".agent-eval/suite.yaml", yaml.safe_dump(_raw())) @@ -335,3 +361,36 @@ def test_a_suite_gap_is_not_checked_even_when_the_task_also_needs_egress(tmp_pat "--task", "net_add", tasks=net) assert code == 0 assert report["tasks"][0]["unsupported_by"] == "suite" + + +def test_a_skill_the_scaffold_base_ships_is_refused_before_any_cell(run, capsys): + scaffold, _ = _scaffold(run["tmp"]) + write(scaffold, ".claude/skills/lint/SKILL.md", "---\nname: lint\ndescription: x\n---\nLint.\n") + git(scaffold, "add", "-A") + git(scaffold, "commit", "-qm", "ships lint") + sha = git(scaffold, "rev-parse", "HEAD") + write(run["root"], ".agent-eval/skills/lint/SKILL.md", "---\nname: lint\ndescription: x\n---\nLint.\n") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:raw+skill:skills/lint") + assert exc.value.code == 1 + assert "the base of task fix_add already has .claude/skills/lint" in capsys.readouterr().err + assert not list((run["out"] / "b1").glob("*__*/result.json")) + + +def test_a_malformed_mcp_config_in_a_scaffold_base_is_refused_before_any_cell(run, capsys): + profile = run["tmp"] / "fake.yaml" # a harness that passes the repository's .mcp.json to the agent + profile.write_text(yaml.safe_dump({**PROFILE, "args_if_present": {".mcp.json": ["--mcp-config", ".mcp.json"]}}), + encoding="utf-8") + scaffold, _ = _scaffold(run["tmp"]) + write(scaffold, ".mcp.json", "{") + git(scaffold, "add", "-A") + git(scaffold, "commit", "-qm", "bad mcp") + sha = git(scaffold, "rev-parse", "HEAD") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", "guidance:none") + assert exc.value.code == 1 + assert "not valid JSON" in capsys.readouterr().err + assert not list((run["out"] / "b1").glob("*__*/result.json")) From f7110ebb3a11ae751567aaa55e42067c999669f9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:35:47 -0800 Subject: [PATCH 04/14] Address review: record the suite fixes already on this branch The review threads on this PR are answered by commits already on the branch (c8732661c and 4b80490e4) and by a merge of intel/main. This commit names them. Thread: PRRT_kwDOVBg7g863m7sG: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sI: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sJ: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sN: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sQ: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sS: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sT: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sV: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sX: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sa: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863m7sd: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUe: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUg: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUk: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUn: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUq: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUr: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUt: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863paUw: already fixed in c8732661c Thread: PRRT_kwDOVBg7g863sVHe: already fixed in 4b80490e4 Thread: PRRT_kwDOVBg7g863sVHh: already fixed in 4b80490e4 Thread: PRRT_kwDOVBg7g863sVHk: already fixed in 4b80490e4 Thread: PRR_kwDOVBg7g88AAAABYE3Cvg: already fixed in c8732661c Thread: IC_kwDOVBg7g88AAAABm3m9qg: accept, PR description corrected to the resume behaviour Signed-off-by: Nikolay Petrov From 10dcfafcebfe8e5817318940140d4df72d08ada8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 15 Nov 2026 21:54:50 -0800 Subject: [PATCH 05/14] docs(repo): a resumed suite batch refuses a moved floating revision A floating suite revision is resolved when the run starts and pinned in the batch. A resume after the branch moved is refused, so the contract now says so instead of promising reuse. The PR description says the same. Thread: PRRT_kwDOVBg7g863m7sD: accept, contract changed to refuse the resume Thread: IC_kwDOVBg7g88AAAABmzV5cw: accept, same decision Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 9f6472e06..478d2f96d 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -337,11 +337,14 @@ tasks: task must be a task of `tasks` (§2), or the run stops before any cell. - Each repository is `@` or a local `@`, as in §1.1. The revision must be a full commit SHA. A branch, tag or other ref is refused - unless the run passes `--allow-floating-suite-revs`; it is then resolved once - when the run starts, and the batch pin (`_eval/batch.json`, key `suite`) - records the commits, so a resumed batch reuses them. The pin also records the - suite file's path, the task-to-repository mapping and the repository locations (key `suite_plan`): a - resume with another suite file, or without `--suite`, is refused. + unless the run passes `--allow-floating-suite-revs`; it is resolved again + when each run starts, and the batch pin (`_eval/batch.json`, key `suite`) + records the commits. A resume is refused with `use another --batch` when the + revision now resolves to another commit: the batch does not follow a + moved branch or tag, and it does not reuse a later commit. The pin also + records the suite file's path, the task-to-repository mapping and the + repository locations (key `suite_plan`): a resume with another suite file, or + without `--suite`, is refused. - A repository that cannot be opened, or a base that is not a commit of it, leaves that task's cells unsupported (§9): `unsupported.json` gives the reason, and `repo report` counts the cells neither as failed nor as missing. A cell From a4131e06466f9ad27b0fb48e3cf3dcc32c685878 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:21:20 -0800 Subject: [PATCH 06/14] fix(repo): refuse an empty --suite path instead of reading it as no suite An explicit empty path disabled suite mode, so a new run used eval-tree bases. Thread: PRRT_kwDOVBg7g863zTxO Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo_run.py | 4 ++-- dokimi/tests/test_repo_suite.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index ae70e0764..3e4a6d857 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -42,7 +42,7 @@ def _refuse_typed(source, args: argparse.Namespace) -> None: sys.exit(1) if args.manifest and not args.manifest_file: _refuse_escape(args.manifest) - if getattr(args, "suite", None): + if getattr(args, "suite", None) is not None: # an empty path is given, not omitted: refused below, not skipped _refuse_escape(args.suite) @@ -133,7 +133,7 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): def _suite_bases(args: argparse.Namespace, tree, content: Path, manifest: RepoManifest, cache_dir: Path | None): """The base checkouts of ``--suite`` (contract §2.1), or None: every base is then in the eval source.""" - if not args.suite: + if args.suite is None: return None from dokimi.repos.suite import SuiteError, load_suite, resolve_suite from dokimi.repos.workspace import PrepError, load_tasks diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index b87604553..df9f01b21 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -394,3 +394,24 @@ def test_a_malformed_mcp_config_in_a_scaffold_base_is_refused_before_any_cell(ru assert exc.value.code == 1 assert "not valid JSON" in capsys.readouterr().err assert not list((run["out"] / "b1").glob("*__*/result.json")) + + +def test_an_empty_suite_path_is_refused_by_repo_run_and_not_read_as_no_suite(run, capsys): + _commit_suite(run["root"], None, _raw(repos={"sc": f"file://{run['tmp']}/missing@{'c' * 40}"}, + tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", "", "--task", "review_add", "--arm", "guidance:none") + assert exc.value.code == 1 + assert not (run["out"] / "b1" / "review_add__guidance-none__m-1__r1/result.json").exists() + + +def test_an_empty_suite_path_is_refused_by_self_check_too(tmp_path, monkeypatch, capsys): + root, _ = make_repo(tmp_path) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + out = tmp_path / "check-out" + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(out), "--cache-root", + str(out / "caches"), "--suite", "", "--task", "review_add"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + assert not (out / "check/check.json").exists() From d72786d2d3a1b89192f1be900476cc842aa86954 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:21:39 -0800 Subject: [PATCH 07/14] fix(repo): a suite gap is unsupported in the batch, not a run abort A task whose suite base is missing stopped the whole batch under no network. Thread: PRRT_kwDOVBg7g863zTxX Signed-off-by: Nikolay Petrov --- dokimi/dokimi/commands/repo_run.py | 5 ++++- dokimi/tests/test_repo_suite.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index 3e4a6d857..475fdc19b 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -385,7 +385,10 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] - for task in tasks: # refuse before anything runs, not cell by cell + gaps = ctx.base_gaps() + for task in tasks: # refuse before anything runs, not cell by cell; a suite gap is unsupported, not run + if task.id in gaps: + continue task_caps(task, options[0]) agent_network(options[0], ctx, task) docker_network(ctx.manifest.image.network, task.egress) # grade has no egress route either diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index df9f01b21..1d0073c6a 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -415,3 +415,17 @@ def test_an_empty_suite_path_is_refused_by_self_check_too(tmp_path, monkeypatch, args.func(args) assert exc.value.code == 1 assert not (out / "check/check.json").exists() + + +def test_a_suite_gap_with_egress_does_not_abort_the_batch_under_no_network(run, capsys): + task = run["root"] / ".agent-eval/tasks/fix_add/task.json" + write(run["root"], ".agent-eval/tasks/fix_add/task.json", + json.dumps({**json.loads(task.read_text(encoding="utf-8")), "egress": ["example.com"]})) + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(run["root"], None, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + unsupported = json.loads((batch / "fix_add__guidance-none__m-1__r1/unsupported.json").read_text(encoding="utf-8")) + assert unsupported["task"] == "fix_add" + assert (batch / "review_add__guidance-none__m-1__r1/result.json").is_file() From 43a1fe947300dcfed73536fb496394ff5dc9649b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:22:44 -0800 Subject: [PATCH 08/14] fix(repo): a floating suite pin re-fetches a hash-shaped branch before it resolves A branch named like a commit kept its old pin in the warm cache after it moved. Thread: PRRT_kwDOVBg7g863zTxY Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/source.py | 13 +++++++------ dokimi/dokimi/repos/suite.py | 7 ++++--- dokimi/tests/test_repo_suite.py | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/dokimi/dokimi/repos/source.py b/dokimi/dokimi/repos/source.py index 78c2ca579..f71687c50 100644 --- a/dokimi/dokimi/repos/source.py +++ b/dokimi/dokimi/repos/source.py @@ -424,8 +424,8 @@ def check_pinned_root(source: RepoSource) -> None: raise SourceArgumentError(f"{source.location} is {prefix!r} inside a repository; give its root") -def open_source(source: RepoSource, cache_dir: Path | None = None) -> RepoTree: - """Return a readable tree for ``source``, cloning or fetching as needed.""" +def open_source(source: RepoSource, cache_dir: Path | None = None, refresh: bool = False) -> RepoTree: + """Return a readable tree for ``source``, cloning or fetching as needed; ``refresh`` re-fetches its refs.""" if not source.remote: if source.rev is None: return WorkTree(source.location) @@ -434,7 +434,7 @@ def open_source(source: RepoSource, cache_dir: Path | None = None) -> RepoTree: return GitTree(Path(source.location), sha, label=f"{source.location}@{sha[:12]}") git_dir = ensure_clone(source.location, cache_dir or default_cache_dir()) assert source.rev is not None # parse() refuses unpinned remotes - sha = ensure_rev(git_dir, source.rev) + sha = ensure_rev(git_dir, source.rev, refresh) return GitTree(git_dir, sha, label=f"{source.location}@{sha[:12]}") @@ -493,10 +493,11 @@ def ensure_clone(url: str, cache_dir: Path) -> Path: return git_dir -def ensure_rev(git_dir: Path, rev: str) -> str: - """Resolve ``rev`` in the cached clone; a branch or tag is re-fetched on every call.""" +def ensure_rev(git_dir: Path, rev: str, refresh: bool = False) -> str: + """Resolve ``rev`` in the cached clone; a branch or tag is re-fetched on every call. ``refresh`` also + re-fetches a hash-shaped name, which may be a branch that moved (a floating suite pin).""" base = re.split(r"[\^~]", rev, maxsplit=1)[0] - if _SHA_RE.match(base): + if _SHA_RE.match(base) and not refresh: try: return resolve_rev(git_dir, rev) except SourceError: diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index 81c1751ef..434366090 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -134,10 +134,10 @@ def load_suite(tree: RepoTree, path: str, allow_floating: bool = False) -> Suite return parse_suite(raw, path, allow_floating) -def _checkout(source: RepoSource, cache_dir: Path | None) -> tuple[Path, str] | str: +def _checkout(source: RepoSource, cache_dir: Path | None, refresh: bool = False) -> tuple[Path, str] | str: """``(git dir, commit)`` of a repository of the suite, or the reason it cannot be opened.""" try: - tree = open_source(source, cache_dir) + tree = open_source(source, cache_dir, refresh=refresh) git_dir = source_git_dir(source, tree, cache_dir) except (ValueError, OSError, PrepError) as exc: # SourceError is a ValueError return f"{source} is unavailable: {exc}" @@ -162,7 +162,8 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat ``bases`` maps each suite task to the ``base`` of its task.json. Repositories are opened once each, all of them, so the pins do not depend on which tasks a run selects. """ - checkouts = {name: _checkout(source, cache_dir) for name, source in suite.repos.items()} + checkouts = {name: _checkout(source, cache_dir, refresh=suite.floating and not _SHA_RE.match(source.rev or "")) + for name, source in suite.repos.items()} for name, found in checkouts.items(): # a name shaped like a hash (a branch) is not the commit the pin says if suite.floating: # the run asked for refs to be resolved: the commit it found is the pin break diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 1d0073c6a..9704568ff 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -429,3 +429,18 @@ def test_a_suite_gap_with_egress_does_not_abort_the_batch_under_no_network(run, unsupported = json.loads((batch / "fix_add__guidance-none__m-1__r1/unsupported.json").read_text(encoding="utf-8")) assert unsupported["task"] == "fix_add" assert (batch / "review_add__guidance-none__m-1__r1/result.json").is_file() + + +def test_a_moved_hash_shaped_floating_branch_is_resolved_again_not_from_the_warm_cache(tmp_path): + root, first = _scaffold(tmp_path) + git(root, "branch", "abcdef0", first) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@abcdef0"}), allow_floating=True) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": first}, cache).pins == {"sc": first} # warms the cache at the branch + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "moved") + moved = git(root, "rev-parse", "HEAD") + git(root, "branch", "-f", "abcdef0", moved) + found = resolve_suite(suite, {"t1": moved}, cache) + assert found.pins == {"sc": moved} and found.gaps == {} From db5e8df88567cf9867d4e5b38f2087291b43fe71 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 10:22:51 -0800 Subject: [PATCH 09/14] fix(repo): a suite base in a warm cache is fetched before it is a gap A base that is the parent of a pinned commit was recorded as unsupported when the remote cache held only the pin. Thread: PRRT_kwDOVBg7g863zTxd Thread: PRRT_kwDOVBg7g863z-U9 Review: PRR_kwDOVBg7g88AAAABYKBVag Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/suite.py | 5 +++-- dokimi/tests/test_repo_suite.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index 434366090..c8e0c22b0 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -150,9 +150,10 @@ def _resolve_base(source: RepoSource, git_dir: Path, base: str) -> str: try: return resolve_rev(git_dir, base) except SourceError: - if not source.remote or not _SHA_RE.match(base): + commit = re.split(r"[\^~]", base, maxsplit=1)[0] # a parent (``^``) is fetched by its commit + if not source.remote or not _SHA_RE.match(commit): raise - process.git("-C", str(git_dir), "fetch", "origin", "--", base) # a failure leaves the miss below + process.git("-C", str(git_dir), "fetch", "origin", "--", commit) # a failure leaves the miss below return resolve_rev(git_dir, base) diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 9704568ff..a6629033e 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -444,3 +444,18 @@ def test_a_moved_hash_shaped_floating_branch_is_resolved_again_not_from_the_warm git(root, "branch", "-f", "abcdef0", moved) found = resolve_suite(suite, {"t1": moved}, cache) assert found.pins == {"sc": moved} and found.gaps == {} + + +def test_a_parent_base_the_warm_cache_of_a_remote_lacks_is_fetched_not_a_gap(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + for name in ("parent", "child"): + write(root, "src/more.py", f"X = '{name}'\n") + git(root, "add", "-A") + git(root, "commit", "-qm", name) + child = git(root, "rev-parse", "HEAD") + found = resolve_suite(suite, {"t1": f"{child}^"}, cache) + assert found.gaps == {} and "t1" in found.git_dirs From 4c791c9498cc1c629912054290a54098c315c6b8 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 16 Nov 2026 11:19:28 -0800 Subject: [PATCH 10/14] Address review: fetch an abbreviated suite base; refresh floating pins A short SHA base missing from a warm remote cache is fetched by refreshing the branches and tags first, since git cannot fetch an abbreviated name. A floating pin is refreshed whatever its spelling, so a branch named like a full hash that moved is resolved again; a commit on no branch is still fetched by its name. Thread: PRRT_kwDOVBg7g864E4pz Thread: PRRT_kwDOVBg7g864E4qA Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/source.py | 6 ++++ dokimi/dokimi/repos/suite.py | 11 ++++++-- dokimi/tests/test_repo_suite.py | 50 +++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/repos/source.py b/dokimi/dokimi/repos/source.py index f71687c50..0b0fe7e28 100644 --- a/dokimi/dokimi/repos/source.py +++ b/dokimi/dokimi/repos/source.py @@ -511,6 +511,12 @@ def ensure_rev(git_dir: Path, rev: str, refresh: bool = False) -> str: try: return resolve_rev(git_dir, rev) except SourceError as exc: + if refresh and len(base) in (40, 64) and re.fullmatch(r"[0-9a-f]+", base): # a full commit on no branch + try: + _git(git_dir, "fetch", "--filter=blob:none", "origin", "--", base) + return resolve_rev(git_dir, rev) + except SourceError: + pass short = _SHA_RE.match(base) and len(base) < 40 hint = "; a short SHA on no branch or tag cannot be fetched, give the full SHA" if short else "" raise SourceError(f"revision not found in origin: {rev}{hint}") from exc diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index c8e0c22b0..0f6851958 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -28,6 +28,7 @@ SCHEMA_VERSION = "suite_manifest.v1" _SHA_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") # a full commit: a branch, tag, short SHA or other ref is not +_ABBREV_RE = re.compile(r"^[0-9a-f]{7,39}$") # a short commit SHA, as manifest._BASE_RE reads a task base class SuiteError(ValueError): @@ -151,9 +152,13 @@ def _resolve_base(source: RepoSource, git_dir: Path, base: str) -> str: return resolve_rev(git_dir, base) except SourceError: commit = re.split(r"[\^~]", base, maxsplit=1)[0] # a parent (``^``) is fetched by its commit - if not source.remote or not _SHA_RE.match(commit): + if not source.remote or not (_SHA_RE.match(commit) or _ABBREV_RE.match(commit)): raise - process.git("-C", str(git_dir), "fetch", "origin", "--", commit) # a failure leaves the miss below + if _SHA_RE.match(commit): + process.git("-C", str(git_dir), "fetch", "origin", "--", commit) # a failure leaves the miss below + else: # git cannot fetch a short SHA: its refs are fetched, then the base is looked up again; a failure leaves the miss + process.git("-C", str(git_dir), "fetch", "--filter=blob:none", "origin", + "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*") return resolve_rev(git_dir, base) @@ -163,7 +168,7 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat ``bases`` maps each suite task to the ``base`` of its task.json. Repositories are opened once each, all of them, so the pins do not depend on which tasks a run selects. """ - checkouts = {name: _checkout(source, cache_dir, refresh=suite.floating and not _SHA_RE.match(source.rev or "")) + checkouts = {name: _checkout(source, cache_dir, refresh=suite.floating) # a hash-shaped name may be a branch for name, source in suite.repos.items()} for name, found in checkouts.items(): # a name shaped like a hash (a branch) is not the commit the pin says if suite.floating: # the run asked for refs to be resolved: the commit it found is the pin diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index a6629033e..2f13f75d7 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -459,3 +459,53 @@ def test_a_parent_base_the_warm_cache_of_a_remote_lacks_is_fetched_not_a_gap(tmp child = git(root, "rev-parse", "HEAD") found = resolve_suite(suite, {"t1": f"{child}^"}, cache) assert found.gaps == {} and "t1" in found.git_dirs + + +@pytest.mark.parametrize("suffix", ["", "^"]) +def test_an_abbreviated_base_the_warm_cache_lacks_is_fetched_not_a_gap(tmp_path, suffix): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + short = git(root, "rev-parse", "--short=7", "HEAD") + found = resolve_suite(suite, {"t1": f"{short}{suffix}"}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + + +def test_a_floating_full_sha_pin_on_no_branch_is_fetched_by_name(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "checkout", "-q", "-b", "gone") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "on a branch that is deleted") + pin = git(root, "rev-parse", "HEAD") + git(root, "checkout", "-q", "main") + git(root, "branch", "-D", "gone") # the pinned commit is on no branch or tag any more + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{pin}"}), allow_floating=True) + found = resolve_suite(suite, {"t1": pin}, tmp_path / "cache") + assert found.pins == {"sc": pin} and found.gaps == {} + + +def test_a_moved_full_length_floating_branch_is_resolved_again_in_a_sha256_scaffold(tmp_path): + root = tmp_path / "sc256" + write(root, "src/calc.py", CALC) + git(root, "init", "-q", "-b", "main", "--object-format=sha256") + git(root, "add", "-A") + git(root, "commit", "-qm", "scaffold") + first = git(root, "rev-parse", "HEAD") + name = "a" * 40 # a branch in a SHA-256 repository: its spelling is not a commit + git(root, "branch", name, first) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{name}"}), allow_floating=True) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": first}, cache).pins == {"sc": first} # warms the cache at the branch + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "moved") + moved = git(root, "rev-parse", "HEAD") + git(root, "branch", "-f", name, moved) + found = resolve_suite(suite, {"t1": moved}, cache) + assert found.pins == {"sc": moved} and found.gaps == {} From 39554f3d7c83ee1069a87425fb3ba4b3d07e40fa Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 13:55:34 -0800 Subject: [PATCH 11/14] fix(repo): a short base leaves the full clone full, and the pin lists every repository A short-SHA base is fetched into the suite's full clone without a blob filter, since the filter would make every later fetch of that clone partial. A full-SHA pin is recorded for its repository whether or not the repository opened, so a repository that is down does not change the batch pin of a resume. Thread: PRRT_kwDOVBg7g864ZRyU Thread: PRRT_kwDOVBg7g864ZRyZ Review: PRR_kwDOVBg7g88AAAABYeMkog Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/suite.py | 8 ++++++-- dokimi/tests/test_repo_suite.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index 0f6851958..ad3a7c24a 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -157,7 +157,8 @@ def _resolve_base(source: RepoSource, git_dir: Path, base: str) -> str: if _SHA_RE.match(commit): process.git("-C", str(git_dir), "fetch", "origin", "--", commit) # a failure leaves the miss below else: # git cannot fetch a short SHA: its refs are fetched, then the base is looked up again; a failure leaves the miss - process.git("-C", str(git_dir), "fetch", "--filter=blob:none", "origin", + # no --filter here: on the full clone it would make every later fetch partial + process.git("-C", str(git_dir), "fetch", "origin", "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*") return resolve_rev(git_dir, base) @@ -177,7 +178,10 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat if not isinstance(found, str) and rev and _SHA_RE.match(rev) and found[1] != rev: raise SuiteError(suite.path, [f"repos.{name}: {rev!r} resolves to {found[1]}, not to that commit; " "pin the full commit SHA"]) - out = SuiteBases(pins={name: found[1] for name, found in checkouts.items() if not isinstance(found, str)}, + # a full-SHA pin is the declared commit, whether or not its repository opened + pins = {name: suite.repos[name].rev for name in suite.repos if not suite.floating} + pins.update({name: found[1] for name, found in checkouts.items() if suite.floating and not isinstance(found, str)}) + out = SuiteBases(pins=pins, plan={"path": suite.path, "repos": {name: str(source) for name, source in suite.repos.items()}, "tasks": dict(suite.tasks)}) for task_id, repo in suite.tasks.items(): diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 2f13f75d7..412726b0a 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -7,6 +7,7 @@ from __future__ import annotations import json +import subprocess from pathlib import Path import pytest @@ -476,6 +477,33 @@ def test_an_abbreviated_base_the_warm_cache_lacks_is_fetched_not_a_gap(tmp_path, assert found.gaps == {} and "t1" in found.git_dirs +def test_a_short_base_fetch_leaves_the_full_clone_without_a_partial_filter(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "config", "uploadpack.allowFilter", "true") + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the full clone at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + short = git(root, "rev-parse", "--short=7", "HEAD") + found = resolve_suite(suite, {"t1": short}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + promisor = subprocess.run(["git", "-C", str(found.git_dirs["t1"]), "config", "--get", "remote.origin.promisor"], + capture_output=True, text=True, check=False) + assert promisor.returncode != 0 # the full clone is not a partial clone + + +def test_a_full_sha_pin_is_recorded_even_when_its_repository_is_down(tmp_path): + good, sha = _scaffold(tmp_path) + gone = "b" * 40 + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"{tmp_path}/missing@{gone}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "lost", "repo": "gone"}])) + found = resolve_suite(suite, {"ok": sha, "lost": gone}, tmp_path / "cache") + assert found.pins == {"good": sha, "gone": gone} + + def test_a_floating_full_sha_pin_on_no_branch_is_fetched_by_name(tmp_path): root, sha = _scaffold(tmp_path) git(root, "checkout", "-q", "-b", "gone") From a7935542d7603b6f2b0e1011b346522f11be79b3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:01:38 -0800 Subject: [PATCH 12/14] test(repo): expect the pin of a down repository in the suite test The pin lists every declared repository since 39554f3d7; this test still expected only the opened one. Found by the local ci-local run. Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 412726b0a..349320103 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -110,7 +110,7 @@ def test_each_task_takes_its_base_from_the_repo_it_names_and_a_gap_is_recorded(t found = resolve_suite(suite, bases, tmp_path / "cache") assert set(found.git_dirs) == {"ok"} and found.git_dirs["ok"].is_dir() assert found.meta == {"ok": {"repo": "good", "source": f"{good}@{sha}"}} - assert found.pins == {"good": sha} + assert found.pins == {"good": sha, "gone": "b" * 40} # every declared pin, the down repository's too assert "is not a commit of good" in found.gaps["absent"] assert found.gaps["lost"].startswith("base repository gone:") and "unavailable" in found.gaps["lost"] From 40e596fe7f9bd6efb97c3aa1857a67584bab424e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 14:24:42 -0800 Subject: [PATCH 13/14] fix(repo): narrow the suite pin type so mypy passes A declared repository always has a rev (validation refuses none), so the pin is the rev wherever it is set. The mypy job of ci-local failed on the old comprehension. Signed-off-by: Nikolay Petrov --- dokimi/dokimi/repos/suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py index ad3a7c24a..0a59690b6 100644 --- a/dokimi/dokimi/repos/suite.py +++ b/dokimi/dokimi/repos/suite.py @@ -179,7 +179,7 @@ def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Pat raise SuiteError(suite.path, [f"repos.{name}: {rev!r} resolves to {found[1]}, not to that commit; " "pin the full commit SHA"]) # a full-SHA pin is the declared commit, whether or not its repository opened - pins = {name: suite.repos[name].rev for name in suite.repos if not suite.floating} + pins = {name: source.rev for name, source in suite.repos.items() if not suite.floating and source.rev is not None} pins.update({name: found[1] for name, found in checkouts.items() if suite.floating and not isinstance(found, str)}) out = SuiteBases(pins=pins, plan={"path": suite.path, "repos": {name: str(source) for name, source in suite.repos.items()}, From d1229e19ddf2ee1242e44fcb860add6b1d83a720 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 17 Nov 2026 15:21:49 -0800 Subject: [PATCH 14/14] fix(repo): tasks check exits 1 when every selected task is a suite gap A self-check that checked nothing is incomplete, as repo run reads it, not a pass. Thread: PRRT_kwDOVBg7g864ZRyi Signed-off-by: Nikolay Petrov --- .../20261114_120000_napetrov_suite-manifest.md | 2 +- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/commands/repo_run.py | 5 ++++- dokimi/tests/test_repo_suite.py | 9 +++++---- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md index 860462a8c..370dadc49 100644 --- a/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md +++ b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Added -- **Suite manifests for tasks whose base is in another repository** — `suite_manifest.v1` lists eval-tree tasks whose base is a commit of a scaffold repository, referenced as `@`. `repo run --suite ` resolves each base there; a repository that cannot be opened leaves its cells unsupported, not failed. Floating refs need `--allow-floating-suite-revs`. See `docs/repo-evaluation-contract.md` §2.1. +- **Suite manifests for tasks whose base is in another repository** — `suite_manifest.v1` lists eval-tree tasks whose base is a commit of a scaffold repository, referenced as `@`. `repo run --suite ` resolves each base there; a repository that cannot be opened leaves its cells unsupported, not failed. Floating refs need `--allow-floating-suite-revs`. `repo tasks check` exits 1 (incomplete) when every selected task is such a gap, since nothing was checked. See `docs/repo-evaluation-contract.md` §2.1. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 6b09ae25e..33ab5727b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -353,7 +353,7 @@ tasks: the cached clone of a remote lacks is fetched once before it counts as missing. - `repo tasks check` reports these tasks as not checked when their repository is missing and fails only on the other bad tasks; when every selected task is - not checked it exits 0. + not checked it exits 1 (incomplete), as `repo run` does, since nothing was checked. - Not covered yet: `repo validate` checks the bases of the eval tree only; a suite task's `{source_dir}` is still the eval-tree source; and a `guidance:rev:` arm is pinned against the eval-tree source. diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index d2ed39282..8097355a4 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -255,6 +255,9 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: unavailable = [r["task"] for r in bad if r.get("unsupported_by") == "suite"] if unavailable: # the base's repository of --suite is missing: not checked, and not a failure print(f"{len(unavailable)} not checked, the repository of their base is missing: " + ", ".join(unavailable)) + if results and len(unavailable) == len(results): # a gate that checked nothing is incomplete, not a pass + print("✗ incomplete: no task was checked, the repository of every base is missing", file=sys.stderr) + sys.exit(1) allowed = [r for r in bad if args.allow_unsupported and r.get("unsupported_by") == "host"] if allowed: print(f"{len(allowed)} unsupported on this host, allowed by --allow-unsupported: " @@ -537,7 +540,7 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit "pinned local path inside a repository (give its root) or a --cache-dir that is not a usable directory; exit 1 for a source that cannot be opened " "(clone, fetch, revision) and " ) -CHECK_EXIT = _SOURCE_EXIT + "for a task that fails the check." +CHECK_EXIT = _SOURCE_EXIT + "for a task that fails the check, or when no task was checked (every one a suite gap)." RUN_EXIT = _SOURCE_EXIT + "for a cell that crashed, a batch budget stop or an incomplete batch." diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py index 349320103..6268c0faf 100644 --- a/dokimi/tests/test_repo_suite.py +++ b/dokimi/tests/test_repo_suite.py @@ -345,22 +345,23 @@ def _check(tmp: Path, monkeypatch, suite: dict, *extra: str, tasks: dict | None return code, json.loads((out / "check/check.json").read_text(encoding="utf-8")) -def test_a_self_check_where_every_selected_task_is_a_suite_gap_exits_0(tmp_path, monkeypatch, capsys): +def test_a_self_check_where_every_selected_task_is_a_suite_gap_exits_1(tmp_path, monkeypatch, capsys): gone = f"file://{tmp_path}/missing@{'c' * 40}" code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}]), "--task", "fix_add") captured = capsys.readouterr() - assert code == 0 and "incomplete" not in captured.err + assert code == 1 and "incomplete: no task was checked" in captured.err assert [(r["task"], r["unsupported_by"]) for r in report["tasks"]] == [("fix_add", "suite")] -def test_a_suite_gap_is_not_checked_even_when_the_task_also_needs_egress(tmp_path, monkeypatch): +def test_a_suite_gap_is_not_checked_even_when_the_task_also_needs_egress(tmp_path, monkeypatch, capsys): gone = f"file://{tmp_path}/missing@{'c' * 40}" net = {"net_add": {"family": "review", "grader": "review", "review_patch": "change.patch", "egress": ["example.com"], "score": {"metric": "recall", "max": 1.0}, "prompt": "Review HEAD."}} code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "net_add", "repo": "sc"}]), "--task", "net_add", tasks=net) - assert code == 0 + assert code == 1 + assert "incomplete: no task was checked" in capsys.readouterr().err assert report["tasks"][0]["unsupported_by"] == "suite"