diff --git a/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md new file mode 100644 index 000000000..370dadc49 --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_napetrov_suite-manifest.md @@ -0,0 +1,8 @@ + + +### Added + +- **Suite manifests for tasks whose base is in another repository** — `suite_manifest.v1` lists eval-tree tasks whose base is a commit of a scaffold repository, referenced as `@`. `repo run --suite ` resolves each base there; a repository that cannot be opened leaves its cells unsupported, not failed. Floating refs need `--allow-floating-suite-revs`. `repo tasks check` exits 1 (incomplete) when every selected task is such a gap, since nothing was checked. See `docs/repo-evaluation-contract.md` §2.1. diff --git a/dokimi/data/suites/example-suite.yaml b/dokimi/data/suites/example-suite.yaml new file mode 100644 index 000000000..6ebd9b6ae --- /dev/null +++ b/dokimi/data/suites/example-suite.yaml @@ -0,0 +1,12 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Example suite_manifest.v1 (contract §2.1): `repos` pins each scaffold, `tasks` names the tasks whose base is in one. +schema: suite_manifest.v1 +id: linreg-cpu-solver +repos: + sklearn-scaffold: https://github.com/example-org/sklearn-scaffold@9f8e7d6c5b4a39281706f5e4d3c2b1a098765432 + numpy-scaffold: https://github.com/example-org/numpy-scaffold@0a1b2c3d4e5f60718293a4b5c6d7e8f901234567 +tasks: + - {id: solver-pick-cholesky, repo: sklearn-scaffold} + - {id: solver-pick-lstsq, repo: numpy-scaffold} diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index a9b377d5b..e9d29cc40 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -565,6 +565,7 @@ "allow_abbrev": true, "params": [ {"key": "source", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, + {"key": "--allow-floating-suite-revs", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--allow-unsupported", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--arm", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--build-network", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -582,6 +583,7 @@ {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": "repo-check", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--suite", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--task", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--warm", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null} ], @@ -1372,6 +1374,7 @@ "params": [ {"key": "source", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": null}, {"key": "--agent-network", "action": "_StoreAction", "choices": ["'bridge'", "'host'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--allow-floating-suite-revs", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--arm", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--batch", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": null}, {"key": "--bin", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, @@ -1398,6 +1401,7 @@ {"key": "--rebuild-image", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--reps", "action": "_StoreAction", "choices": null, "const": null, "default": 1, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--source-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--suite", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--task", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--timeout", "action": "_StoreAction", "choices": null, "const": null, "default": 2700, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--tools", "action": "_StoreAction", "choices": null, "const": null, "default": "all", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index e0b1be55b..33ab5727b 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -21,8 +21,8 @@ SPDX-License-Identifier: Apache-2.0 > Implemented so far (slice S1): the `repo_manifest.v1` schema, validation of > the manifest and of every `task.json` against one repository snapshot > (with git history, every task base must resolve and the guidance globs are -> evaluated at each base), the `repo_static.v1` schema (§4.4), and the `repo` -> subject kind: +> evaluated at each base), the `repo_static.v1` schema (§4.4), the suite +> manifest for bases in other repositories (§2.1), and the `repo` subject kind: > > ```bash > python cli.py repo validate --manifest [--json] [--out FILE] [--fail-on-warning] @@ -313,6 +313,54 @@ Rules: harness short-lived credentials scoped to the model endpoint. Every connection through the proxy is logged with the run. +### 2.1 Suite manifest (`suite_manifest.v1`) + +A task's `base` is normally a commit of the eval tree's source (§1.1). A task +whose base is in another repository, such as a scaffold that a benchmark shares +or a library at a fixed revision, is listed in a suite file kept in the eval +tree: + +```yaml +schema: suite_manifest.v1 +id: linreg-cpu-solver +repos: + sklearn-scaffold: https://github.com/example/scikit-learn-scaffold@<40-hex sha> + numpy-scaffold: /path/to/numpy-scaffold@<40-hex sha> +tasks: + - id: solver-pick-cholesky + repo: sklearn-scaffold +``` + +- `repo run --suite ` reads the file from the eval tree; the path is + relative to its root and may not leave it. Each listed task takes its base + checkout from the repository it names; its `task.json` keeps the rest of the + task. A task the suite does not list keeps its eval-tree base. Every listed + task must be a task of `tasks` (§2), or the run stops before any cell. +- Each repository is `@` or a local `@`, as in §1.1. The + revision must be a full commit SHA. A branch, tag or other ref is refused + unless the run passes `--allow-floating-suite-revs`; it is resolved again + when each run starts, and the batch pin (`_eval/batch.json`, key `suite`) + records the commits. A resume is refused with `use another --batch` when the + revision now resolves to another commit: the batch does not follow a + moved branch or tag, and it does not reuse a later commit. The pin also + records the suite file's path, the task-to-repository mapping and the + repository locations (key `suite_plan`): a resume with another suite file, or + without `--suite`, is refused. +- A repository that cannot be opened, or a base that is not a commit of it, + leaves that task's cells unsupported (§9): `unsupported.json` gives the reason, + and `repo report` counts the cells neither as failed nor as missing. A cell + that runs records `base_repo` (`{"repo", "source"}`) in `meta.json`. A base that + the cached clone of a remote lacks is fetched once before it counts as missing. +- `repo tasks check` reports these tasks as not checked when their repository is + missing and fails only on the other bad tasks; when every selected task is + not checked it exits 1 (incomplete), as `repo run` does, since nothing was checked. +- Not covered yet: `repo validate` checks the bases of the eval tree only; a suite + task's `{source_dir}` is still the eval-tree source; and a `guidance:rev:` + arm is pinned against the eval-tree source. + +The schema is `dokimi/schemas/suite_manifest.v1.json`; an example is +`data/suites/example-suite.yaml`. + ## 3. `task.json` The framework interprets the **core** and **workspace** keys. Every other key diff --git a/dokimi/dokimi/cli_flags.py b/dokimi/dokimi/cli_flags.py index 7edb0d3ef..d80590b4e 100644 --- a/dokimi/dokimi/cli_flags.py +++ b/dokimi/dokimi/cli_flags.py @@ -227,6 +227,13 @@ class Concept: canonical=(Flag("--network", "[PHASE=]MODE[:hosts]", "text"),), banned=("--agent-network", "--build-network"), ), + Concept( + name="suite", + help="a suite manifest: tasks whose base is in another repository (contract §2.1)", + canonical=(Flag("--suite", "PATH", "text", meaning="the suite file, in the eval tree"), + Flag("--allow-floating-suite-revs", kind="flag", + meaning="resolve a branch or tag of a suite repository when the run starts")), + ), Concept( name="online", help="network checks of a catalog's skills, opt in", diff --git a/dokimi/dokimi/commands/repo_run.py b/dokimi/dokimi/commands/repo_run.py index a0b1b3c86..8097355a4 100644 --- a/dokimi/dokimi/commands/repo_run.py +++ b/dokimi/dokimi/commands/repo_run.py @@ -14,7 +14,7 @@ import shutil import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from dokimi.repos.manifest import RepoManifest @@ -41,12 +41,19 @@ def _refuse_typed(source, args: argparse.Namespace) -> None: print(f"✗ {exc}", file=sys.stderr) sys.exit(1) if args.manifest and not args.manifest_file: - if os.path.splitdrive(args.manifest)[0]: # `D:/x`, `D:x`, `//host/share/x`: no `/` check sees a drive - raise InputError(f"path escapes the repository: {args.manifest}") - try: # as the host reads it: on Windows `..\x` is `../x`, also for a pinned tree, which is read with `/` - tree_path(os.path.normpath(args.manifest).replace(os.path.sep, "/")) - except SourceArgumentError as exc: - raise InputError(f"path escapes the repository: {args.manifest}") from exc + _refuse_escape(args.manifest) + if getattr(args, "suite", None) is not None: # an empty path is given, not omitted: refused below, not skipped + _refuse_escape(args.suite) + + +def _refuse_escape(path: str) -> None: + """Exit 2 for a path that leaves the tree, as the host reads it.""" + if os.path.splitdrive(path)[0]: # `D:/x`, `D:x`, `//host/share/x`: no `/` check sees a drive + raise InputError(f"path escapes the repository: {path}") + try: # as the host reads it: on Windows `..\x` is `../x`, also for a pinned tree, which is read with `/` + tree_path(os.path.normpath(path).replace(os.path.sep, "/")) + except SourceArgumentError as exc: + raise InputError(f"path escapes the repository: {path}") from exc def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): @@ -80,6 +87,7 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): # a pinned batch keeps its content until the pin is checked staged = export_eval_dir(tree, manifest, content.with_name("content.next") if pin else content) _require_grade(staged, manifest, args.task) # before the pin and the image build + suite = _suite_bases(args, tree, staged, manifest, _cache_dir(args)) # its pins go in the batch pin runner = DockerRunner(args.docker) image, ctx_dir = args.image, None if not image: @@ -93,7 +101,8 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): elif not runner.image_exists(image): raise ValueError(f"--image {image} does not exist") if pin: - stamp = {"tree": tree.label, "eval_dir": tree_digest(staged), "image": image} + stamp: dict[str, Any] = {"tree": tree.label, "eval_dir": tree_digest(staged), "image": image} + stamp.update(_suite_pin(suite)) if _uses_builtin_graders(staged, manifest): # else no score depends on them: an upgrade keeps the batch stamp["graders"] = graders_digest() _pin_batch(out, stamp) @@ -118,10 +127,31 @@ def _context(args: argparse.Namespace, out: Path, source, pin: bool = False): uses_source = any("{source_dir}" in v for v in manifest.entry_points.values()) source_dir = Path(args.source_dir).resolve() if args.source_dir else (git_dir if uses_source else None) ctx = EvalContext(manifest=manifest, git_dir=git_dir, content_dir=content, image=image, runner=runner, - caches=caches, source_dir=source_dir, entry_timeout_s=args.grade_timeout) + caches=caches, source_dir=source_dir, entry_timeout_s=args.grade_timeout, suite=suite) return ctx, tree +def _suite_bases(args: argparse.Namespace, tree, content: Path, manifest: RepoManifest, cache_dir: Path | None): + """The base checkouts of ``--suite`` (contract §2.1), or None: every base is then in the eval source.""" + if args.suite is None: + return None + from dokimi.repos.suite import SuiteError, load_suite, resolve_suite + from dokimi.repos.workspace import PrepError, load_tasks + + suite = load_suite(tree, args.suite, args.allow_floating_suite_revs) + try: # only the listed tasks are read: an unrelated malformed task.json is not this run's + found = {t.id: t for t in load_tasks(content, manifest, list(suite.tasks))} + except PrepError as exc: + raise SuiteError(suite.path, [str(exc)]) from exc + return resolve_suite(suite, {t: str(found[t].raw.get("base", "")) for t in suite.tasks}, cache_dir) + + +def _suite_pin(suite) -> dict[str, Any]: + """The commit each suite repository resolved to and the task-to-repository mapping, for the batch pin: + a resume with another suite file, or with none, is refused (``_pin_batch``).""" + return {} if suite is None else {"suite": suite.pins, "suite_plan": suite.plan} + + def _require_grade(content: Path, manifest: RepoManifest, ids: list[str] | None) -> None: """Refuse the selected tasks when one needs ``entry_points.grade`` and the manifest has none.""" from dokimi.repos.manifest import ManifestError, TaskSummary, grade_problems @@ -149,6 +179,9 @@ def _uses_builtin_graders(content: Path, manifest: RepoManifest) -> bool: return any(isinstance(t.raw, dict) and builtin_name(t.raw.get("grader")) is not None for t in tasks) +_SUITE_KEYS = ("suite", "suite_plan") + + def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None: path = batch_dir / "_eval" / "batch.json" if path.exists(): @@ -159,9 +192,10 @@ def _pin_batch(batch_dir: Path, pin: dict[str, str]) -> None: if not isinstance(old, dict): raise ValueError(f"batch {batch_dir.name} has an unreadable pin ({path}); use another --batch") changed = [k for k in pin if old.get(k) != pin[k]] + changed += [k for k in _SUITE_KEYS if k in old and k not in pin] # a suite batch resumed without --suite if changed: was = ", ".join(f"{k} {old.get(k)}" for k in changed) - now = ", ".join(f"{k} {pin[k]}" for k in changed) + now = ", ".join(f"{k} {pin.get(k, 'none')}" for k in changed) raise ValueError(f"batch {batch_dir.name} was run from {was}, not {now}; use another --batch") return path.parent.mkdir(parents=True, exist_ok=True) @@ -206,7 +240,8 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: ctx, _ = _context(args, out, source) tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) arms = [a for spec in args.arm or () for a in spec.split(",") if a] - check_arms([parse_arm(a) for a in arms], ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id) # skill paths exist, catalogs hold each expected_skill + check_arms([parse_arm(a) for a in arms], ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, + ctx.manifest.id, ctx.base_dirs()) # skill paths exist, catalogs hold each expected_skill if arms: print(f"{len(arms)} arm(s) ok: {', '.join(arms)}", file=sys.stderr) print(f"{len(tasks)} task(s), image {ctx.image}{' (warm: with network)' if args.warm else ''}", @@ -217,13 +252,19 @@ def cmd_repo_tasks_check(args: argparse.Namespace) -> None: sys.exit(1) bad = [r for r in results if r["verdict"] != "ok"] print(f"{len(results) - len(bad)}/{len(results)} ok; wrote {out / 'check' / 'check.json'}") + unavailable = [r["task"] for r in bad if r.get("unsupported_by") == "suite"] + if unavailable: # the base's repository of --suite is missing: not checked, and not a failure + print(f"{len(unavailable)} not checked, the repository of their base is missing: " + ", ".join(unavailable)) + if results and len(unavailable) == len(results): # a gate that checked nothing is incomplete, not a pass + print("✗ incomplete: no task was checked, the repository of every base is missing", file=sys.stderr) + sys.exit(1) allowed = [r for r in bad if args.allow_unsupported and r.get("unsupported_by") == "host"] if allowed: print(f"{len(allowed)} unsupported on this host, allowed by --allow-unsupported: " + ", ".join(r["task"] for r in allowed)) - if len(bad) > len(allowed): + if len(bad) > len(allowed) + len(unavailable): sys.exit(1) - if len(bad) == len(results): # --allow-unsupported passed every task by: none was checked + if allowed and len(allowed) == len(results) - len(unavailable): # every task that could be checked was passed by print(f"✗ incomplete: no task passed the self-check (0 ok, {len(allowed)} unsupported)", file=sys.stderr) sys.exit(1) @@ -339,14 +380,18 @@ def _run_locked(args: argparse.Namespace, arms: list[str], batch_dir: Path, lock lock.enter_context(batch_lock(batch_dir)) # before _context, which rewrites _eval/content ctx, _ = _context(args, batch_dir, source, pin=True) tasks = load_tasks(ctx.content_dir, ctx.manifest, args.task) - check_arms(arms, ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id) # skill paths exist, catalogs hold each expected_skill - check_mcp_configs(ctx.git_dir, tasks, profile, args.tools) # a malformed .mcp.json fails here, not in every cell + check_arms(arms, ctx.content_dir, tasks, ctx.git_dir, ctx.manifest.guidance_globs, ctx.manifest.id, + ctx.base_dirs()) # skill paths exist, catalogs hold each expected_skill + check_mcp_configs(ctx.git_dir, tasks, profile, args.tools, ctx.base_dirs()) # a malformed .mcp.json fails here, not in every cell skill_pin = pin_skills(batch_dir, arms, ctx.content_dir) # a resume may not swap an arm's skills validate_tasks(tasks) # refuse malformed host requirements before containers start options = [AgentOptions(profile=profile, model=m, harness_bin=harness_bin, agent_network=args.agent_network, tools=args.tools, delegation=args.delegation, timeout_s=args.timeout, max_turns=args.max_turns, budget_usd=args.budget) for m in args.model] - for task in tasks: # refuse before anything runs, not cell by cell + gaps = ctx.base_gaps() + for task in tasks: # refuse before anything runs, not cell by cell; a suite gap is unsupported, not run + if task.id in gaps: + continue task_caps(task, options[0]) agent_network(options[0], ctx, task) docker_network(ctx.manifest.image.network, task.egress) # grade has no egress route either @@ -460,6 +505,12 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit p.add_argument("source", help=source_help) p.add_argument("--manifest", default=".agent-eval/repo-eval.yaml", help="Manifest path relative to the repository root") + p.add_argument("--suite", default=None, + help="Suite manifest (suite_manifest.v1) path relative to the repository root: tasks whose " + "base is a commit of another repository, named as @ (contract §2.1)") + p.add_argument("--allow-floating-suite-revs", action="store_true", dest="allow_floating_suite_revs", + help="Let --suite name a branch or tag: it is resolved once, when the run starts, and the " + "batch pins the commit it resolved to") p.add_argument("--manifest-file", default=None, dest="manifest_file", help="Read the manifest from this local file instead of from the tree") p.add_argument("--task", action="append", default=None, help="Only this task id; repeatable (default: all)") @@ -489,7 +540,7 @@ def _common(p: argparse.ArgumentParser, source_help: str, cache_help: str, posit "pinned local path inside a repository (give its root) or a --cache-dir that is not a usable directory; exit 1 for a source that cannot be opened " "(clone, fetch, revision) and " ) -CHECK_EXIT = _SOURCE_EXIT + "for a task that fails the check." +CHECK_EXIT = _SOURCE_EXIT + "for a task that fails the check, or when no task was checked (every one a suite gap)." RUN_EXIT = _SOURCE_EXIT + "for a cell that crashed, a batch budget stop or an incomplete batch." diff --git a/dokimi/dokimi/repos/arms.py b/dokimi/dokimi/repos/arms.py index 23974f242..3e26ae63e 100644 --- a/dokimi/dokimi/repos/arms.py +++ b/dokimi/dokimi/repos/arms.py @@ -21,6 +21,7 @@ import hashlib import os import re +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any @@ -191,7 +192,8 @@ def _base_skill_files(git_dir: Path, base: str) -> tuple[list[str], list[str]]: def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Path | None = None, - guidance_globs: tuple[str, ...] | list[str] = (), repo_id: str | None = None) -> None: + guidance_globs: tuple[str, ...] | list[str] = (), repo_id: str | None = None, + base_dirs: Mapping[str, Path | None] = {}) -> None: """Find the skills of every parsed arm before anything runs; a catalog must hold each task's ``expected_skill``. Two arms that would share a name or a result directory are refused, and so is a skill the base of a task (in ``git_dir``) already ships, which preparing the workspace would refuse cell by cell. @@ -201,9 +203,11 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa arm writes its own skill, ``.claude/skills/-guidance``: a skill of the arm with that name, or a file the base tracks under it, is refused here (``repo_id`` given). The files of a ``patch:`` arm's patch and of a task's ``setup_patches`` are not read: a skill directory they add is refused only in - preparation.""" + preparation. ``base_dirs`` maps a suite task to the git dir of its base: that base is read there, and a task + mapped to None (its repository is missing) is not read.""" + own = [(t, base_dirs[t.id] if t.id in base_dirs else git_dir) for t in tasks] # (task, git dir of its base) cells: dict[str, str] = {} # cell-name slug of an arm -> the arm spec that owns it - shipped: dict[str, tuple[list[str], list[str]]] = {} # base -> (tracked files under .claude/skills, its links) + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]] = {} # (git dir, commit) -> (tracked files under .claude/skills, its links) for arm in arms: cell = slug(str(arm).replace(":", "-")) if cells.setdefault(cell, arm.spec) != arm.spec: @@ -212,19 +216,20 @@ def check_arms(arms: list[Any], content_dir: Path, tasks: list[Any], git_dir: Pa names = {s.name for s in skills} generated = None # the skill preparation installs beside the catalog's; its usage is scored apart if repo_id is not None and arm.patch is None and arm.base == "guidance:as-skill": - _check_generated_skill(arm, skills, repo_id, tasks, git_dir, shipped) + _check_generated_skill(arm, skills, repo_id, own, git_dir, shipped) generated = guidance_skill_name(repo_id) drops = arm.patch is None and (arm.base in ("guidance:none", "guidance:as-skill") or arm.rev is not None) rev = (arm.rev_sha or arm.rev) if arm.patch is None else None if skills and git_dir is not None: - for task in tasks: - _check_shipped(arm, skills, task, git_dir, shipped, (drops, rev, guidance_globs)) + for task, base_dir in own: + if base_dir is not None: + _check_shipped(arm, skills, task, (git_dir, base_dir), shipped, (drops, rev, guidance_globs)) if any(kind == "skill-catalog" for kind, _ in arm.skills): _check_expected_skills(arm, tasks, names, generated) -def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: list[Any], git_dir: Path | None, - shipped: dict[str, tuple[list[str], list[str]]]) -> None: +def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: list[tuple[Any, Path | None]], + git_dir: Path | None, shipped: dict[tuple[Path, str], tuple[list[str], list[str]]]) -> None: """Refuse a ``guidance:as-skill`` arm whose generated skill directory is a skill of the arm, or is tracked by the base of a task, or whose base links ``.claude`` or ``.claude/skills`` (preparation installs through both): it would fail cell by cell, not here. A ``shipped`` cache entry holds the base's files and links.""" @@ -234,8 +239,10 @@ def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: l raise ArmError(f"arm {arm.spec!r}: its generated guidance skill {target} is also one of its skills") if git_dir is None: return - for task in tasks: - files, links = _cached_shipped(git_dir, task.base, shipped) + for task, base_dir in tasks: + if base_dir is None: + continue + files, links = _cached_shipped(base_dir, task.base, shipped) for path in files: if path == target or path.startswith(target + "/"): raise ArmError(f"arm {arm.spec!r}: the base of task {task.id} already has {target}") @@ -244,15 +251,17 @@ def _check_generated_skill(arm: Any, skills: list[Skill], repo_id: str, tasks: l f"refusing to install {target} through it") -def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dir: Path, - shipped: dict[str, tuple[list[str], list[str]]], +def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dirs: tuple[Path, Path], + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]], dropping: tuple[bool, str | None, tuple[str, ...] | list[str]]) -> None: """Refuse ``skills`` that the base of ``task`` (or the arm's guidance revision) already ships, or a link that - preparation would refuse to install through; ``shipped`` caches ``_base_skill_files`` per commit.""" + preparation would refuse to install through; ``git_dirs`` is the git dir of the guidance revision and that of + the task's base; ``shipped`` caches ``_base_skill_files`` per git dir and commit.""" from dokimi.repos.manifest import match_glob drops, rev, guidance_globs = dropping - base_files, base_links = _cached_shipped(git_dir, task.base, shipped) + git_dir, base_dir = git_dirs + base_files, base_links = _cached_shipped(base_dir, task.base, shipped) if rev is not None: rev_files = _cached_shipped(git_dir, rev, shipped)[0] left = [p for p in base_files if not (drops and any(match_glob(g, p) for g in guidance_globs))] @@ -267,11 +276,11 @@ def _check_shipped(arm: Any, skills: list[Skill], task: Any, git_dir: Path, def _cached_shipped(git_dir: Path, commit: str, - shipped: dict[str, tuple[list[str], list[str]]]) -> tuple[list[str], list[str]]: - """``_base_skill_files`` of a commit, one ``git ls-tree`` per commit for the whole check.""" - if commit not in shipped: - shipped[commit] = _base_skill_files(git_dir, commit) - return shipped[commit] + shipped: dict[tuple[Path, str], tuple[list[str], list[str]]]) -> tuple[list[str], list[str]]: + """``_base_skill_files`` of a commit, one ``git ls-tree`` per git dir and commit for the whole check.""" + if (git_dir, commit) not in shipped: + shipped[git_dir, commit] = _base_skill_files(git_dir, commit) + return shipped[git_dir, commit] def _check_expected_skills(arm: Any, tasks: list[Any], names: set[str], generated: str | None = None) -> None: diff --git a/dokimi/dokimi/repos/bases.py b/dokimi/dokimi/repos/bases.py index c9f20e63d..3ff7d9c77 100644 --- a/dokimi/dokimi/repos/bases.py +++ b/dokimi/dokimi/repos/bases.py @@ -14,6 +14,7 @@ import json import tempfile +from collections.abc import Mapping from pathlib import Path from typing import Any, Callable @@ -58,13 +59,17 @@ def tree_mcp_error(tree: RepoTree) -> str | None: return None -def check_mcp_configs(git_dir: Any, tasks: list[Any], profile: Any, tools: str) -> None: +def check_mcp_configs(git_dir: Any, tasks: list[Any], profile: Any, tools: str, + base_dirs: Mapping[str, Any] = {}) -> None: """Before a run: refuse a task base whose ``.mcp.json`` the profile would pass to the agent (the ``all`` tool - mode) and the agent's CLI would not accept, instead of every cell of the task failing as a task failure.""" + mode) and the agent's CLI would not accept, instead of every cell of the task failing as a task failure. + A task in ``base_dirs`` (a suite's) is read in the git dir it maps to; one mapped to None is not read.""" if not profile.applied_args(tools, frozenset({MCP_CONFIG})): return - for base in dict.fromkeys(t.base for t in tasks): - problem = tree_mcp_error(GitTree(git_dir, base)) + for where, base in dict.fromkeys((base_dirs.get(t.id, git_dir) if base_dirs else git_dir, t.base) for t in tasks): + if where is None: + continue + problem = tree_mcp_error(GitTree(where, base)) if problem: raise ValueError(f"{MCP_CONFIG} at task base {base} is {problem}; `--tools all` passes it to the agent, " "which exits on it before its first turn. Fix or remove the file, or use --tools read/none") diff --git a/dokimi/dokimi/repos/grading.py b/dokimi/dokimi/repos/grading.py index ae19c43f7..7b906c6e4 100644 --- a/dokimi/dokimi/repos/grading.py +++ b/dokimi/dokimi/repos/grading.py @@ -30,7 +30,7 @@ import stat from dataclasses import dataclass, field from pathlib import Path, PurePosixPath -from typing import Any +from typing import TYPE_CHECKING, Any from dokimi.artifacts import schema_errors from dokimi.measure import SCORE_BOUND, as_float, is_finite @@ -75,6 +75,10 @@ _PYTHON3_NOT_FOUND_SIG = "exec python3 failed" # tini's message under --init, which every container runs with +if TYPE_CHECKING: + from .suite import SuiteBases + + @dataclass class EvalContext: """What every container of one evaluation needs: the trees, the image and the caches.""" @@ -87,6 +91,19 @@ class EvalContext: caches: dict[str, Path] = field(default_factory=dict) # cache name -> host directory source_dir: Path | None = None # host side of {source_dir} entry_timeout_s: float = 3600 + suite: SuiteBases | None = None # base checkouts of a suite run (suite.py); None: every base is in git_dir + + def git_dir_for(self, task_id: str) -> Path: + """The git dir a task's base is checked out from: its suite repository, else ``git_dir``.""" + return self.suite.git_dirs.get(task_id, self.git_dir) if self.suite else self.git_dir + + def base_dirs(self) -> dict[str, Path | None]: + """The git dir of each suite task's base, None for a task whose repository is missing.""" + return {**dict.fromkeys(self.suite.gaps), **self.suite.git_dirs} if self.suite else {} + + def base_gaps(self) -> dict[str, str]: + """Why each suite task has no base checkout; its cells are unsupported, not run.""" + return dict(self.suite.gaps) if self.suite else {} def cache_mounts(self) -> list[Mount]: image = self.manifest.image diff --git a/dokimi/dokimi/repos/runner.py b/dokimi/dokimi/repos/runner.py index a08eefaa4..06c47f79f 100644 --- a/dokimi/dokimi/repos/runner.py +++ b/dokimi/dokimi/repos/runner.py @@ -221,9 +221,12 @@ def run_cell(ctx: EvalContext, task: Task, arm: str, rep: int, batch_dir: Path, if rd.exists(): _rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir) + meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, arm, rd, opts.profile, ctx.content_dir, + base_git_dir=ctx.git_dir_for(task.id)) meta.update(harness=opts.profile.name, harness_version=opts.harness_version, model=opts.model, rep=rep, batch=batch_dir.name, agent_network=network, egress_proxy=False, execution=execution, host=host) + if ctx.suite is not None and task.id in ctx.suite.meta: # additive: only a suite task's base is elsewhere + meta["base_repo"] = ctx.suite.meta[task.id] present = _workspace_args(task, opts, rd, meta) _write_json(rd / "meta.json", meta) skills: dict[str, Any] | None = None @@ -383,7 +386,8 @@ def run_batch(ctx: EvalContext, tasks: list[Task], arms: list[str], options: lis check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells} if force else None) host = probe_host(ctx, batch_dir / "_eval") _write_json(batch_dir / "_eval" / "host.json", host) - verdicts = {t.id: unmet(t, host) for t in tasks} + gaps = ctx.base_gaps() # a task whose base repository is missing is unsupported, as a host miss is + verdicts = {t.id: (gaps[t.id], "suite") if t.id in gaps else unmet(t, host) for t in tasks} unsupported = {task_id: why for task_id, (why, _) in verdicts.items() if why} if force and unsupported: # --force keeps their finished cells, so those must match this run too check_batch_executions(batch_dir, options, {cell_name(t.id, a, o.model, r) for t, a, o, r in cells diff --git a/dokimi/dokimi/repos/selfcheck.py b/dokimi/dokimi/repos/selfcheck.py index f4b10cdaf..2f7dbb764 100644 --- a/dokimi/dokimi/repos/selfcheck.py +++ b/dokimi/dokimi/repos/selfcheck.py @@ -63,9 +63,9 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, guidance_names: tuple[str, ...] = ANCESTOR_NAMES, host: dict[str, Any] | None = None) -> dict[str, Any]: """Null, oracle and isolation checks for one task; ``verdict`` is ok, broken, flaky or unsupported. - An unsupported result says why in ``unsupported_by``: ``egress``, ``host`` (a hardware miss, which - ``--allow-unsupported`` may pass) or ``host-probe`` (the probe could not run or gave no CPU count, - which it never passes). + An unsupported result says why in ``unsupported_by``: ``egress``, ``suite`` (the base's repository + of a suite is missing: not a failure), ``host`` (a hardware miss, which ``--allow-unsupported`` may + pass) or ``host-probe`` (the probe could not run or gave no CPU count, which it never passes). A task the host cannot show still has its workspace prepared and isolation probed, and its oracle run when only CPUs are short; only the null check, which depends on the hardware, is skipped. ``--warm`` @@ -75,6 +75,10 @@ def check_task(ctx: EvalContext, task: Task, work_dir: Path, warm: bool = False, ``host`` requirements probes for itself.""" network = ctx.manifest.image.network if ctx.manifest.image else "none" res: dict[str, Any] = {"task": task.id, "family": task.family, "warm": warm, "problems": []} + gap = ctx.base_gaps().get(task.id) # its base is in a suite repository that is missing: before anything else + if gap: + res.update(verdict="unsupported", unsupported_by="suite", problems=[gap]) + return res try: net = "bridge" if warm else docker_network(network, task.egress) except ContainerError as exc: @@ -167,7 +171,7 @@ def _prepare(ctx: EvalContext, task: Task, rd: Path) -> None: if rd.exists(): shutil.rmtree(rd) rd.mkdir(parents=True) - meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, "guidance:raw", rd) + meta = prepare_workspace(ctx.git_dir, ctx.manifest, task, "guidance:raw", rd, base_git_dir=ctx.git_dir_for(task.id)) meta.update(harness="none", harness_version=None, model="none", rep=0) for name, data in (("meta.json", meta), ("task.json", task.raw)): (rd / name).write_text(json.dumps(data, indent=1) + "\n", encoding="utf-8") diff --git a/dokimi/dokimi/repos/source.py b/dokimi/dokimi/repos/source.py index 0f7b21ea3..b123f97f4 100644 --- a/dokimi/dokimi/repos/source.py +++ b/dokimi/dokimi/repos/source.py @@ -432,8 +432,8 @@ def check_pinned_root(source: RepoSource) -> None: raise SourceArgumentError(f"{source.location} is {prefix!r} inside a repository; give its root") -def open_source(source: RepoSource, cache_dir: Path | None = None) -> RepoTree: - """Return a readable tree for ``source``, cloning or fetching as needed.""" +def open_source(source: RepoSource, cache_dir: Path | None = None, refresh: bool = False) -> RepoTree: + """Return a readable tree for ``source``, cloning or fetching as needed; ``refresh`` re-fetches its refs.""" if not source.remote: if source.rev is None: return WorkTree(source.location) @@ -442,7 +442,7 @@ def open_source(source: RepoSource, cache_dir: Path | None = None) -> RepoTree: return GitTree(Path(source.location), sha, label=f"{source.location}@{sha[:12]}") git_dir = ensure_clone(source.location, cache_dir or default_cache_dir()) assert source.rev is not None # parse() refuses unpinned remotes - sha = ensure_rev(git_dir, source.rev) + sha = ensure_rev(git_dir, source.rev, refresh) return GitTree(git_dir, sha, label=f"{source.location}@{sha[:12]}") @@ -552,10 +552,11 @@ def _remove_tree(path: Path) -> None: shutil.rmtree(path) -def ensure_rev(git_dir: Path, rev: str) -> str: - """Resolve ``rev`` in the cached clone; a branch or tag is re-fetched on every call.""" +def ensure_rev(git_dir: Path, rev: str, refresh: bool = False) -> str: + """Resolve ``rev`` in the cached clone; a branch or tag is re-fetched on every call. ``refresh`` also + re-fetches a hash-shaped name, which may be a branch that moved (a floating suite pin).""" base = re.split(r"[\^~]", rev, maxsplit=1)[0] - if _SHA_RE.match(base): + if _SHA_RE.match(base) and not refresh: try: return resolve_rev(git_dir, rev) except SourceError: @@ -569,6 +570,12 @@ def ensure_rev(git_dir: Path, rev: str) -> str: try: return resolve_rev(git_dir, rev) except SourceError as exc: + if refresh and len(base) in (40, 64) and re.fullmatch(r"[0-9a-f]+", base): # a full commit on no branch + try: + _git(git_dir, "fetch", "--filter=blob:none", "origin", "--", base) + return resolve_rev(git_dir, rev) + except SourceError: + pass short = _SHA_RE.match(base) and len(base) < 40 hint = "; a short SHA on no branch or tag cannot be fetched, give the full SHA" if short else "" raise SourceError(f"revision not found in origin: {rev}{hint}") from exc diff --git a/dokimi/dokimi/repos/suite.py b/dokimi/dokimi/repos/suite.py new file mode 100644 index 000000000..0a59690b6 --- /dev/null +++ b/dokimi/dokimi/repos/suite.py @@ -0,0 +1,202 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Suite manifests (``suite_manifest.v1``, contract §2.1): tasks whose base is in another repository. + +A suite names the scaffold repositories, each as ``@``, and the tasks of the eval tree that +take their base checkout from one of them. The task itself (prompt, grader, ``task.json``) stays in the +eval tree; only its ``base`` is looked up in the named repository. A repository that cannot be opened, +or a base that is not a commit of it, leaves those tasks' cells unsupported: not run, not failed. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +from dokimi import process +from dokimi.artifacts import schema_errors + +from .errors import PrepError +from .source import RepoSource, RepoTree, SourceError, open_source, resolve_rev +from .workspace import source_git_dir + +SCHEMA_VERSION = "suite_manifest.v1" +_SHA_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$") # a full commit: a branch, tag, short SHA or other ref is not +_ABBREV_RE = re.compile(r"^[0-9a-f]{7,39}$") # a short commit SHA, as manifest._BASE_RE reads a task base + + +class SuiteError(ValueError): + """The suite manifest is malformed; ``errors`` lists every problem found.""" + + def __init__(self, path: str, errors: list[str]): + self.path = path + self.errors = errors + super().__init__(f"{path}: invalid {SCHEMA_VERSION}:\n" + "\n".join(f" - {e}" for e in errors)) + + +@dataclass(frozen=True) +class SuiteManifest: + """A validated suite: ``repos`` maps each name to its parsed ``@``, ``tasks`` each task id to a name.""" + + path: str + id: str + repos: dict[str, RepoSource] + tasks: dict[str, str] + floating: bool = False # the run allows a branch, tag or other ref as a pin + + +@dataclass +class SuiteBases: + """Where each task of a suite run takes its base from, and why a task has no base. + + ``git_dirs`` and ``meta`` (``{"repo", "source"}``, for meta.json) cover the tasks whose base + resolved; ``gaps`` gives the reason for each task that did not; ``pins`` names the commit each + repository resolved to and ``plan`` the suite file's path, the task-to-repository mapping and the repository + locations, which a pinned batch records. + """ + + git_dirs: dict[str, Path] = field(default_factory=dict) + gaps: dict[str, str] = field(default_factory=dict) + meta: dict[str, dict[str, str]] = field(default_factory=dict) + pins: dict[str, str] = field(default_factory=dict) + plan: dict[str, Any] = field(default_factory=dict) + + +def parse_suite(raw: Any, path: str = "", allow_floating: bool = False) -> SuiteManifest: + """Validate an already-parsed suite mapping. A branch, tag or other ref is refused as a pin unless + ``allow_floating``, in which case it is resolved when the run starts and the batch pins the result.""" + _refuse_odd_names(raw, path) + errors = schema_errors(SCHEMA_VERSION, raw) + if errors: + raise SuiteError(path, errors) + problems: list[str] = [] + repos: dict[str, RepoSource] = {} + for name, spec in raw["repos"].items(): + try: + source = _parse_repo(spec) + except SourceError as exc: + problems.append(f"repos.{name}: {exc}") + continue + if source.rev is None: + problems.append(f"repos.{name}: {spec!r} is not pinned; give @") + elif not _SHA_RE.match(source.rev) and not allow_floating: + problems.append(f"repos.{name}: {source.rev!r} is a branch, tag or ref, not a commit; pin its SHA, " + "or pass --allow-floating-suite-revs to resolve it when the run starts") + else: + repos[name] = source + tasks: dict[str, str] = {} + for item in raw["tasks"]: + task_id, repo = item["id"], item["repo"] + if task_id in tasks: + problems.append(f"task {task_id} is listed more than once") + elif repo not in raw["repos"]: + problems.append(f"task {task_id}: repo {repo!r} is not declared in repos") + else: + tasks[task_id] = repo + if problems: + raise SuiteError(path, problems) + return SuiteManifest(path=path, id=raw["id"], repos=repos, tasks=tasks, floating=allow_floating) + + +def _refuse_odd_names(raw: Any, path: str) -> None: + """A ``repos`` name that is not a string is a suite error; ``schema_errors`` cannot sort it among the others.""" + repos = raw.get("repos") if isinstance(raw, dict) else None + odd = [k for k in repos if not isinstance(k, str)] if isinstance(repos, dict) else [] + if odd: + raise SuiteError(path, [f"repos: the name {k!r} is not a string" for k in odd]) + + +def _parse_repo(spec: str) -> RepoSource: + """``RepoSource.parse``, except that a pinned local path that is not a directory is kept: it is + unavailable, which leaves its tasks unsupported (``_checkout``), not a malformed suite.""" + try: + return RepoSource.parse(spec) + except SourceError as exc: + head, sep, tail = spec.strip().rpartition("@") + if "local source is not a directory" not in str(exc) or not (sep and head and tail) or "://" in head: + raise + return RepoSource(location=head, rev=tail, remote=False) + + +def load_suite(tree: RepoTree, path: str, allow_floating: bool = False) -> SuiteManifest: + """Read and validate the suite manifest at ``path`` inside the eval tree.""" + if not tree.is_file(path): + raise SuiteError(path, [f"suite manifest not found in {tree.label}"]) + try: + raw = yaml.safe_load(tree.read_text(path)) + except (yaml.YAMLError, UnicodeDecodeError) as exc: + raise SuiteError(path, [f"not valid YAML: {exc}"]) from exc + return parse_suite(raw, path, allow_floating) + + +def _checkout(source: RepoSource, cache_dir: Path | None, refresh: bool = False) -> tuple[Path, str] | str: + """``(git dir, commit)`` of a repository of the suite, or the reason it cannot be opened.""" + try: + tree = open_source(source, cache_dir, refresh=refresh) + git_dir = source_git_dir(source, tree, cache_dir) + except (ValueError, OSError, PrepError) as exc: # SourceError is a ValueError + return f"{source} is unavailable: {exc}" + sha = getattr(tree, "sha", None) + return (git_dir, sha) if isinstance(sha, str) else f"{source} did not resolve to a commit" + + +def _resolve_base(source: RepoSource, git_dir: Path, base: str) -> str: + """``base`` in ``git_dir``. A remote's cached clone may predate it, so a miss fetches it once before it is a gap.""" + try: + return resolve_rev(git_dir, base) + except SourceError: + commit = re.split(r"[\^~]", base, maxsplit=1)[0] # a parent (``^``) is fetched by its commit + if not source.remote or not (_SHA_RE.match(commit) or _ABBREV_RE.match(commit)): + raise + if _SHA_RE.match(commit): + process.git("-C", str(git_dir), "fetch", "origin", "--", commit) # a failure leaves the miss below + else: # git cannot fetch a short SHA: its refs are fetched, then the base is looked up again; a failure leaves the miss + # no --filter here: on the full clone it would make every later fetch partial + process.git("-C", str(git_dir), "fetch", "origin", + "+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*") + return resolve_rev(git_dir, base) + + +def resolve_suite(suite: SuiteManifest, bases: Mapping[str, str], cache_dir: Path | None = None) -> SuiteBases: + """Open every repository of the suite and find each task's base in the repository it names. + + ``bases`` maps each suite task to the ``base`` of its task.json. Repositories are opened once + each, all of them, so the pins do not depend on which tasks a run selects. + """ + checkouts = {name: _checkout(source, cache_dir, refresh=suite.floating) # a hash-shaped name may be a branch + for name, source in suite.repos.items()} + for name, found in checkouts.items(): # a name shaped like a hash (a branch) is not the commit the pin says + if suite.floating: # the run asked for refs to be resolved: the commit it found is the pin + break + rev = suite.repos[name].rev + if not isinstance(found, str) and rev and _SHA_RE.match(rev) and found[1] != rev: + raise SuiteError(suite.path, [f"repos.{name}: {rev!r} resolves to {found[1]}, not to that commit; " + "pin the full commit SHA"]) + # a full-SHA pin is the declared commit, whether or not its repository opened + pins = {name: source.rev for name, source in suite.repos.items() if not suite.floating and source.rev is not None} + pins.update({name: found[1] for name, found in checkouts.items() if suite.floating and not isinstance(found, str)}) + out = SuiteBases(pins=pins, + plan={"path": suite.path, "repos": {name: str(source) for name, source in suite.repos.items()}, + "tasks": dict(suite.tasks)}) + for task_id, repo in suite.tasks.items(): + found = checkouts[repo] + if isinstance(found, str): + out.gaps[task_id] = f"base repository {repo}: {found}" + continue + git_dir, sha = found + base = bases.get(task_id, "") + try: + _resolve_base(suite.repos[repo], git_dir, base) + except SourceError as exc: + out.gaps[task_id] = (f"base {base!r} is not a commit of {repo} " + f"({suite.repos[repo].location}@{sha[:12]}): {exc}") + continue + out.git_dirs[task_id] = git_dir + out.meta[task_id] = {"repo": repo, "source": f"{suite.repos[repo].location}@{sha}"} + return out diff --git a/dokimi/dokimi/repos/workspace.py b/dokimi/dokimi/repos/workspace.py index 2f73a3b2c..277a912be 100644 --- a/dokimi/dokimi/repos/workspace.py +++ b/dokimi/dokimi/repos/workspace.py @@ -362,9 +362,13 @@ def _apply_review_patch(ws: Path, task: Task, patches: list[dict[str, str]]) -> def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: str, run_dir: Path, - profile: HarnessProfile | None = None, content_dir: Path | None = None) -> dict[str, Any]: + profile: HarnessProfile | None = None, content_dir: Path | None = None, + base_git_dir: Path | None = None) -> dict[str, Any]: """Steps 1-7 into ``run_dir/workspace``; returns the ``meta.json`` fields prep owns. + The base is checked out from ``base_git_dir`` (default ``git_dir``): a suite task's base is in another + repository, while the arm's revisions and guidance are still read from ``git_dir``. + ``arm`` is an :class:`Arm` (``repo run`` parses and pins each once per batch) or an arm string, parsed and pinned here. ``content_dir`` (the exported ``{eval_dir}``) is where relative skill paths of the arm are found. @@ -385,17 +389,18 @@ def prepare_workspace(git_dir: Path, manifest: RepoManifest, task: Task, arm: st if arm.base == OVERLAY_ARM and overlay not in overlay_names(): names = ", ".join(sorted(overlay_names())) raise PrepError(f"guidance:overlay: overlay {overlay!r} is not a builtin overlay name ({names})") + base_dir = base_git_dir or git_dir try: - base_sha = resolve_rev(git_dir, task.base) + base_sha = resolve_rev(base_dir, task.base) except SourceError as exc: - raise PrepError(f"task {task.id}: base {task.base!r} is not in {git_dir}: {exc}") from exc - base_date = _run(["git", "-C", str(git_dir), "log", "-1", "--format=%cI", base_sha]).strip() + raise PrepError(f"task {task.id}: base {task.base!r} is not in {base_dir}: {exc}") from exc + base_date = _run(["git", "-C", str(base_dir), "log", "-1", "--format=%cI", base_sha]).strip() ws = run_dir / "workspace" if ws.exists(): shutil.rmtree(ws) ws.mkdir(parents=True) - _checkout_tree(git_dir, base_sha, ws) # 1 + _checkout_tree(base_dir, base_sha, ws) # 1 shutil.rmtree(ws / manifest.run_paths()[0], ignore_errors=True) # 2 env = _git_env(SNAPSHOT_AUTHOR) _git(ws, env, "init", "-q") # 3 diff --git a/dokimi/dokimi/schemas/suite_manifest.v1.json b/dokimi/dokimi/schemas/suite_manifest.v1.json new file mode 100644 index 000000000..1e0a65bd9 --- /dev/null +++ b/dokimi/dokimi/schemas/suite_manifest.v1.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/suite_manifest.v1.json", + "title": "Cross-repository suite manifest v1", + "description": "suite.yaml: tasks of the eval tree whose base is a commit of another repository (docs/repo-evaluation-contract.md §2.1). `repos` names the scaffold repositories, each referenced as @ (a local pinned path works too); `tasks` lists the tasks of the eval tree that take their base checkout from one of them. This is an input the benchmark repository owns, so it carries `schema`, not `schema_version`. Semantic rules (every task is a task of the tree, every repo name is declared, revisions are full commits unless the run allows floating ones) are checked in dokimi/repos/suite.py, which says them in friendlier words.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "repos", "tasks"], + "properties": { + "schema": {"const": "suite_manifest.v1"}, + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"}, + "repos": { + "type": "object", + "minProperties": 1, + "propertyNames": {"pattern": "^[a-z0-9][a-z0-9._-]*$"}, + "additionalProperties": {"type": "string", "minLength": 1} + }, + "tasks": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "repo"], + "properties": { + "id": {"type": "string", "pattern": "^[^\\s,]+$"}, + "repo": {"type": "string", "pattern": "^[a-z0-9][a-z0-9._-]*$"} + } + } + } + } +} diff --git a/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json b/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json new file mode 100644 index 000000000..dff4ecdd1 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/suite_manifest.v1.json @@ -0,0 +1,12 @@ +{ + "schema": "suite_manifest.v1", + "id": "linreg-cpu-solver", + "repos": { + "sklearn-scaffold": "https://github.com/example-org/sklearn-scaffold@9f8e7d6c5b4a39281706f5e4d3c2b1a098765432", + "numpy-scaffold": "https://github.com/example-org/numpy-scaffold@0a1b2c3d4e5f60718293a4b5c6d7e8f901234567" + }, + "tasks": [ + {"id": "solver-pick-cholesky", "repo": "sklearn-scaffold"}, + {"id": "solver-pick-lstsq", "repo": "numpy-scaffold"} + ] +} diff --git a/dokimi/tests/test_repo_host.py b/dokimi/tests/test_repo_host.py index 845d2071f..f8f2c7494 100644 --- a/dokimi/tests/test_repo_host.py +++ b/dokimi/tests/test_repo_host.py @@ -831,10 +831,10 @@ def test_a_prepare_error_on_the_null_step_is_a_problem_and_the_oracle_still_runs root = _repo(tmp_path) real = prepare_workspace - def prepare(git_dir, manifest, task, arm, rd): + def prepare(git_dir, manifest, task, arm, rd, **kwargs): if rd.name.endswith("__null"): raise PrepError("no such revision") - return real(git_dir, manifest, task, arm, rd) + return real(git_dir, manifest, task, arm, rd, **kwargs) monkeypatch.setattr("dokimi.repos.selfcheck.prepare_workspace", prepare) with pytest.raises(SystemExit) as exc: diff --git a/dokimi/tests/test_repo_suite.py b/dokimi/tests/test_repo_suite.py new file mode 100644 index 000000000..6268c0faf --- /dev/null +++ b/dokimi/tests/test_repo_suite.py @@ -0,0 +1,540 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Suite manifests (suite_manifest.v1, contract §2.1): the schema, the pins, the base of each task +and the run. Every repository here is a local git repository; nothing reaches the network.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +import yaml + +from dokimi.cli import build_parser +from dokimi.repos.source import RepoSource, open_source +from dokimi.repos.suite import SuiteError, load_suite, parse_suite, resolve_suite +from dokimi.utils import InputError +from repo_eval_fixture import PROFILE, FakeRunner, fake_harness, git, make_repo, write + +EXAMPLE = Path(__file__).resolve().parent.parent / "data" / "suites" / "example-suite.yaml" +PINNED = "https://example.invalid/scaffold@" + "a" * 40 +CALC = "def add(a, b):\n return a + b\n" + + +def _raw(**over): + raw = {"schema": "suite_manifest.v1", "id": "s", "repos": {"sc": PINNED}, "tasks": [{"id": "t1", "repo": "sc"}]} + raw.update(over) + return raw + + +def test_the_example_suite_parses_and_every_repo_is_pinned(): + suite = parse_suite(yaml.safe_load(EXAMPLE.read_text(encoding="utf-8")), str(EXAMPLE)) + assert suite.tasks == {"solver-pick-cholesky": "sklearn-scaffold", "solver-pick-lstsq": "numpy-scaffold"} + assert all(len(src.rev or "") == 40 for src in suite.repos.values()) + + +@pytest.mark.parametrize("rev", ["main", "v1.0", "HEAD~1", "abcdef0", "a" * 39, "a" * 41, "a" * 63]) +def test_a_branch_tag_or_ref_is_refused_as_a_pin(rev): + with pytest.raises(SuiteError, match="not a commit; pin its SHA"): + parse_suite(_raw(repos={"sc": f"https://example.invalid/scaffold@{rev}"})) + + +@pytest.mark.parametrize("rev", ["a" * 40, "b" * 64]) +def test_a_full_commit_sha_is_a_pin(rev): + suite = parse_suite(_raw(repos={"sc": f"https://example.invalid/scaffold@{rev}"})) + assert suite.repos["sc"].rev == rev + + +def test_a_floating_rev_is_kept_only_when_the_run_allows_it(): + suite = parse_suite(_raw(repos={"sc": "https://example.invalid/scaffold@main"}), allow_floating=True) + assert suite.repos["sc"].rev == "main" + + +def test_a_repo_without_a_revision_is_refused_even_when_floating_is_allowed(tmp_path): + with pytest.raises(SuiteError, match="not pinned"): + parse_suite(_raw(repos={"sc": str(tmp_path)}), allow_floating=True) + with pytest.raises(SuiteError, match="must be pinned"): + parse_suite(_raw(repos={"sc": "https://example.invalid/scaffold"})) + + +def test_a_task_must_name_a_declared_repo_and_appear_once(): + with pytest.raises(SuiteError, match="repo 'nope' is not declared"): + parse_suite(_raw(tasks=[{"id": "t1", "repo": "nope"}])) + with pytest.raises(SuiteError, match="listed more than once"): + parse_suite(_raw(tasks=[{"id": "t1", "repo": "sc"}, {"id": "t1", "repo": "sc"}])) + + +@pytest.mark.parametrize("change", [ + {"schema": "suite_manifest.v2"}, + {"unexpected": True}, + {"repos": {}}, + {"tasks": [{"id": "a,b", "repo": "sc"}]}, + {"tasks": [{"id": "t1"}]}, + {"repos": {1: PINNED, "sc": PINNED}}, +]) +def test_the_schema_refuses_a_malformed_suite(change): + with pytest.raises(SuiteError): + parse_suite(_raw(**change)) + + +def test_a_suite_file_missing_from_the_tree_is_refused(tmp_path): + root, _ = make_repo(tmp_path) + tree = open_source(RepoSource.parse(str(root))) + with pytest.raises(SuiteError, match="suite manifest not found"): + load_suite(tree, ".agent-eval/suite.yaml") + + +def test_a_repo_name_that_is_not_a_string_is_a_suite_error_and_names_it(): + with pytest.raises(SuiteError, match="the name 1 is not a string"): + parse_suite(_raw(repos={1: None, "sc": PINNED})) + + +def _scaffold(tmp_path: Path, name: str = "scaffold") -> tuple[Path, str]: + root = tmp_path / name + write(root, "src/calc.py", CALC) + git(root, "init", "-q", "-b", "main") + git(root, "add", "-A") + git(root, "commit", "-qm", "scaffold") + return root, git(root, "rev-parse", "HEAD") + + +def test_each_task_takes_its_base_from_the_repo_it_names_and_a_gap_is_recorded(tmp_path): + good, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"file://{tmp_path}/missing@{'b' * 40}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "absent", "repo": "good"}, + {"id": "lost", "repo": "gone"}])) + bases = {"ok": sha, "absent": "1" * 40, "lost": sha} + found = resolve_suite(suite, bases, tmp_path / "cache") + assert set(found.git_dirs) == {"ok"} and found.git_dirs["ok"].is_dir() + assert found.meta == {"ok": {"repo": "good", "source": f"{good}@{sha}"}} + assert found.pins == {"good": sha, "gone": "b" * 40} # every declared pin, the down repository's too + assert "is not a commit of good" in found.gaps["absent"] + assert found.gaps["lost"].startswith("base repository gone:") and "unavailable" in found.gaps["lost"] + + +def test_a_floating_branch_resolves_once_to_the_commit_it_names(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"{root}@main"}, tasks=[{"id": "t1", "repo": "sc"}]), + allow_floating=True) + found = resolve_suite(suite, {"t1": sha}, tmp_path / "cache") + assert found.pins == {"sc": sha} and found.gaps == {} + + +def test_a_branch_named_like_a_hash_resolves_when_the_run_allows_floating_revs(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "branch", "b" * 64, sha) + suite = parse_suite(_raw(repos={"sc": f"{root}@{'b' * 64}"}, tasks=[{"id": "t1", "repo": "sc"}]), + allow_floating=True) + assert resolve_suite(suite, {"t1": sha}, tmp_path / "cache").pins == {"sc": sha} + + +def test_a_branch_named_like_a_hash_is_not_the_pinned_commit(tmp_path): + root, sha = _scaffold(tmp_path) + write(root, "src/more.py", "x = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "more") + fake = "b" * 64 # a SHA-1 repository: this is a branch, whatever its spelling + git(root, "branch", fake, sha) + suite = parse_suite(_raw(repos={"sc": f"{root}@{fake}"})) + with pytest.raises(SuiteError, match="resolves to .*not to that commit"): + resolve_suite(suite, {"t1": sha}, tmp_path / "cache") + + +def test_the_suite_file_is_read_from_the_eval_tree(tmp_path): + root, _ = make_repo(tmp_path) + write(root, ".agent-eval/suite.yaml", yaml.safe_dump(_raw())) + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + tree = open_source(RepoSource.parse(str(root))) + assert load_suite(tree, ".agent-eval/suite.yaml").tasks == {"t1": "sc"} + + +@pytest.fixture +def run(tmp_path, monkeypatch): + root, _ = make_repo(tmp_path) + exe, profile = fake_harness(tmp_path) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + monkeypatch.setenv("FAKE_ACTION", "fix") + state = {"root": root, "out": tmp_path / "runs", "tmp": tmp_path} + + def invoke(*extra: str) -> None: + argv = ["repo", "run", f"{root}@HEAD", "--manifest", ".agent-eval/repo-eval.yaml", "--harness", + str(profile), "--bin", str(exe), "--model", "m-1", "--batch", "b1", "--out", str(state["out"]), + "--cache-root", str(state["out"] / "caches"), "--agent-network", "host", *extra] + args = build_parser().parse_args(argv) + args.func(args) + + state["invoke"] = invoke + return state + + +def _commit_suite(root: Path, base_of_fix: str | None, suite: dict) -> None: + """Point fix_add at ``base_of_fix`` (when given) and commit the suite file into the eval tree.""" + if base_of_fix is not None: + task = root / ".agent-eval/tasks/fix_add/task.json" + write(root, ".agent-eval/tasks/fix_add/task.json", + json.dumps({**json.loads(task.read_text(encoding="utf-8")), "base": base_of_fix})) + write(root, ".agent-eval/suite.yaml", yaml.safe_dump(suite)) + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + + +def test_a_run_takes_the_base_of_a_suite_task_from_its_repo(run): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + fix = json.loads((batch / "fix_add__guidance-none__m-1__r1/meta.json").read_text(encoding="utf-8")) + assert fix["base_sha"] == sha and fix["base_repo"] == {"repo": "sc", "source": f"{scaffold}@{sha}"} + assert (batch / "fix_add__guidance-none__m-1__r1/result.json").is_file() + review = json.loads((batch / "review_add__guidance-none__m-1__r1/meta.json").read_text(encoding="utf-8")) + assert "base_repo" not in review # a task the suite does not name keeps its eval-tree base + pin = json.loads((batch / "_eval/batch.json").read_text(encoding="utf-8")) + assert pin["suite"] == {"sc": sha} + + +def test_a_missing_suite_repo_leaves_its_cells_unsupported_and_not_failed(run, capsys): + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(run["root"], None, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + assert not (batch / "fix_add__guidance-none__m-1__r1/result.json").exists() + unsupported = json.loads((batch / "fix_add__guidance-none__m-1__r1/unsupported.json").read_text(encoding="utf-8")) + assert unsupported["reason"].startswith("base repository sc:") and unsupported["task"] == "fix_add" + assert (batch / "review_add__guidance-none__m-1__r1/result.json").is_file() + assert "fix_add: UNSUPPORTED, its cells are not run: base repository sc:" in capsys.readouterr().out + + +def test_a_suite_task_the_tree_lacks_is_refused_before_any_cell(run, capsys): + _commit_suite(run["root"], None, _raw(tasks=[{"id": "nope", "repo": "sc"}], + repos={"sc": f"{run['root']}@{'d' * 40}"})) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--arm", "guidance:none") + assert exc.value.code == 1 + assert "no such task(s) under" in capsys.readouterr().err + assert not (run["out"] / "b1" / "nope__guidance-none__m-1__r1").exists() + + +def test_a_suite_path_that_leaves_the_tree_is_a_usage_error(run): + with pytest.raises(InputError, match="path escapes the repository"): + run["invoke"]("--suite", "../suite.yaml", "--arm", "guidance:none") + + +def test_a_suite_file_that_is_not_utf8_is_a_suite_error(tmp_path): + root, _ = make_repo(tmp_path) + write(root, ".agent-eval/suite.yaml", "") + (root / ".agent-eval/suite.yaml").write_bytes(b"schema: \xff\xfe\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "suite") + tree = open_source(RepoSource.parse(str(root))) + with pytest.raises(SuiteError, match="not valid YAML"): + load_suite(tree, ".agent-eval/suite.yaml") + + +@pytest.mark.parametrize("scheme", ["", "file://"]) +def test_a_missing_repository_leaves_its_tasks_unsupported_in_either_spelling(tmp_path, scheme): + good, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"{scheme}{tmp_path}/missing@{'b' * 40}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "lost", "repo": "gone"}])) + found = resolve_suite(suite, {"ok": sha, "lost": sha}, tmp_path / "cache") + assert set(found.git_dirs) == {"ok"} + assert found.gaps["lost"].startswith("base repository gone:") and "unavailable" in found.gaps["lost"] + + +def test_a_local_path_that_is_not_pinned_is_still_refused(tmp_path): + with pytest.raises(SuiteError, match="local source is not a directory"): + parse_suite(_raw(repos={"sc": f"{tmp_path}/missing"}), allow_floating=True) + + +def test_a_base_the_warm_cache_of_a_remote_predates_is_fetched_not_a_gap(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + later = git(root, "rev-parse", "HEAD") + found = resolve_suite(suite, {"t1": later}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + assert resolve_suite(suite, {"t1": "1" * 40}, cache).gaps["t1"].startswith(f"base {'1' * 40!r} is not a commit of sc") + + +def _suite_of(root: Path, tmp: Path, name: str, scaffold: Path, sha: str, *tasks: str) -> str: + """Commit ``.agent-eval/.yaml``: both suites declare ``a`` and ``b`` (the same scaffold), ``tasks`` map to ``b``.""" + repos = {"a": f"{scaffold}@{sha}", "b": f"{scaffold}@{sha}"} + path = f".agent-eval/{name}.yaml" + write(root, path, yaml.safe_dump(_raw(repos=repos, tasks=[{"id": t, "repo": "b"} for t in tasks]))) + git(root, "add", "-A") + git(root, "commit", "-qm", name) + return path + + +def test_a_resume_with_another_suite_file_or_none_is_refused(run, capsys): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw()) # fix_add's base is the scaffold's commit + first = _suite_of(run["root"], run["tmp"], "first", scaffold, sha, "fix_add") + second = _suite_of(run["root"], run["tmp"], "second", scaffold, sha, "review_add") + same = _suite_of(run["root"], run["tmp"], "same", scaffold, sha, "fix_add") # first's content, another file + argv = ["--task", "fix_add", "--arm", "guidance:none"] + run["invoke"]("--suite", first, *argv) + run["invoke"]("--suite", first, *argv) # the same suite resumes + for extra in (["--suite", second], ["--suite", same], []): + with pytest.raises(SystemExit) as exc: + run["invoke"](*extra, *argv) + assert exc.value.code == 1 + assert "use another --batch" in capsys.readouterr().err + + +def test_a_malformed_task_the_suite_and_the_selection_leave_out_does_not_stop_the_run(run): + scaffold, sha = _scaffold(run["tmp"]) + write(run["root"], ".agent-eval/tasks/broken/task.json", "{bad") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", "guidance:none") + assert (run["out"] / "b1/fix_add__guidance-none__m-1__r1/result.json").is_file() + + +def test_a_guidance_revision_arm_reads_the_eval_tree_while_the_base_comes_from_the_scaffold(run): + scaffold, sha = _scaffold(run["tmp"]) + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + rev = git(run["root"], "rev-parse", "HEAD") + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", f"guidance:rev:{rev}") + cell = next((run["out"] / "b1").glob("fix_add__*/meta.json")) + meta = json.loads(cell.read_text(encoding="utf-8")) + assert meta["base_sha"] == sha + assert "AGENTS.md" in meta["arm_files"] # the scaffold has none: it is the eval tree's, at the revision + assert (cell.parent / "workspace/AGENTS.md").read_text(encoding="utf-8") == "# guidance\n" + + +def test_a_suite_gap_does_not_abort_the_batch_on_a_skill_its_eval_base_ships(run): + root = run["root"] + skill = "---\nname: lint\ndescription: x\n---\nLint.\n" + write(root, ".claude/skills/lint/SKILL.md", skill) + git(root, "add", "-A") + git(root, "commit", "-qm", "base ships lint") + base = git(root, "rev-parse", "HEAD") + write(root, ".agent-eval/skills/lint/SKILL.md", skill) + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(root, base, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:raw+skill:skills/lint") + batch = run["out"] / "b1" + assert list(batch.glob("fix_add__*/unsupported.json")) + assert list(batch.glob("review_add__*/result.json")) + + +def _check(tmp: Path, monkeypatch, suite: dict, *extra: str, tasks: dict | None = None) -> tuple[int, dict]: + """``eval check --suite`` over a repository whose suite is ``suite``; returns (exit code, check.json).""" + root, base = make_repo(tmp, tasks) + _commit_suite(root, None, suite) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + out = tmp / "check-out" + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(out), "--cache-root", + str(out / "caches"), "--suite", ".agent-eval/suite.yaml", *extra]) + code = 0 + try: + args.func(args) + except SystemExit as exc: + code = exc.code + return code, json.loads((out / "check/check.json").read_text(encoding="utf-8")) + + +def test_a_self_check_where_every_selected_task_is_a_suite_gap_exits_1(tmp_path, monkeypatch, capsys): + gone = f"file://{tmp_path}/missing@{'c' * 40}" + code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}]), + "--task", "fix_add") + captured = capsys.readouterr() + assert code == 1 and "incomplete: no task was checked" in captured.err + assert [(r["task"], r["unsupported_by"]) for r in report["tasks"]] == [("fix_add", "suite")] + + +def test_a_suite_gap_is_not_checked_even_when_the_task_also_needs_egress(tmp_path, monkeypatch, capsys): + gone = f"file://{tmp_path}/missing@{'c' * 40}" + net = {"net_add": {"family": "review", "grader": "review", "review_patch": "change.patch", "egress": ["example.com"], + "score": {"metric": "recall", "max": 1.0}, "prompt": "Review HEAD."}} + code, report = _check(tmp_path, monkeypatch, _raw(repos={"sc": gone}, tasks=[{"id": "net_add", "repo": "sc"}]), + "--task", "net_add", tasks=net) + assert code == 1 + assert "incomplete: no task was checked" in capsys.readouterr().err + assert report["tasks"][0]["unsupported_by"] == "suite" + + +def test_a_skill_the_scaffold_base_ships_is_refused_before_any_cell(run, capsys): + scaffold, _ = _scaffold(run["tmp"]) + write(scaffold, ".claude/skills/lint/SKILL.md", "---\nname: lint\ndescription: x\n---\nLint.\n") + git(scaffold, "add", "-A") + git(scaffold, "commit", "-qm", "ships lint") + sha = git(scaffold, "rev-parse", "HEAD") + write(run["root"], ".agent-eval/skills/lint/SKILL.md", "---\nname: lint\ndescription: x\n---\nLint.\n") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:raw+skill:skills/lint") + assert exc.value.code == 1 + assert "the base of task fix_add already has .claude/skills/lint" in capsys.readouterr().err + assert not list((run["out"] / "b1").glob("*__*/result.json")) + + +def test_a_malformed_mcp_config_in_a_scaffold_base_is_refused_before_any_cell(run, capsys): + profile = run["tmp"] / "fake.yaml" # a harness that passes the repository's .mcp.json to the agent + profile.write_text(yaml.safe_dump({**PROFILE, "args_if_present": {".mcp.json": ["--mcp-config", ".mcp.json"]}}), + encoding="utf-8") + scaffold, _ = _scaffold(run["tmp"]) + write(scaffold, ".mcp.json", "{") + git(scaffold, "add", "-A") + git(scaffold, "commit", "-qm", "bad mcp") + sha = git(scaffold, "rev-parse", "HEAD") + _commit_suite(run["root"], sha, _raw(repos={"sc": f"{scaffold}@{sha}"}, tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--arm", "guidance:none") + assert exc.value.code == 1 + assert "not valid JSON" in capsys.readouterr().err + assert not list((run["out"] / "b1").glob("*__*/result.json")) + + +def test_an_empty_suite_path_is_refused_by_repo_run_and_not_read_as_no_suite(run, capsys): + _commit_suite(run["root"], None, _raw(repos={"sc": f"file://{run['tmp']}/missing@{'c' * 40}"}, + tasks=[{"id": "fix_add", "repo": "sc"}])) + with pytest.raises(SystemExit) as exc: + run["invoke"]("--suite", "", "--task", "review_add", "--arm", "guidance:none") + assert exc.value.code == 1 + assert not (run["out"] / "b1" / "review_add__guidance-none__m-1__r1/result.json").exists() + + +def test_an_empty_suite_path_is_refused_by_self_check_too(tmp_path, monkeypatch, capsys): + root, _ = make_repo(tmp_path) + monkeypatch.setattr("dokimi.repos.container.DockerRunner", lambda *a, **k: FakeRunner()) + out = tmp_path / "check-out" + args = build_parser().parse_args(["eval", "check", f"{root}@HEAD", "--out", str(out), "--cache-root", + str(out / "caches"), "--suite", "", "--task", "review_add"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 1 + assert not (out / "check/check.json").exists() + + +def test_a_suite_gap_with_egress_does_not_abort_the_batch_under_no_network(run, capsys): + task = run["root"] / ".agent-eval/tasks/fix_add/task.json" + write(run["root"], ".agent-eval/tasks/fix_add/task.json", + json.dumps({**json.loads(task.read_text(encoding="utf-8")), "egress": ["example.com"]})) + gone = f"file://{run['tmp']}/missing@{'c' * 40}" + _commit_suite(run["root"], None, _raw(repos={"sc": gone}, tasks=[{"id": "fix_add", "repo": "sc"}])) + run["invoke"]("--suite", ".agent-eval/suite.yaml", "--task", "fix_add", "--task", "review_add", + "--arm", "guidance:none") + batch = run["out"] / "b1" + unsupported = json.loads((batch / "fix_add__guidance-none__m-1__r1/unsupported.json").read_text(encoding="utf-8")) + assert unsupported["task"] == "fix_add" + assert (batch / "review_add__guidance-none__m-1__r1/result.json").is_file() + + +def test_a_moved_hash_shaped_floating_branch_is_resolved_again_not_from_the_warm_cache(tmp_path): + root, first = _scaffold(tmp_path) + git(root, "branch", "abcdef0", first) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@abcdef0"}), allow_floating=True) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": first}, cache).pins == {"sc": first} # warms the cache at the branch + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "moved") + moved = git(root, "rev-parse", "HEAD") + git(root, "branch", "-f", "abcdef0", moved) + found = resolve_suite(suite, {"t1": moved}, cache) + assert found.pins == {"sc": moved} and found.gaps == {} + + +def test_a_parent_base_the_warm_cache_of_a_remote_lacks_is_fetched_not_a_gap(tmp_path): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + for name in ("parent", "child"): + write(root, "src/more.py", f"X = '{name}'\n") + git(root, "add", "-A") + git(root, "commit", "-qm", name) + child = git(root, "rev-parse", "HEAD") + found = resolve_suite(suite, {"t1": f"{child}^"}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + + +@pytest.mark.parametrize("suffix", ["", "^"]) +def test_an_abbreviated_base_the_warm_cache_lacks_is_fetched_not_a_gap(tmp_path, suffix): + root, sha = _scaffold(tmp_path) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the cache at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + short = git(root, "rev-parse", "--short=7", "HEAD") + found = resolve_suite(suite, {"t1": f"{short}{suffix}"}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + + +def test_a_short_base_fetch_leaves_the_full_clone_without_a_partial_filter(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "config", "uploadpack.allowFilter", "true") + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{sha}"})) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": sha}, cache).gaps == {} # warms the full clone at the pin + git(root, "checkout", "-q", "-b", "other") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "base on another branch") + short = git(root, "rev-parse", "--short=7", "HEAD") + found = resolve_suite(suite, {"t1": short}, cache) + assert found.gaps == {} and "t1" in found.git_dirs + promisor = subprocess.run(["git", "-C", str(found.git_dirs["t1"]), "config", "--get", "remote.origin.promisor"], + capture_output=True, text=True, check=False) + assert promisor.returncode != 0 # the full clone is not a partial clone + + +def test_a_full_sha_pin_is_recorded_even_when_its_repository_is_down(tmp_path): + good, sha = _scaffold(tmp_path) + gone = "b" * 40 + suite = parse_suite(_raw(repos={"good": f"{good}@{sha}", "gone": f"{tmp_path}/missing@{gone}"}, + tasks=[{"id": "ok", "repo": "good"}, {"id": "lost", "repo": "gone"}])) + found = resolve_suite(suite, {"ok": sha, "lost": gone}, tmp_path / "cache") + assert found.pins == {"good": sha, "gone": gone} + + +def test_a_floating_full_sha_pin_on_no_branch_is_fetched_by_name(tmp_path): + root, sha = _scaffold(tmp_path) + git(root, "checkout", "-q", "-b", "gone") + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "on a branch that is deleted") + pin = git(root, "rev-parse", "HEAD") + git(root, "checkout", "-q", "main") + git(root, "branch", "-D", "gone") # the pinned commit is on no branch or tag any more + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{pin}"}), allow_floating=True) + found = resolve_suite(suite, {"t1": pin}, tmp_path / "cache") + assert found.pins == {"sc": pin} and found.gaps == {} + + +def test_a_moved_full_length_floating_branch_is_resolved_again_in_a_sha256_scaffold(tmp_path): + root = tmp_path / "sc256" + write(root, "src/calc.py", CALC) + git(root, "init", "-q", "-b", "main", "--object-format=sha256") + git(root, "add", "-A") + git(root, "commit", "-qm", "scaffold") + first = git(root, "rev-parse", "HEAD") + name = "a" * 40 # a branch in a SHA-256 repository: its spelling is not a commit + git(root, "branch", name, first) + suite = parse_suite(_raw(repos={"sc": f"file://{root}@{name}"}), allow_floating=True) + cache = tmp_path / "cache" + assert resolve_suite(suite, {"t1": first}, cache).pins == {"sc": first} # warms the cache at the branch + write(root, "src/more.py", "X = 1\n") + git(root, "add", "-A") + git(root, "commit", "-qm", "moved") + moved = git(root, "rev-parse", "HEAD") + git(root, "branch", "-f", name, moved) + found = resolve_suite(suite, {"t1": moved}, cache) + assert found.pins == {"sc": moved} and found.gaps == {} diff --git a/dokimi/tests/test_repo_typecheck_grader.py b/dokimi/tests/test_repo_typecheck_grader.py index 7508d0c72..1ef5cf2a8 100644 --- a/dokimi/tests/test_repo_typecheck_grader.py +++ b/dokimi/tests/test_repo_typecheck_grader.py @@ -435,7 +435,7 @@ def test_check_task_runs_no_oracle_step_for_a_typecheck_task_without_reference(t monkeypatch.setattr(selfcheck, "_check_step", lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) - ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None), base_gaps=dict) # no suite: no gaps res = selfcheck.check_task(ctx, Task("t", raw, bare), tmp_path / "work") assert ran == [("null", True), ("oracle", False)] assert res["verdict"] == "ok" @@ -456,7 +456,7 @@ def test_check_task_runs_the_oracle_step_for_a_typecheck_task_with_reference(tmp monkeypatch.setattr(selfcheck, "_check_step", lambda ctx, task, rd, kind, run_it, net, res: ran.append((kind, run_it))) monkeypatch.setattr(selfcheck, "isolation_probe", lambda *args, **kwargs: []) - ctx = SimpleNamespace(manifest=SimpleNamespace(image=None)) + ctx = SimpleNamespace(manifest=SimpleNamespace(image=None), base_gaps=dict) # no suite: no gaps selfcheck.check_task(ctx, Task("t", raw, with_reference), tmp_path / "work") assert ran == [("null", True), ("oracle", True)]