From f6afa44d02cd5c775e36c9f58c81c091cf9cad6b Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 09:07:32 -0700 Subject: [PATCH 1/9] fix(harnesses): run a Harbor task on its prebuilt docker_image A Harbor task whose task.toml sets [environment] docker_image and ships no environment/Dockerfile failed every docker-* cell with "open Dockerfile: no such file": the harness always ran docker build. That is 10 of the 17 uxlfoundation/skills tasks (every answer-quality task and sycl-device-discovery). Follow Harbor 0.24 (environments/definition.py): should_use_prebuilt_docker_image starts from docker_image whenever it is set, also beside a Dockerfile (only --force-build builds then), and should_upload_environment_dir copies a non-empty environment/ without a Dockerfile or docker-compose.yaml into the workdir after the start; that copy of environment/.keep is what creates /app in python:3.12-slim for the uxl tasks. The image is pulled only when the daemon lacks it (compose up's policy), within build_timeout_sec; a pull timeout errors the cell like a build timeout. --network build=none does not stop the pull, as it does not stop a build's FROM. metrics.image records which image ran, with the id and repo digests of a prebuilt one. Only the Harbor format reads the key: a staged SWE-bench task builds its FROM-image Dockerfile as before. The new code is in harnesses/task_image.py; docker_solver.py shrinks (893 -> 889 lines, run() 17 -> 16). The task.toml template in contributing-terminal-bench-task.md set a local-only docker_image beside a required Dockerfile; that line goes, as such a task would now pull a tag no registry has. Signed-off-by: Vladislav Perevezentsev --- ..._vlad-perevezentsev_harbor-docker-image.md | 16 ++ dokimi/docs/coding-harnesses.md | 33 ++- .../docs/contributing-terminal-bench-task.md | 3 +- dokimi/dokimi/harnesses/docker_solver.py | 36 ++- dokimi/dokimi/harnesses/task_image.py | 83 ++++++ dokimi/dokimi/harnesses/tasks.py | 24 ++ dokimi/scripts/growth_baseline.json | 4 +- dokimi/tests/test_harbor_docker_image.py | 247 ++++++++++++++++++ 8 files changed, 420 insertions(+), 26 deletions(-) create mode 100644 dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md create mode 100644 dokimi/dokimi/harnesses/task_image.py create mode 100644 dokimi/tests/test_harbor_docker_image.py diff --git a/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md new file mode 100644 index 000000000..47c152226 --- /dev/null +++ b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md @@ -0,0 +1,16 @@ + + +### Fixed + +- **`docker-*` harnesses run a Harbor task on its prebuilt `[environment] docker_image`** — a task + that sets one and ships no `environment/Dockerfile` (10 of the 17 uxlfoundation/skills tasks: + every answer-quality task and `sycl-device-discovery`) failed its cell with `docker build`'s + "open Dockerfile: no such file". As in Harbor 0.24, the image is used whenever it is set (also + beside a Dockerfile), pulled only when the daemon lacks it, within `build_timeout_sec`, and the + loose files of `environment/` are copied into the workdir after the start (what creates `/app` + in `python:3.12-slim`). `--network build=none` does not stop the pull, as it does not stop a + build's `FROM`. `metrics.image` records the image a cell ran on: its id and repo digests for a + prebuilt one. A task without `docker_image` builds as before. See `docs/coding-harnesses.md`. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index fe78e6c56..54c8d3b81 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -155,7 +155,7 @@ real files in each format, with their sources in its `MANIFEST.yaml`. | Format | Recognised by | Kind | Consumed by | |---|---|---|---| -| `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` (with `environment/Dockerfile`) | +| `harbor` | directory with `task.toml` | executable | Harbor-backed harnesses (unless Harbor would refuse it), `docker-*` (with `environment/Dockerfile` or a [prebuilt `docker_image`](#prebuilt-images-environment-docker_image)) | | `task-md` | directory with a BenchFlow `task.md` | executable | `benchflow*`, `docker-*` (with `environment/Dockerfile`) | | `terminal-bench-v1` | directory with `task.yaml` + `run-tests.sh` / `Dockerfile` / `solution.sh` | executable | `docker-*` run it as it is; `harbor`, `benchflow` through a cached Harbor export (not a `solution.yaml` task) | | `swe-bench` | JSON/JSONL rows with `instance_id`, `repo`, `base_commit`, `problem_statement` (`FAIL_TO_PASS`) | executable | `docker-*` via `tasks run --swe-bench` (Python SWE-bench repos) | @@ -241,6 +241,35 @@ split) rather than a converter here. An export renames `oracle/` and as it works in `/app`, for the agent arms); the Harbor-backed harnesses refuse it. +### Prebuilt images: `[environment] docker_image` + +A Harbor task whose `task.toml` sets `[environment] docker_image` runs on that +image, as `harbor run` does, and a docker-* run starts from that image instead of +building: Harbor 0.24 uses it whenever it is set, also beside an +`environment/Dockerfile` (only Harbor's `--force-build` builds then; +`should_use_prebuilt_docker_image` in `harbor/environments/definition.py`). A +task without one builds `dokimi/:latest` from `environment/`, as before; so +does a staged SWE-bench task, whose Dockerfile starts `FROM` its image. Other +formats do not read the key. + +- The image is pulled only when the daemon does not have it, as Harbor's + `docker compose up` does, within the task's `build_timeout_sec`; a pull that + runs out of time errors the cell, like a build. A value that is not an image + reference (it starts with `-`, or holds a space) is refused before any docker + call: docker would read it as a flag. +- The pull is the daemon's, like the `FROM` image of a `docker build`: + `--network build=none` takes the network from a build's `RUN` steps and does + not stop it. To run offline, have the image on the daemon first. +- After the container starts, the files of a non-empty `environment/` without a + `Dockerfile` or `docker-compose.yaml` are copied into `environment.workdir`, + else the image's `WORKDIR`, else `/` (Harbor's + `should_upload_environment_dir`). The uxl answer-quality tasks rely on it: their + `environment/.keep` is what creates `/app` in `python:3.12-slim`. A top-level + `environment/skills` is left out, as it is left out of a build context. +- `metrics.image` says which image ran: `{"source": "docker_image", "ref", + "pulled", "id", "repo_digests"}` for a prebuilt one (the id and digests of what + ran, since a tag moves), `{"source": "build", "ref"}` for a built one. + ## Supported Harnesses Harbor-backed (delegate to an installed `harbor`): @@ -620,7 +649,7 @@ be checked for the same network before their verdicts are compared: | Phase | Default | What runs | |---|---|---| -| `build` | `public` (host network, host proxy as build args) | `docker build` | +| `build` | `public` (host network, host proxy as build args) | `docker build` (a [prebuilt image](#prebuilt-images-environment-docker_image)'s pull ignores it) | | `solve` | `none` (an in-container agent still reaches its LLM through the sidecar) | the oracle, or the agent | | `verify` | `none` | `tests/test.sh` | diff --git a/dokimi/docs/contributing-terminal-bench-task.md b/dokimi/docs/contributing-terminal-bench-task.md index 46105f72f..401efdd82 100644 --- a/dokimi/docs/contributing-terminal-bench-task.md +++ b/dokimi/docs/contributing-terminal-bench-task.md @@ -96,7 +96,6 @@ timeout_sec = 300.0 [environment] build_timeout_sec = 300.0 -docker_image = "intel-hpc-bench/:latest" cpus = 4 memory_mb = 2048 storage_mb = 5120 @@ -115,7 +114,7 @@ Field by field, with the consumer that actually reads it: | `agent.timeout_sec` | `TaskSpec.timeout_sec`, the solver harnesses | Wall-clock budget for the agent's attempt. Typically 300. | | `verifier.timeout_sec` | Harbor, human reviewers | Verifier budget. Keep it small (≤ 60s where possible) so CI stays fast. | | `environment.build_timeout_sec` | `docker-*` harnesses | Image-build budget; used instead of the default when set (oneAPI/pip-heavy images need > 10 min). | -| `environment.docker_image` | Harbor | Image tag Harbor expects to exist (`harbor run` does **not** build it for you). The in-repo `docker-*` harnesses build and tag `dokimi/:latest` themselves. | +| `environment.docker_image` | Harbor, `docker-*` harnesses | A prebuilt image both run instead of building `environment/Dockerfile`, pulled when the daemon lacks it, so leave it out of a task that ships a Dockerfile. Without it, the `docker-*` harnesses build and tag `dokimi/:latest`. See [Prebuilt images](./coding-harnesses.md#prebuilt-images-environment-docker_image). | | `environment.cpus`, `memory_mb` | `docker-*` harnesses | Container limits (`--cpus`, `--memory`); default to 4 CPUs / 2048 MB when absent. A malformed value falls back to the default rather than failing the run. | | `environment.network_mode` | `docker-*` harnesses, Harbor | `"no-network"` (absent means the same), `"allowlist"` with `allowed_hosts`, or `"public"` (NAT'd internet, bridge). It applies to the **verifier** and `docker-oracle`'s solve, not to an agent: an agent solves on an internal network whose only way out is the egress proxy (see `environment.egress`), unless `[agent] network_mode` says otherwise. `[agent]` and `[verifier]` take the same keys for their phase. Any other value, or `allowed_hosts` without `"allowlist"`, errors the run. The image *build* gets host network unless the operator passes `--network build=none`. See [Network policy](./coding-harnesses.md#network-policy). Prefer offline. | | `environment.egress` | `docker-claude` | Hosts the agent's solve may reach through the egress proxy, e.g. `["pypi.org", "files.pythonhosted.org"]`. A bare host means `host:443`; `host:port` is accepted. No wildcards (`*.pythonhosted.org`), commas or leading/trailing dots: list each host. A plain-HTTP host is reached directly, so it must be in the operator's `NO_PROXY`. | diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 0bb95430c..49837efa7 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -79,6 +79,7 @@ resolve_recorded, skill_needs, start_network, redact as _redact) from .skill_treatment import Treatment, build_plugin, check_skill, skill_listing, treatment_notice from .solver_stream import record_solver, run_solver +from .task_image import ignore_top_level_skills, ready_image, upload_environment from .task_env import _exec_env, unresolved_env # re-exported from .tbench_v1 import reward as _pytest_reward from .tasks import ( # noqa: F401 (timeouts re-exported) @@ -208,7 +209,6 @@ def run( log_path = output_dir / "run.log" log = log_path.open("w", encoding="utf-8") usage: Optional[UsageRecord] = None - image = f"dokimi/{task.name}:latest" container = f"dokimi-run-{task.name}-{os.getpid()}-{os.urandom(3).hex()}" # two cells in one process solve_timeout = timeout_sec or task.timeout_sec or _DEFAULT_SOLVE_TIMEOUT start = time.time() @@ -217,20 +217,13 @@ def run( network = self._network_metrics(plan) reward: Optional[float] = None sandbox: Optional[ContainerSandbox] = None + image_record: dict[str, Any] = {} try: - try: - self._build_image(task, image, log, plan.build) - except subprocess.TimeoutExpired as exc: - # The image never got built, so no agent ran and there is no - # verdict: raise so the cell is errored like a build that fails - # fast. Only a solve or verify timeout below is a reward-0 failure. - log.write(f"\nBUILD TIMEOUT: {_describe_timeout(exc)}\n") - raise RuntimeError( - f"docker build timed out after {exc.timeout}s; see run.log" - ) from exc + image, image_record = self._ready_image(task, log, plan.build) sandbox = self._open_sandbox(plan, container, log) try: bridged = self._start_container(task, image, container, log, sandbox, plan) + upload_environment(task, container, log, self._docker) try: if self.mode == "oracle": self._solve_oracle(task, container, log, plan.solve, timeout=solve_timeout) @@ -270,7 +263,7 @@ def run( (output_dir / "network.json").write_text(json.dumps(network, indent=1), encoding="utf-8") elapsed = time.time() - start - metrics: dict[str, Any] = {"dry_run": False, "mode": self.mode, "network": network} + metrics: dict[str, Any] = {"dry_run": False, "mode": self.mode, "network": network, "image": image_record} passed = reward is not None and reward >= 1.0 if usage is not None: metrics.update(usage.as_metrics_dict()) @@ -297,6 +290,16 @@ def _docker(self, args: list[str], log, **kwargs: Any) -> process.Completed: # ── pipeline steps ────────────────────────────────────────────────────── + def _ready_image(self, task: TaskSpec, log, net: PhasePolicy) -> tuple[str, dict[str, Any]]: + """The task image, built or pulled (:mod:`.task_image`). Running out of time errors the cell: + no agent ran, so there is no verdict (a solve or verify timeout is a reward-0 failure).""" + step = "pull" if task.docker_image else "build" + try: + return ready_image(task, log, self._docker, lambda tag: self._build_image(task, tag, log, net)) + except subprocess.TimeoutExpired as exc: + log.write(f"\n{step.upper()} TIMEOUT: {_describe_timeout(exc)}\n") + raise RuntimeError(f"docker {step} timed out after {exc.timeout}s; see run.log") from exc + def _build_image(self, task: TaskSpec, image: str, log, net: Optional[PhasePolicy] = None) -> None: """Build the task image under *net*: ``public`` passes the host proxy through. @@ -319,7 +322,7 @@ def _build_image(self, task: TaskSpec, image: str, log, net: Optional[PhasePolic # following them would pull host files into the image, and a # dangling one would fail the copy. shutil.copytree(env_dir, context, symlinks=True, - ignore=_ignore_top_level_skills(env_dir)) + ignore=ignore_top_level_skills(env_dir)) # Left empty rather than absent: some Dockerfiles `COPY skills # ...` (radar-vital-signs copies it into seven agent paths), and # a missing source fails the build in every arm. @@ -835,13 +838,6 @@ def _user(step: ContainerStep) -> list[str]: return ["-u", step.user] if step.user else [] -def _ignore_top_level_skills(env_dir: Path): - """``shutil.copytree`` ignore hook that drops only ``/skills``.""" - def ignore(directory: str, names: list[str]) -> list[str]: - return ["skills"] if Path(directory) == env_dir and "skills" in names else [] - return ignore - - def _describe_timeout(exc: subprocess.TimeoutExpired) -> str: """``TimeoutExpired`` for run.log: its ``str()`` holds the raw argv.""" cmd = exc.cmd if isinstance(exc.cmd, (list, tuple)) else [str(exc.cmd)] diff --git a/dokimi/dokimi/harnesses/task_image.py b/dokimi/dokimi/harnesses/task_image.py new file mode 100644 index 000000000..b8e1d7507 --- /dev/null +++ b/dokimi/dokimi/harnesses/task_image.py @@ -0,0 +1,83 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The image a ``docker-*`` task container starts from: built from the task, or Harbor's prebuilt one. + +A Harbor task with ``[environment] docker_image`` (:attr:`.TaskSpec.docker_image`) runs on that +image, as ``harbor run`` does: the image is pulled only when the daemon lacks it (what Harbor's +``docker compose up`` does), and once the container runs, the task's loose ``environment/`` files +are copied into its workdir (:attr:`.TaskSpec.environment_upload`). Any other task builds +``dokimi/:latest``. ``metrics.image`` records which ran, and a pulled image's id and digests. + +The pull is the daemon's, like a ``docker build``'s base image: ``--network build=none``, which +takes the network away from a build's ``RUN`` steps, does not stop it. +""" + +from __future__ import annotations + +import json +import re +import shutil +import tempfile +from pathlib import Path +from typing import Any, Callable + +from dokimi.docker_cmd import Docker + +from .tasks import TaskSpec, _task_build_timeout + +# One JSON object per image; RepoDigests is empty for an image that came from no registry. +_RECORD_FORMAT = '{"id": {{json .Id}}, "repo_digests": {{json .RepoDigests}}}' +# A reference starts with a letter or digit (distribution/reference); "-..." would be a docker flag. +_REFERENCE_RE = re.compile(r"[A-Za-z0-9]\S*") + + +def ready_image(task: TaskSpec, log, docker: Docker, build: Callable[[str], None]) -> tuple[str, dict[str, Any]]: + """The image to start the task container from, and its ``metrics.image`` record. + + *build* builds the task's Dockerfile under the tag it is given. The pull gets the task's + ``build_timeout_sec``: running out raises :class:`subprocess.TimeoutExpired`, as a build does. + """ + image = task.docker_image + if image is None: + tag = f"dokimi/{task.name}:latest" + build(tag) + return tag, {"source": "build", "ref": tag} + if not _REFERENCE_RE.fullmatch(image): + raise ValueError(f"{task.name}: [environment] docker_image {image!r} is not a docker image reference") + inspect = ["image", "inspect", "-f", _RECORD_FORMAT, image] + found = docker(inspect, log, check=False) + pulled = found.returncode != 0 + if pulled: + docker(["pull", "-q", image], log, timeout=_task_build_timeout(task)) + found = docker(inspect, log) + return image, {"source": "docker_image", "ref": image, "pulled": pulled, **json.loads(found.stdout or "")} + + +def upload_environment(task: TaskSpec, container: str, log, docker: Docker) -> None: + """Copy :attr:`.TaskSpec.environment_upload` into the started container, as Harbor does. + + Into ``environment.workdir``, else the image's ``WORKDIR``, else ``/`` (Harbor's + ``_upload_environment_dir_after_start`` runs ``pwd``). A top-level ``skills/`` stays out, as + it stays out of a build context. + """ + src = task.environment_upload + if src is None: + return + workdir = task.environment.get("workdir") + if not isinstance(workdir, str) or not workdir: + workdir = (docker(["inspect", "-f", "{{.Config.WorkingDir}}", container], log).stdout or "").strip() or "/" + with tempfile.TemporaryDirectory(prefix="dokimi-upload-") as tmp: + if task.skills_dir is not None: + staged = Path(tmp) / "environment" + shutil.copytree(src, staged, symlinks=True, ignore=ignore_top_level_skills(src)) + src = staged + docker(["exec", "-u", "root", container, "mkdir", "-p", workdir], log) + docker(["cp", f"{src}/.", f"{container}:{workdir}"], log) + + +def ignore_top_level_skills(env_dir: Path): + """``shutil.copytree`` ignore hook that drops only ``/skills``.""" + def ignore(directory: str, names: list[str]) -> list[str]: + return ["skills"] if Path(directory) == env_dir and "skills" in names else [] + return ignore diff --git a/dokimi/dokimi/harnesses/tasks.py b/dokimi/dokimi/harnesses/tasks.py index adecabf5c..60d66090f 100644 --- a/dokimi/dokimi/harnesses/tasks.py +++ b/dokimi/dokimi/harnesses/tasks.py @@ -128,6 +128,30 @@ def dockerfile(self) -> str: """The Dockerfile, relative to :attr:`build_context`.""" return str(self._tb.get("dockerfile") or "Dockerfile") + @property + def docker_image(self) -> str | None: + """A Harbor task's prebuilt ``[environment] docker_image``, run instead of a build; else ``None``. + + Harbor 0.24 starts from it whenever it is set, also beside an ``environment/Dockerfile`` + (``environments/definition.py::should_use_prebuilt_docker_image``; only ``--force-build`` + builds then, and dokimi has no such flag). Other formats build: a staged SWE-bench task's + Dockerfile starts ``FROM`` its ``docker_image``. + """ + image = self.environment.get("docker_image") + return image if self.format == FORMAT_HARBOR and isinstance(image, str) and image else None + + @property + def environment_upload(self) -> Path | None: + """``environment/``, when Harbor copies its files into the workdir of the started container. + + Harbor does so for a :attr:`docker_image` task whose non-empty ``environment/`` has no + ``Dockerfile`` or ``docker-compose.yaml`` (``should_upload_environment_dir``, same file). + """ + env = self.path / "environment" + if self.docker_image is None or any((env / n).exists() for n in ("Dockerfile", "docker-compose.yaml")): + return None + return env if env.is_dir() and any(env.iterdir()) else None + @property def container_env(self) -> dict[str, str]: """Environment the task container starts with (terminal-bench's ``TEST_DIR`` and compose env).""" diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 909294f3d..5f5fa7ddc 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,7 +1,7 @@ { "files": { "dokimi/harnesses/benchflow.py": {"lines": 869}, - "dokimi/harnesses/docker_solver.py": {"lines": 893, "rationale": "pre-existing when the gate was introduced"}, + "dokimi/harnesses/docker_solver.py": {"lines": 889, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1031, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, "dokimi/report/model_compare.py": {"lines": 1150, "rationale": "grew on main in #73 (judge-validity --context stored) before the ratchet landed; split it in its own PR"}, "tests/test_arm_runner.py": {"lines": 1175, "rationale": "pre-existing when the gate was introduced"}, @@ -35,7 +35,7 @@ "dokimi/harnesses/benchflow.py::BenchflowHarness.run": {"complexity": 19}, "dokimi/harnesses/benchflow.py::skill_loads": {"complexity": 12}, "dokimi/harnesses/command.py::CommandHarness.run": {"complexity": 15, "rationale": "pre-existing when the gate was introduced"}, - "dokimi/harnesses/docker_solver.py::DockerSolverHarness.run": {"complexity": 17}, + "dokimi/harnesses/docker_solver.py::DockerSolverHarness.run": {"complexity": 16}, "dokimi/harnesses/exploration.py::summarize_exploration": {"complexity": 15}, "dokimi/harnesses/registry.py::_create": {"complexity": 17, "rationale": "pre-existing when the gate was introduced"}, "dokimi/harnesses/runner.py::summarize_task_results": {"complexity": 14, "rationale": "pre-existing when the gate was introduced"}, diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py new file mode 100644 index 000000000..a0662de04 --- /dev/null +++ b/dokimi/tests/test_harbor_docker_image.py @@ -0,0 +1,247 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A Harbor task's prebuilt ``[environment] docker_image`` in the ``docker-*`` harnesses. + +The contract is docs/coding-harnesses.md "Prebuilt images": the image Harbor 0.24 would start +(``should_use_prebuilt_docker_image``), pulled only when missing, the loose ``environment/`` +files copied into the workdir (``should_upload_environment_dir``), and ``metrics.image``. +Docker is a :class:`DockerCLI` fake: the tests read the argv the harness would run. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import replace +from pathlib import Path + +import pytest + +from dokimi.harnesses import TaskSuiteRunner, load_task +from dokimi.harnesses.registry import create_harness +from dokimi.harnesses.tasks import FORMAT_SWE_BENCH, TaskSpec, _task_build_timeout +from dokimi.process import reply +from docker_cli_fake import DockerCLI + +IMAGE = "python:3.12-slim" +RECORD = {"id": "sha256:5e2b", "repo_digests": ["python@sha256:5e2b"]} + + +@pytest.fixture(autouse=True) +def _hermetic(tmp_path, monkeypatch): + """No developer proxy or provider reaches the argv; the host `claude` is a path never run.""" + import dokimi.harnesses.container_agent as container_mod + + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path / "no-operator-config")) + for name in list(os.environ): + if name.lower() in ("http_proxy", "https_proxy", "no_proxy", "all_proxy") or name.startswith( + ("ANTHROPIC_", "CLAUDE_CODE_", "AWS_", "CLOUD_ML_", "GOOGLE_", "VERTEX_")): + monkeypatch.delenv(name) + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + monkeypatch.setattr(container_mod, "claude_binary", lambda: tmp_path / "claude") + + +def _task(tmp_path: Path, *, image: str | None = IMAGE, dockerfile: bool = False, + env_files: tuple[str, ...] = (".keep",), workdir: str | None = "/app") -> Path: + """A uxl answer-quality task (onetbb-bounded-image-flow-graph's shape at uxlfoundation/skills 6feb3f3).""" + task = tmp_path / "answer-task" + for d in ("environment", "tests", "solution"): + (task / d).mkdir(parents=True) + (task / "tests" / "test.sh").write_text("#!/bin/bash\n", encoding="utf-8") + (task / "solution" / "solve.sh").write_text("cp /solution/answer.md /app/answer.md\n", encoding="utf-8") + (task / "instruction.md").write_text("Write /app/answer.md.", encoding="utf-8") + for name in env_files: + (task / "environment" / name).parent.mkdir(parents=True, exist_ok=True) + (task / "environment" / name).write_text("", encoding="utf-8") + if dockerfile: + (task / "environment" / "Dockerfile").write_text("FROM ubuntu:24.04\n", encoding="utf-8") + env = [f'docker_image = "{image}"'] if image else [] + env += [f'workdir = "{workdir}"'] if workdir else [] + (task / "task.toml").write_text( + "[agent]\ntimeout_sec = 30\n[environment]\nbuild_timeout_sec = 77\n" + "\n".join(env) + "\n", + encoding="utf-8") + return task + + +def _daemon(*, has_image: bool = False, answer=None) -> DockerCLI: + """A daemon that has *IMAGE* only after a pull (or from the start, with *has_image*).""" + state = {"present": has_image} + + def handle(call): + args = call.argv[1:] + if answer is not None and (got := answer(call)) is not None: + return got + if args[:1] == ("pull",): + state["present"] = True + if args[:2] == ("image", "inspect") and args[-1] == IMAGE: + if not state["present"]: + return reply(stderr=f"Error: No such image: {IMAGE}", returncode=1) + return reply(json.dumps(RECORD) + "\n") + return None + + return DockerCLI(answer=handle) + + +def _run_argv(cli: DockerCLI) -> list[str]: + return next(a for a in cli.args if a[0] == "run" and "sleep" in a) + + +@pytest.mark.parametrize("harness", ["docker-oracle", "docker-claude", "docker-probe"]) +def test_a_docker_image_task_pulls_and_runs_that_image_without_a_build(tmp_path, harness): + """docs/coding-harnesses.md: "a docker-* run starts from that image instead of building".""" + task = load_task(_task(tmp_path)) + cli = _daemon() + result = create_harness(harness, runner=cli).run(task, model="haiku", output_dir=tmp_path / "out") + + assert not [a for a in cli.args if a[0] == "build"] + assert ["pull", "-q", IMAGE] in cli.args + run = _run_argv(cli) + assert run[run.index("sleep") - 1] == IMAGE + pull = next(c for c in cli.calls if c.argv[1] == "pull") + assert pull.timeout == _task_build_timeout(task) == 77 + assert result.metrics["image"] == {"source": "docker_image", "ref": IMAGE, "pulled": True, **RECORD} + + +def test_an_image_the_daemon_has_is_not_pulled(tmp_path): + """docs/coding-harnesses.md: "pulled only when the daemon does not have it", as compose up does.""" + task = load_task(_task(tmp_path)) + cli = _daemon(has_image=True) + result = create_harness("docker-oracle", runner=cli).run(task, output_dir=tmp_path / "out") + assert not [a for a in cli.args if a[0] in ("pull", "build")] + assert result.metrics["image"]["pulled"] is False + assert result.metrics["image"]["repo_digests"] == RECORD["repo_digests"] + + +def test_the_loose_environment_files_land_in_the_workdir_before_the_solve(tmp_path): + """docs/coding-harnesses.md: the files of environment/ are copied into the workdir after the start. + + python:3.12-slim has no /app; Harbor's copy of environment/.keep is what creates it.""" + path = _task(tmp_path, env_files=(".keep", "probe.py")) + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + + copy = [a for a in cli.args if a[0] == "cp" and a[1] == f"{path / 'environment'}/."] + assert len(copy) == 1 and copy[0][2].endswith(":/app") + order = [a[0] if a[0] != "cp" else a[1] for a in cli.args] + assert order.index("run") < order.index(f"{path / 'environment'}/.") < order.index(f"{path / 'solution'}/.") + assert ["exec", "-u", "root", copy[0][2].split(":")[0], "mkdir", "-p", "/app"] in cli.args + + +def test_without_a_declared_workdir_the_files_go_to_the_image_workdir_else_root(tmp_path): + """Harbor uploads into environment.workdir, else the container's `pwd` (the image's WORKDIR, or /).""" + path = _task(tmp_path, workdir=None) + for workdir, dest in (("/srv/task\n", "/srv/task"), ("\n", "/")): + cli = _daemon(answer=lambda call, w=workdir: reply(w) if "{{.Config.WorkingDir}}" in call.argv else None) + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + copy = next(a for a in cli.args if a[0] == "cp" and a[1] == f"{path / 'environment'}/.") + assert copy[2].split(":", 1)[1] == dest + + +def test_bundled_skills_are_not_copied_into_the_workdir(tmp_path): + """environment/skills stays out of the container, as it stays out of a build context.""" + path = _task(tmp_path, env_files=("probe.py", "skills/s/SKILL.md")) + seen: list[list[str]] = [] + + def record(call): + if call.argv[1] == "cp" and call.argv[-1].endswith(":/app"): + src = Path(call.argv[2].rstrip(".")) + seen.append(sorted(str(p.relative_to(src)) for p in src.rglob("*"))) + return None + + create_harness("docker-oracle", runner=_daemon(answer=record)).run(load_task(path), output_dir=tmp_path / "out") + assert seen == [["probe.py"]] + + +def test_an_empty_environment_dir_is_not_copied(tmp_path): + """Harbor's should_upload_environment_dir: nothing to upload from an empty environment/.""" + path = _task(tmp_path, env_files=()) + assert load_task(path).environment_upload is None + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + assert not [a for a in cli.args if a[0] == "cp" and a[1] == f"{path / 'environment'}/."] + + +def test_a_dockerfile_only_task_still_builds_its_own_image(tmp_path): + """docs/coding-harnesses.md: a task without docker_image builds dokimi/:latest, as before.""" + path = _task(tmp_path, image=None, dockerfile=True, env_files=("src.cpp",)) + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + + builds = [a for a in cli.args if a[0] == "build"] + env = path / "environment" + assert builds == [["build", "-q", "--network", "host", "-t", "dokimi/answer-task:latest", + "-f", str(env / "Dockerfile"), str(env)]] + assert not [a for a in cli.args if a[0] == "pull" or a[:2] == ["image", "inspect"]] + assert not [a for a in cli.args if a[0] == "cp" and a[1] == f"{env}/."] + run = _run_argv(cli) + assert run[run.index("sleep") - 1] == "dokimi/answer-task:latest" + + +def test_a_built_image_is_recorded_as_built(tmp_path): + """docs/coding-harnesses.md: `metrics.image` says which path ran.""" + path = _task(tmp_path, image=None, dockerfile=True) + result = create_harness("docker-oracle", runner=_daemon()).run(load_task(path), output_dir=tmp_path / "out") + assert result.metrics["image"] == {"source": "build", "ref": "dokimi/answer-task:latest"} + + +def test_with_both_the_docker_image_wins_and_nothing_is_uploaded(tmp_path): + """Harbor 0.24 should_use_prebuilt_docker_image: docker_image wins unless --force-build; + should_upload_environment_dir: no upload beside a Dockerfile.""" + path = _task(tmp_path, dockerfile=True, env_files=("src.cpp",)) + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + assert not [a for a in cli.args if a[0] == "build"] + assert ["pull", "-q", IMAGE] in cli.args + assert not [a for a in cli.args if a[0] == "cp" and a[1] == f"{path / 'environment'}/."] + + +def test_only_a_harbor_task_reads_docker_image(tmp_path): + """A staged SWE-bench task's Dockerfile starts FROM its docker_image: it is built, not pulled.""" + spec = TaskSpec(name="t", path=tmp_path, instruction="", environment={"docker_image": IMAGE}, + format=FORMAT_SWE_BENCH) + assert spec.docker_image is None + assert replace(spec, format="harbor").docker_image == IMAGE + + +def test_the_pull_happens_under_network_build_none(tmp_path): + """docs/coding-harnesses.md: the pull is the daemon's, like a build's FROM; build=none does not stop it.""" + harness = replace(create_harness("docker-oracle", runner=_daemon()), network=("build=none",)) + result = harness.run(load_task(_task(tmp_path)), output_dir=tmp_path / "out") + assert ["pull", "-q", IMAGE] in harness.runner.args + assert result.metrics["network"]["build"]["mode"] == "none" + assert result.metrics["image"]["pulled"] is True + + +def test_a_pull_that_runs_out_of_time_errors_the_cell(tmp_path): + """Like a build timeout: no agent ran, so there is no verdict (docs/coding-harnesses.md).""" + task = load_task(_task(tmp_path)) + cli = _daemon(answer=lambda call: reply(timed_out=True) if call.argv[1] == "pull" else None) + with pytest.raises(RuntimeError, match="docker pull timed out after 77"): + create_harness("docker-oracle", runner=cli).run(task, output_dir=tmp_path / "out") + assert "PULL TIMEOUT" in (tmp_path / "out" / "run.log").read_text(encoding="utf-8") + + row = TaskSuiteRunner(tasks=[task], harnesses=["docker-oracle"], output_root=tmp_path / "runs", + runner=cli).run()["results"][0] + assert row["reward"] is None and "docker pull timed out" in row["metrics"]["error"] + + +def test_a_failed_pull_errors_the_cell_before_any_container(tmp_path): + """docs/coding-harnesses.md "Prebuilt images": a pull fails the cell like a build, before any run.""" + task = load_task(_task(tmp_path)) + cli = _daemon(answer=lambda call: reply(stderr="manifest unknown", returncode=1) + if call.argv[1] == "pull" else None) + with pytest.raises(RuntimeError, match=r"docker pull failed \(exit 1\)"): + create_harness("docker-oracle", runner=cli).run(task, output_dir=tmp_path / "out") + assert not [a for a in cli.args if a[0] == "run"] + + +@pytest.mark.parametrize("image", ["--privileged", "python:3.12 -slim"]) +def test_a_docker_image_that_is_not_a_reference_is_refused(tmp_path, image): + """distribution/reference: a reference starts with a letter or digit and has no space; + "--privileged" would reach docker as a flag.""" + task = load_task(_task(tmp_path, image=image)) + cli = _daemon() + with pytest.raises(ValueError, match="is not a docker image reference"): + create_harness("docker-oracle", runner=cli).run(task, output_dir=tmp_path / "out") + assert not [a for a in cli.args if image in a] From a5131a7e6d53d757a0931e9c70b3175270c8c135 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 09:23:33 -0700 Subject: [PATCH 2/9] fix(harnesses): remove the task container when the environment upload fails The copy of environment/ into a prebuilt-image container ran after the container started but outside the try whose finally runs `docker rm -f`, so a failed mkdir, inspect or cp left a container sleeping for at least 1800 s. It now runs inside that try. Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/harnesses/docker_solver.py | 2 +- dokimi/tests/test_harbor_docker_image.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index 49837efa7..d770f8091 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -223,8 +223,8 @@ def run( sandbox = self._open_sandbox(plan, container, log) try: bridged = self._start_container(task, image, container, log, sandbox, plan) - upload_environment(task, container, log, self._docker) try: + upload_environment(task, container, log, self._docker) if self.mode == "oracle": self._solve_oracle(task, container, log, plan.solve, timeout=solve_timeout) elif self.mode == "claude": diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index a0662de04..ab133f651 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -138,6 +138,18 @@ def test_without_a_declared_workdir_the_files_go_to_the_image_workdir_else_root( assert copy[2].split(":", 1)[1] == dest +def test_a_failed_upload_still_removes_the_task_container(tmp_path): + """The task container (`sleep` >= 1800 s) is removed whatever step fails after its start.""" + path = _task(tmp_path) + cli = _daemon(answer=lambda call: reply(stderr="cp failed", returncode=1) if ( + call.argv[1] == "cp" and call.argv[2] == f"{path / 'environment'}/.") else None) + with pytest.raises(RuntimeError, match=r"docker cp failed \(exit 1\)"): + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + run = next(i for i, a in enumerate(cli.args) if a[0] == "run") + container = cli.args[run][cli.args[run].index("--name") + 1] + assert ["rm", "-f", container] in cli.args[run + 1:] + + def test_bundled_skills_are_not_copied_into_the_workdir(tmp_path): """environment/skills stays out of the container, as it stays out of a build context.""" path = _task(tmp_path, env_files=("probe.py", "skills/s/SKILL.md")) From f517a49426a77a20050817bd951aa2de7d8e252f Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 09:24:31 -0700 Subject: [PATCH 3/9] fix(task-formats): eval inspect lists docker-* for a prebuilt docker_image task The Harbor probe counted a task as docker-* runnable only with an environment/Dockerfile, so `eval inspect` left docker-* out for the docker_image-only tasks the docker harnesses now run (e.g. uxl's onetbb-bounded-image-flow-graph printed `-> harbor`). It now accepts TaskSpec.docker_image as well; the verifier is still required. Signed-off-by: Vladislav Perevezentsev --- dokimi/dokimi/task_formats/dirs.py | 12 +++++++----- dokimi/tests/test_task_format_detect.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/dokimi/dokimi/task_formats/dirs.py b/dokimi/dokimi/task_formats/dirs.py index 4d03d1299..03c574dd0 100644 --- a/dokimi/dokimi/task_formats/dirs.py +++ b/dokimi/dokimi/task_formats/dirs.py @@ -73,7 +73,7 @@ def describe(self, path: Path) -> Report: report.problems.append(f"Harbor would refuse it: {problem}") else: report.consumers.append(BY_HARBOR) - if _docker_runnable(path): + if _docker_runnable(path, spec): report.consumers.append(BY_DOCKER) return report @@ -115,7 +115,7 @@ def describe(self, path: Path) -> Report: report.details["metadata"] = _summary(spec.metadata) report.details["schema_version"] = spec.schema_version report.consumers.append(BY_BENCHFLOW) - if _docker_runnable(path): + if _docker_runnable(path, spec): report.consumers.append(BY_DOCKER) return report @@ -174,9 +174,11 @@ def describe(self, path: Path) -> Report: return report -def _docker_runnable(path: Path) -> bool: - """docker-* builds ``environment/Dockerfile`` and runs a verifier: without either it errors.""" - return (path / "environment" / "Dockerfile").is_file() and _any_dir(path, "tests", "verifier") +def _docker_runnable(path: Path, spec: task_layouts.TaskSpec) -> bool: + """docker-* builds ``environment/Dockerfile`` or starts from a prebuilt ``docker_image``, and runs + a verifier: without either it errors.""" + image = (path / "environment" / "Dockerfile").is_file() or spec.docker_image is not None + return image and _any_dir(path, "tests", "verifier") def _summary(table: dict[str, Any], keys: tuple[str, ...] = ("difficulty", "category", "tags")) -> dict[str, Any]: diff --git a/dokimi/tests/test_task_format_detect.py b/dokimi/tests/test_task_format_detect.py index 74555bf84..05c6e7818 100644 --- a/dokimi/tests/test_task_format_detect.py +++ b/dokimi/tests/test_task_format_detect.py @@ -126,6 +126,16 @@ def test_task_without_environment_dockerfile_is_not_claimed_for_docker(tmp_path) assert "docker-*" not in _one(task).consumers +def test_harbor_task_with_a_prebuilt_docker_image_is_claimed_for_docker(tmp_path): + """docs/coding-harnesses.md: docker-* run a Harbor task on its [environment] docker_image.""" + task = _harbor(tmp_path) + (task / "environment" / "Dockerfile").unlink() + _write(task / "task.toml", '[metadata]\ndifficulty = "easy"\n[environment]\ndocker_image = "python:3.12-slim"\n') + assert _one(task).consumers == ["harbor", "docker-*"] + shutil.rmtree(task / "tests") # still no verifier, no docker-* + assert "docker-*" not in _one(task).consumers + + def test_task_without_tests_or_verifier_is_not_claimed_for_docker(tmp_path): for task in (_harbor(tmp_path), _task_md(tmp_path)): for verifier in ("tests", "verifier"): From ba13076c492543c183a5a9b2ee065db7da5ed7cb Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 09:24:55 -0700 Subject: [PATCH 4/9] fix(harnesses): say in run.log that docker_image skips the Dockerfile A task that sets [environment] docker_image beside environment/Dockerfile now runs on the image, as Harbor does, and no longer builds the Dockerfile (the old task.toml template wrote such a pair with a local-only tag). run.log now names the Dockerfile that was not built, and the changelog fragment and the doc say to remove docker_image to keep building it. Signed-off-by: Vladislav Perevezentsev --- ...261117_160000_vlad-perevezentsev_harbor-docker-image.md | 7 ++++++- dokimi/docs/coding-harnesses.md | 3 ++- dokimi/dokimi/harnesses/task_image.py | 2 ++ dokimi/tests/test_harbor_docker_image.py | 7 +++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md index 47c152226..63a69dbe4 100644 --- a/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md +++ b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md @@ -13,4 +13,9 @@ SPDX-License-Identifier: Apache-2.0 loose files of `environment/` are copied into the workdir after the start (what creates `/app` in `python:3.12-slim`). `--network build=none` does not stop the pull, as it does not stop a build's `FROM`. `metrics.image` records the image a cell ran on: its id and repo digests for a - prebuilt one. A task without `docker_image` builds as before. See `docs/coding-harnesses.md`. + prebuilt one. A task without `docker_image` builds as before. A task that sets `docker_image` + beside `environment/Dockerfile` (as the old `task.toml` template in + `contributing-terminal-bench-task.md` did, with a local `intel-hpc-bench/...` tag) now runs on + that image and no longer builds the Dockerfile; `run.log` says so, and a tag no registry has + errors the cell with `docker pull failed`. Remove `docker_image` to keep building the + Dockerfile. See `docs/coding-harnesses.md`. diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 54c8d3b81..0e5bb8142 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -247,7 +247,8 @@ A Harbor task whose `task.toml` sets `[environment] docker_image` runs on that image, as `harbor run` does, and a docker-* run starts from that image instead of building: Harbor 0.24 uses it whenever it is set, also beside an `environment/Dockerfile` (only Harbor's `--force-build` builds then; -`should_use_prebuilt_docker_image` in `harbor/environments/definition.py`). A +`should_use_prebuilt_docker_image` in `harbor/environments/definition.py`; +`run.log` names the Dockerfile not built: remove `docker_image` to build it). A task without one builds `dokimi/:latest` from `environment/`, as before; so does a staged SWE-bench task, whose Dockerfile starts `FROM` its image. Other formats do not read the key. diff --git a/dokimi/dokimi/harnesses/task_image.py b/dokimi/dokimi/harnesses/task_image.py index b8e1d7507..9a7eb5b82 100644 --- a/dokimi/dokimi/harnesses/task_image.py +++ b/dokimi/dokimi/harnesses/task_image.py @@ -45,6 +45,8 @@ def ready_image(task: TaskSpec, log, docker: Docker, build: Callable[[str], None return tag, {"source": "build", "ref": tag} if not _REFERENCE_RE.fullmatch(image): raise ValueError(f"{task.name}: [environment] docker_image {image!r} is not a docker image reference") + if (task.build_context / task.dockerfile).is_file(): + log.write(f"[image] {task.dockerfile} not built: [environment] docker_image {image} wins, as in Harbor\n") inspect = ["image", "inspect", "-f", _RECORD_FORMAT, image] found = docker(inspect, log, check=False) pulled = found.returncode != 0 diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index ab133f651..09f9987c4 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -206,6 +206,13 @@ def test_with_both_the_docker_image_wins_and_nothing_is_uploaded(tmp_path): assert not [a for a in cli.args if a[0] == "build"] assert ["pull", "-q", IMAGE] in cli.args assert not [a for a in cli.args if a[0] == "cp" and a[1] == f"{path / 'environment'}/."] + log = (tmp_path / "out" / "run.log").read_text(encoding="utf-8") + assert f"Dockerfile not built: [environment] docker_image {IMAGE} wins" in log + + +def test_a_docker_image_only_task_says_nothing_about_a_dockerfile(tmp_path): + create_harness("docker-oracle", runner=_daemon()).run(load_task(_task(tmp_path)), output_dir=tmp_path / "out") + assert "not built" not in (tmp_path / "out" / "run.log").read_text(encoding="utf-8") def test_only_a_harbor_task_reads_docker_image(tmp_path): From 26b7e9b29655028b366a78ce22dc3bd6289ebd6d Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:29:57 -0700 Subject: [PATCH 5/9] fix(harnesses): run a prebuilt-image task container in its declared workdir Harbor runs every exec of a task in environment.workdir (DockerEnvironment.exec: cwd or task_env_config.workdir). The docker-* harnesses gave the container no -w, so on an image without a WORKDIR (python:3.12-slim) the oracle and the verifier ran in / while the environment/ files had been copied to /app, and a relative path in solve.sh or test.sh missed them. A prebuilt-image task with a declared workdir now starts with `docker run -w `; a built image keeps its own WORKDIR, as before. Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/coding-harnesses.md | 3 +++ dokimi/dokimi/harnesses/docker_solver.py | 4 ++-- dokimi/dokimi/harnesses/task_image.py | 7 +++++++ dokimi/tests/test_harbor_docker_image.py | 16 ++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 0e5bb8142..179b82354 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -261,6 +261,9 @@ formats do not read the key. - The pull is the daemon's, like the `FROM` image of a `docker build`: `--network build=none` takes the network from a build's `RUN` steps and does not stop it. To run offline, have the image on the daemon first. +- The container runs in `environment.workdir` when the task declares one (`docker run -w`), as + Harbor runs every exec there, so the oracle and the verifier find relative paths; without one, + in the image's `WORKDIR`. - After the container starts, the files of a non-empty `environment/` without a `Dockerfile` or `docker-compose.yaml` are copied into `environment.workdir`, else the image's `WORKDIR`, else `/` (Harbor's diff --git a/dokimi/dokimi/harnesses/docker_solver.py b/dokimi/dokimi/harnesses/docker_solver.py index d770f8091..a6d1fb8a3 100644 --- a/dokimi/dokimi/harnesses/docker_solver.py +++ b/dokimi/dokimi/harnesses/docker_solver.py @@ -79,7 +79,7 @@ resolve_recorded, skill_needs, start_network, redact as _redact) from .skill_treatment import Treatment, build_plugin, check_skill, skill_listing, treatment_notice from .solver_stream import record_solver, run_solver -from .task_image import ignore_top_level_skills, ready_image, upload_environment +from .task_image import ignore_top_level_skills, ready_image, upload_environment, workdir_args from .task_env import _exec_env, unresolved_env # re-exported from .tbench_v1 import reward as _pytest_reward from .tasks import ( # noqa: F401 (timeouts re-exported) @@ -359,7 +359,7 @@ def _start_container(self, task: TaskSpec, image: str, container: str, log, sandbox_flags = sandbox.run_args() if sandbox else [] self._docker(docker_cmd.run_spec( task_container_policy(network=network, cpus=cpus, memory=memory), - [["-d"], ["--name", container], "init", "cap_drop", "security", "pids", "cap_add", + [["-d"], ["--name", container], workdir_args(task), "init", "cap_drop", "security", "pids", "cap_add", "network", sandbox_flags, docker_cmd.env_args(task.container_env), "resource"], image, ["sleep", str(lifetime)]), log) return enter_phase(plan.solve if inside else DEFAULTS["solve"], container, self._docker, log, diff --git a/dokimi/dokimi/harnesses/task_image.py b/dokimi/dokimi/harnesses/task_image.py index 9a7eb5b82..8d529822b 100644 --- a/dokimi/dokimi/harnesses/task_image.py +++ b/dokimi/dokimi/harnesses/task_image.py @@ -56,6 +56,13 @@ def ready_image(task: TaskSpec, log, docker: Docker, build: Callable[[str], None return image, {"source": "docker_image", "ref": image, "pulled": pulled, **json.loads(found.stdout or "")} +def workdir_args(task: TaskSpec) -> list[str]: + """``-w`` for a prebuilt-image task's ``environment.workdir``: Harbor runs every exec there + (``DockerEnvironment.exec``: ``cwd or task_env_config.workdir``); a built image has its ``WORKDIR``.""" + workdir = task.environment.get("workdir") + return ["-w", workdir] if task.docker_image is not None and isinstance(workdir, str) and workdir else [] + + def upload_environment(task: TaskSpec, container: str, log, docker: Docker) -> None: """Copy :attr:`.TaskSpec.environment_upload` into the started container, as Harbor does. diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index 09f9987c4..2e0362635 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -128,6 +128,22 @@ def test_the_loose_environment_files_land_in_the_workdir_before_the_solve(tmp_pa assert ["exec", "-u", "root", copy[0][2].split(":")[0], "mkdir", "-p", "/app"] in cli.args +def test_the_task_container_works_in_the_declared_workdir(tmp_path): + """Harbor's DockerEnvironment.exec runs in environment.workdir, so a solve.sh or test.sh with a + relative path (`cp seed.txt answer.txt`) finds the uploaded files; python:3.12-slim has no WORKDIR.""" + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(_task(tmp_path)), output_dir=tmp_path / "out") + run = _run_argv(cli) + assert run[run.index("-w") + 1] == "/app" and run.index("-w") < run.index(IMAGE) + + +def test_without_a_declared_workdir_the_container_keeps_the_image_workdir(tmp_path): + for path in (_task(tmp_path / "a", workdir=None), _task(tmp_path / "b", image=None, dockerfile=True)): + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + assert "-w" not in _run_argv(cli) + + def test_without_a_declared_workdir_the_files_go_to_the_image_workdir_else_root(tmp_path): """Harbor uploads into environment.workdir, else the container's `pwd` (the image's WORKDIR, or /).""" path = _task(tmp_path, workdir=None) From ccc543309c8ec0b78076a4fb9ef3366043a7671b Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:31:06 -0700 Subject: [PATCH 6/9] fix(tasks): refuse a Harbor docker_image that is not a string `docker_image = 5` (or a list, a boolean) was read as unset, so the task was built from a Dockerfile it may not have, while Harbor 0.24 refuses it: EnvironmentConfig types the field `str | None`, and pydantic rejects a non-string ("Input should be a valid string"). load_task now raises ValueError for such a value, naming the file. Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/coding-harnesses.md | 4 +++- dokimi/dokimi/harnesses/tasks.py | 9 +++++++++ dokimi/tests/test_harbor_docker_image.py | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 179b82354..075092abf 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -257,7 +257,9 @@ formats do not read the key. `docker compose up` does, within the task's `build_timeout_sec`; a pull that runs out of time errors the cell, like a build. A value that is not an image reference (it starts with `-`, or holds a space) is refused before any docker - call: docker would read it as a flag. + call: docker would read it as a flag. A `docker_image` that is not a string + (`docker_image = 5`) fails the task's load, as Harbor's `EnvironmentConfig` refuses + it; an empty string is no image. - The pull is the daemon's, like the `FROM` image of a `docker build`: `--network build=none` takes the network from a build's `RUN` steps and does not stop it. To run offline, have the image on the daemon first. diff --git a/dokimi/dokimi/harnesses/tasks.py b/dokimi/dokimi/harnesses/tasks.py index 60d66090f..9210c5954 100644 --- a/dokimi/dokimi/harnesses/tasks.py +++ b/dokimi/dokimi/harnesses/tasks.py @@ -376,6 +376,7 @@ def load_task(path_or_name: str | Path, root: Path | None = None) -> TaskSpec: f"'environment' is the legacy Harbor spelling); keep only 'sandbox': {path}" ) environment = _table(data.get("sandbox")) or _table(data.get("environment")) + _check_docker_image(fmt, environment, path) version = data.get("schema_version") spec = TaskSpec( name=path.name, @@ -398,6 +399,14 @@ def load_task(path_or_name: str | Path, root: Path | None = None) -> TaskSpec: return spec +def _check_docker_image(fmt: str, environment: dict[str, Any], path: Path) -> None: + """Refuse a Harbor ``docker_image`` that is not a string, as Harbor 0.24's ``EnvironmentConfig`` + (``docker_image: str | None``) does, rather than build a task that names a prebuilt image.""" + image = environment.get("docker_image") + if fmt == FORMAT_HARBOR and image is not None and not isinstance(image, str): + raise ValueError(f"task.toml [environment] docker_image must be a string, not {image!r}: {path}") + + def read_task_config(path: Path) -> dict[str, Any]: """The raw config tables of a task directory, without the instruction. diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index 2e0362635..14002900d 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -271,6 +271,15 @@ def test_a_failed_pull_errors_the_cell_before_any_container(tmp_path): assert not [a for a in cli.args if a[0] == "run"] +@pytest.mark.parametrize("value", ["5", "true", '["python:3.12-slim"]']) +def test_a_docker_image_that_is_not_a_string_is_refused_at_load(tmp_path, value): + """docs/coding-harnesses.md: refused as Harbor's EnvironmentConfig (docker_image: str | None) refuses it.""" + path = _task(tmp_path, image=None, dockerfile=True) + (path / "task.toml").write_text(f"[environment]\ndocker_image = {value}\n", encoding="utf-8") + with pytest.raises(ValueError, match="docker_image must be a string"): + load_task(path) + + @pytest.mark.parametrize("image", ["--privileged", "python:3.12 -slim"]) def test_a_docker_image_that_is_not_a_reference_is_refused(tmp_path, image): """distribution/reference: a reference starts with a letter or digit and has no space; From 16a11b5a691271a881150ed912f3a1c04f1bfccc Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:42:38 -0700 Subject: [PATCH 7/9] test(harnesses): pin the built-task workdir case under its own name The test named "without a declared workdir" also covered a built task that does declare one. Split it: a prebuilt-image task without a workdir gets no -w, and a built task keeps its image's WORKDIR despite a declared one, which differs from Harbor (it overrides WORKDIR for every task) and is left as on main on purpose. The workdir_args docstring and the doc say so. Signed-off-by: Vladislav Perevezentsev --- dokimi/docs/coding-harnesses.md | 3 ++- dokimi/dokimi/harnesses/task_image.py | 3 ++- dokimi/tests/test_harbor_docker_image.py | 16 ++++++++++++---- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index 075092abf..fce4889f3 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -265,7 +265,8 @@ formats do not read the key. not stop it. To run offline, have the image on the daemon first. - The container runs in `environment.workdir` when the task declares one (`docker run -w`), as Harbor runs every exec there, so the oracle and the verifier find relative paths; without one, - in the image's `WORKDIR`. + in the image's `WORKDIR`. A task built from its Dockerfile keeps the image's `WORKDIR` even + when it declares one (Harbor overrides it there too). - After the container starts, the files of a non-empty `environment/` without a `Dockerfile` or `docker-compose.yaml` are copied into `environment.workdir`, else the image's `WORKDIR`, else `/` (Harbor's diff --git a/dokimi/dokimi/harnesses/task_image.py b/dokimi/dokimi/harnesses/task_image.py index 8d529822b..1ea46733f 100644 --- a/dokimi/dokimi/harnesses/task_image.py +++ b/dokimi/dokimi/harnesses/task_image.py @@ -58,7 +58,8 @@ def ready_image(task: TaskSpec, log, docker: Docker, build: Callable[[str], None def workdir_args(task: TaskSpec) -> list[str]: """``-w`` for a prebuilt-image task's ``environment.workdir``: Harbor runs every exec there - (``DockerEnvironment.exec``: ``cwd or task_env_config.workdir``); a built image has its ``WORKDIR``.""" + (``DockerEnvironment.exec``: ``cwd or task_env_config.workdir``). Harbor does so for a built task + too; a built task keeps its image's ``WORKDIR`` here on purpose, as on main.""" workdir = task.environment.get("workdir") return ["-w", workdir] if task.docker_image is not None and isinstance(workdir, str) and workdir else [] diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index 14002900d..a8c41e44b 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -138,10 +138,18 @@ def test_the_task_container_works_in_the_declared_workdir(tmp_path): def test_without_a_declared_workdir_the_container_keeps_the_image_workdir(tmp_path): - for path in (_task(tmp_path / "a", workdir=None), _task(tmp_path / "b", image=None, dockerfile=True)): - cli = _daemon() - create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") - assert "-w" not in _run_argv(cli) + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(_task(tmp_path, workdir=None)), output_dir=tmp_path / "out") + assert "-w" not in _run_argv(cli) + + +def test_a_built_task_keeps_its_image_workdir_despite_a_declared_one(tmp_path): + """Left as on main on purpose (unlike Harbor, which overrides WORKDIR with it): see the PR's Not in this PR.""" + path = _task(tmp_path, image=None, dockerfile=True) + assert load_task(path).environment["workdir"] == "/app" + cli = _daemon() + create_harness("docker-oracle", runner=cli).run(load_task(path), output_dir=tmp_path / "out") + assert "-w" not in _run_argv(cli) def test_without_a_declared_workdir_the_files_go_to_the_image_workdir_else_root(tmp_path): From a3049cf55bda98d5d89f52c84eba7a4822f92f70 Mon Sep 17 00:00:00 2001 From: Vladislav Perevezentsev Date: Tue, 17 Nov 2026 11:42:49 -0700 Subject: [PATCH 8/9] docs(harnesses): fragment and doc say what -w and the load check change Since the container of a prebuilt-image task starts with `docker run -w `, docker creates /app itself, so the environment/.keep copy no longer is what creates it: that claim goes from the fragment and coding-harnesses.md. The fragment now says the container runs in environment.workdir and that a non-string docker_image fails the task's load. Signed-off-by: Vladislav Perevezentsev --- ...0261117_160000_vlad-perevezentsev_harbor-docker-image.md | 6 ++++-- dokimi/docs/coding-harnesses.md | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md index 63a69dbe4..186b7d412 100644 --- a/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md +++ b/dokimi/changelog.d/20261117_160000_vlad-perevezentsev_harbor-docker-image.md @@ -10,8 +10,10 @@ SPDX-License-Identifier: Apache-2.0 every answer-quality task and `sycl-device-discovery`) failed its cell with `docker build`'s "open Dockerfile: no such file". As in Harbor 0.24, the image is used whenever it is set (also beside a Dockerfile), pulled only when the daemon lacks it, within `build_timeout_sec`, and the - loose files of `environment/` are copied into the workdir after the start (what creates `/app` - in `python:3.12-slim`). `--network build=none` does not stop the pull, as it does not stop a + loose files of `environment/` are copied into the workdir after the start. The container runs + in the task's `environment.workdir` (`docker run -w`), so `solve.sh` and `test.sh` find relative + paths. A `docker_image` that is not a string (`docker_image = 5`) now fails the task's load, as + in Harbor. `--network build=none` does not stop the pull, as it does not stop a build's `FROM`. `metrics.image` records the image a cell ran on: its id and repo digests for a prebuilt one. A task without `docker_image` builds as before. A task that sets `docker_image` beside `environment/Dockerfile` (as the old `task.toml` template in diff --git a/dokimi/docs/coding-harnesses.md b/dokimi/docs/coding-harnesses.md index fce4889f3..102eb2744 100644 --- a/dokimi/docs/coding-harnesses.md +++ b/dokimi/docs/coding-harnesses.md @@ -270,8 +270,7 @@ formats do not read the key. - After the container starts, the files of a non-empty `environment/` without a `Dockerfile` or `docker-compose.yaml` are copied into `environment.workdir`, else the image's `WORKDIR`, else `/` (Harbor's - `should_upload_environment_dir`). The uxl answer-quality tasks rely on it: their - `environment/.keep` is what creates `/app` in `python:3.12-slim`. A top-level + `should_upload_environment_dir`). A top-level `environment/skills` is left out, as it is left out of a build context. - `metrics.image` says which image ran: `{"source": "docker_image", "ref", "pulled", "id", "repo_digests"}` for a prebuilt one (the id and digests of what From a33c20079e3576735fb5efb7ea816aa7989c555a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 19:36:40 +0000 Subject: [PATCH 9/9] fix(harnesses): start the container from the inspected image id Signed-off-by: Anton Volkov --- dokimi/dokimi/harnesses/task_image.py | 3 ++- dokimi/tests/test_harbor_docker_image.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dokimi/dokimi/harnesses/task_image.py b/dokimi/dokimi/harnesses/task_image.py index 1ea46733f..3d17fcfb0 100644 --- a/dokimi/dokimi/harnesses/task_image.py +++ b/dokimi/dokimi/harnesses/task_image.py @@ -53,7 +53,8 @@ def ready_image(task: TaskSpec, log, docker: Docker, build: Callable[[str], None if pulled: docker(["pull", "-q", image], log, timeout=_task_build_timeout(task)) found = docker(inspect, log) - return image, {"source": "docker_image", "ref": image, "pulled": pulled, **json.loads(found.stdout or "")} + record = json.loads(found.stdout or "") + return record["id"], {"source": "docker_image", "ref": image, "pulled": pulled, **record} def workdir_args(task: TaskSpec) -> list[str]: diff --git a/dokimi/tests/test_harbor_docker_image.py b/dokimi/tests/test_harbor_docker_image.py index a8c41e44b..a6a858900 100644 --- a/dokimi/tests/test_harbor_docker_image.py +++ b/dokimi/tests/test_harbor_docker_image.py @@ -97,7 +97,7 @@ def test_a_docker_image_task_pulls_and_runs_that_image_without_a_build(tmp_path, assert not [a for a in cli.args if a[0] == "build"] assert ["pull", "-q", IMAGE] in cli.args run = _run_argv(cli) - assert run[run.index("sleep") - 1] == IMAGE + assert run[run.index("sleep") - 1] == RECORD["id"] pull = next(c for c in cli.calls if c.argv[1] == "pull") assert pull.timeout == _task_build_timeout(task) == 77 assert result.metrics["image"] == {"source": "docker_image", "ref": IMAGE, "pulled": True, **RECORD} @@ -134,7 +134,7 @@ def test_the_task_container_works_in_the_declared_workdir(tmp_path): cli = _daemon() create_harness("docker-oracle", runner=cli).run(load_task(_task(tmp_path)), output_dir=tmp_path / "out") run = _run_argv(cli) - assert run[run.index("-w") + 1] == "/app" and run.index("-w") < run.index(IMAGE) + assert run[run.index("-w") + 1] == "/app" and run.index("-w") < run.index(RECORD["id"]) def test_without_a_declared_workdir_the_container_keeps_the_image_workdir(tmp_path):