diff --git a/docs/plans/rfc008-validation-completion.md b/docs/plans/rfc008-validation-completion.md index 050aba5a2..db0fb36a8 100644 --- a/docs/plans/rfc008-validation-completion.md +++ b/docs/plans/rfc008-validation-completion.md @@ -123,15 +123,18 @@ Done. Harbor `task.toml` packages validate through the `harbor-semantic-v1` profile and an operator-supplied `validation/openenv.json` contract. Fresh Docker trials run a do-nothing agent, the oracle, an oracle replay and declared canaries, all pinned to one immutable image. The binding requires Python 3.12 or newer and -`harbor==0.25.0` alongside `openenv[harbor]`. It refuses task-declared `docker_image` -values, so the NOP trial builds from the task's Dockerfile. The validator keeps +`harbor==0.25.0` alongside `openenv[harbor]`. The NOP trial builds from the task's +Dockerfile, or pulls a task-declared `docker_image` that is pinned by digest +(`name@sha256:`). It refuses a `docker_image` without a digest, because a +tag can move between validation and training. The validator keeps `openenv-validation:` only when the run has no FAIL or ERROR and was not interrupted. It records the tag in `runtime.startup.measured` as `retained_image_tag`. Otherwise it removes the tag. -**Follow-up.** Tasks that ship only a prebuilt image cannot run under -`harbor-semantic-v1`. Some Terminal Bench tasks are affected. Supporting them needs -a reviewable Dockerfile path or digest-pinned image support. +**Follow-up.** Tasks that declare a prebuilt image by tag still cannot run under +`harbor-semantic-v1`. Some Terminal Bench tasks are affected. An operator can +resolve the tag to its digest in a validation overlay and record the original +bytes in `original_source_digest`. The validator does not resolve tags itself. **Remaining.** Decide which runtime ids apply to one-shot tasks. Containment, network policy, resource bounds and oracle containment could apply. Observation diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index ca8084495..753756074 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -115,10 +115,15 @@ the raw trial evidence in the report bundle. Other checks are named SKIPs, so a passing task still reports WARN. This validates task packages, not the outer `harbor_env` server. -Supported tasks use Linux, a single step, one Dockerfile, a shared verifier and -public networking. Unsupported shapes include multi-step or non-Linux tasks, -task-declared `environment.docker_image`, missing Dockerfiles, task-authored Compose -files and separate verifier environments. GPU/TPU or external MCP requirements, +Supported tasks use Linux, a single step, a shared verifier and public networking. +The image comes from one Dockerfile or from a digest-pinned +`environment.docker_image`, such as `org/image@sha256:<64 hex digits>`. Harbor +prefers a declared image over a Dockerfile, so the do-nothing trial pulls the pin +and `runtime.startup` records it as `source_image`. The validator leaves the pulled +image in the local Docker cache. Unsupported shapes include multi-step or non-Linux +tasks, an `environment.docker_image` without a digest, tasks with neither a +Dockerfile nor a pinned image, task-authored Compose files and separate verifier +environments. GPU/TPU or external MCP requirements, credentials, non-public or differing phase networks, task environment variables, healthchecks, skills, verifier collection and artifact extensions are also unsupported. A supplied provider, including `--space`, is unsupported. Each case @@ -133,9 +138,9 @@ episode budget. There is no shared outer trial deadline. Harbor bounds setup, ag and verifier separately with the declared `resources.episode_timeout_s`. A timeout in one of these phases is a package FAIL that names the phase. -The do-nothing trial builds from the task's Dockerfile. The validator keeps -`openenv-validation:` only when the run has no FAIL or ERROR and was not -interrupted. It records the tag in `runtime.startup.measured` as +The do-nothing trial builds from the task's Dockerfile, or pulls the digest-pinned +image when the task declares one. The validator keeps `openenv-validation:` +only when the run has no FAIL or ERROR and was not interrupted. It records the tag in `runtime.startup.measured` as `retained_image_tag`. Otherwise it removes the tag. The run-owned Compose build tag is removed on every run. Tag removal verifies image identity and uses `--no-prune`. A removal failure marks cleanup incomplete and gives a startup ERROR. During an diff --git a/rfcs/008-environment-auto-validation.md b/rfcs/008-environment-auto-validation.md index 9d1183a1a..b209fd166 100644 --- a/rfcs/008-environment-auto-validation.md +++ b/rfcs/008-environment-auto-validation.md @@ -748,10 +748,12 @@ The profile requires Python 3.12 or newer and `harbor==0.25.0` alongside `openenv[harbor]`. A missing or mismatched Harbor installation is a validator environment error. It exits 3 and names `harbor==0.25.0`. -The profile supports single-step Linux tasks with one Dockerfile, one main Docker -service, a shared verifier and public networking. Unsupported shapes include -multi-step or non-Linux tasks, task-declared `environment.docker_image`, missing -Dockerfiles, task-authored Compose files and separate verifier environments. +The profile supports single-step Linux tasks with one main Docker service, a shared +verifier and public networking. The image comes from one Dockerfile or from a +digest-pinned `environment.docker_image` (`name@sha256:`), which the report +records as `source_image`. Unsupported shapes include multi-step or non-Linux tasks, +an `environment.docker_image` without a digest, tasks with neither a Dockerfile nor +a pinned image, task-authored Compose files and separate verifier environments. GPU/TPU or external MCP requirements, credentials, non-public or differing phase networks, task environment variables, healthchecks, skills, verifier collection and artifact extensions are also unsupported. A supplied provider, including `--space`, @@ -762,9 +764,10 @@ at the semantic level, including with `--skip-build` or `--space`. Execution runs fresh Harbor Trials in a fixed order: a do-nothing (NOP) trial, the oracle (`solution/solve.sh`), an oracle replay, and one trial per declared canary. -The NOP trial builds the image. The validator inspects the actual agent container -at `AGENT_START`, retains its immutable image under a run-owned tag, and pins every -later trial to that image and its effective container configuration. +The NOP trial builds the image, or pulls the task's digest-pinned one. The validator +inspects the actual agent container at `AGENT_START`, retains its immutable image +under a run-owned tag, and pins every later trial to that image and its effective +container configuration. The initial image build and environment start share a separate 600-second validator budget. Exceeding it gives `runtime.startup` ERROR. Later environment starts use the diff --git a/src/openenv/validation/harbor.py b/src/openenv/validation/harbor.py index cb1afdae9..e0210ce52 100644 --- a/src/openenv/validation/harbor.py +++ b/src/openenv/validation/harbor.py @@ -1,7 +1,8 @@ """Native Harbor execution: fresh Docker trials graded through RFC 008 check ids. -No served-protocol evidence is synthesized. A do-nothing trial builds the image, and -the oracle, its replay and any canaries run pinned to that image. +No served-protocol evidence is synthesized. A do-nothing trial builds the image, or +pulls the task's digest-pinned one, and the oracle, its replay and any canaries run +pinned to that image. """ import asyncio @@ -41,6 +42,8 @@ from .types import CheckStatus, Lane, Level _IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +# A tag can move between validation and training; a digest cannot. +_PINNED_IMAGE = re.compile(r"[^@\s]+@sha256:[0-9a-f]{64}\Z") _RESOURCE_ID = re.compile(r"[a-zA-Z0-9][a-zA-Z0-9_.-]*\Z") _PROJECT = re.compile( r"openenv-validation-[a-f0-9]+-(nop|oracle|oracle-replay|canary-[1-8])__env" @@ -280,19 +283,22 @@ def _unsupported(task, contract): return "multi-step Harbor tasks" if env.os.value != "linux": return "non-Linux Harbor tasks" - if env.docker_image is not None: - return "task-declared environment.docker_image instead of a Dockerfile build" + if env.docker_image is not None and not _PINNED_IMAGE.fullmatch(env.docker_image): + return "task-declared environment.docker_image without a sha256 digest pin" if config.verifier.environment is not None or ( config.verifier.environment_mode is not None and config.verifier.environment_mode.value != "shared" ): return "separate Harbor verifier environments" - if not (task.task_dir / "environment/Dockerfile").is_file() or any( + if ( + env.docker_image is None + and not (task.task_dir / "environment/Dockerfile").is_file() + ) or any( path.name.lower() in {"docker-compose.yaml", "docker-compose.yml", "compose.yaml", "compose.yml"} for path in (task.task_dir / "environment").rglob("*") ): - return "tasks without one Dockerfile or with task-authored Compose" + return "tasks without one Dockerfile or pinned image, or with task-authored Compose" if contract.resources.gpus or env.gpus or env.tpu or env.mcp_servers: return "GPU/TPU or external MCP requirements" if contract.requires_credentials: @@ -343,7 +349,10 @@ async def _run_compose( *argv, *command, env=environment._compose_env_vars(include_os_env=True), - cwd=str(environment.environment_dir.resolve()), + # A task with a prebuilt image may ship no environment directory. + cwd=str(environment.environment_dir.resolve()) + if environment.environment_dir.is_dir() + else None, stdin=asyncio.subprocess.PIPE if stdin_data is not None else asyncio.subprocess.DEVNULL, @@ -851,7 +860,8 @@ def validate_harbor( inspector = DockerInspector() try: attempted = False - problem = _unsupported(read_harbor_task(subject.root), contract) + task = read_harbor_task(subject.root) + problem = _unsupported(task, contract) if skip_build: problem = "image build skipped by --skip-build" if provider is not None: @@ -978,6 +988,10 @@ def validate_harbor( "task_effective_digest": digest_before, }, ) + source_image = task.config.environment.docker_image + if source_image is not None: + record["source_image"] = source_image + startup.measured["source_image"] = source_image if malformed: reward_result = _result( "runtime.reward_well_formed", diff --git a/tests/test_validation/test_harbor_binding_supervision.py b/tests/test_validation/test_harbor_binding_supervision.py index f27255d7b..0904e30cf 100644 --- a/tests/test_validation/test_harbor_binding_supervision.py +++ b/tests/test_validation/test_harbor_binding_supervision.py @@ -1,4 +1,4 @@ -"""Compose cancellation tests use fake processes and never contact Docker.""" +"""Compose wrapper tests use fake processes and never contact Docker.""" import asyncio import signal @@ -98,3 +98,40 @@ def killpg(pid, sig): "daemon_completion_verified": False, } ] + + +@pytest.mark.asyncio +async def test_compose_runs_without_an_environment_directory(monkeypatch, tmp_path): + options = {} + + async def spawn(*argv, **kwargs): + options.update(kwargs, argv=argv) + return object() + + class Environment: + session_id = "openenv-validation-" + "d" * 32 + "-nop__env" + environment_dir = tmp_path / "environment" + _docker_compose_paths = [] + + @classmethod + def runtime(cls): + return SimpleNamespace( + compose=["docker", "compose"], supports_compose_project_directory=True + ) + + def _compose_env_vars(self, *, include_os_env): + return {} + + async def _collect_buffered_output(self, process, *, timeout_sec, stdin_data): + return SimpleNamespace(return_code=0) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn) + await harbor._run_compose(Environment(), ["up"], cancelled=[]) + # A missing working directory would stop the Compose client from starting. + assert options["cwd"] is None + # As in Harbor, the project directory never falls back to the task root, + # where Compose would read a task-authored .env file. + argv = options["argv"] + assert argv[argv.index("--project-directory") + 1] == str( + Environment.environment_dir.resolve() + ) diff --git a/tests/test_validation/test_harbor_validation.py b/tests/test_validation/test_harbor_validation.py index 9f2316ada..a297c9c8b 100644 --- a/tests/test_validation/test_harbor_validation.py +++ b/tests/test_validation/test_harbor_validation.py @@ -153,9 +153,9 @@ def __init__(self, config): ) self.hooks = {} self._openenv_cancelled_compose = [] - self._environment_build_timeout_sec = read_harbor_task( - config.task.path - ).config.environment.build_timeout_sec + environment = read_harbor_task(config.task.path).config.environment + self._environment_build_timeout_sec = environment.build_timeout_sec + self.docker_image = environment.docker_image state.excluded_copied.append( (config.task.path / "environment/.worktrees").exists() ) @@ -1175,16 +1175,37 @@ def test_separate_verifier_is_named_unsupported(task, boundary): assert not boundary.trials -def test_task_declared_image_is_refused_before_trials(task, boundary): +def test_unpinned_task_image_is_refused_before_trials(task, boundary): with (task / "task.toml").open("a") as stream: stream.write('[environment]\ndocker_image="registry.example/image:latest"\n') report = run_validation(task) startup = result(report, "runtime.startup") assert startup.status is CheckStatus.SKIP - assert "docker_image" in startup.evidence[0] + assert "digest pin" in startup.evidence[0] assert not boundary.trials +@pytest.mark.parametrize("layout", ["dockerfile", "no dockerfile", "no environment"]) +def test_digest_pinned_task_image_runs(task, boundary, tmp_path, layout): + pinned = "registry.example/image@sha256:" + "d" * 64 + if layout != "dockerfile": + (task / "environment/Dockerfile").unlink() + if layout == "no environment": + (task / "environment").rmdir() + with (task / "task.toml").open("a") as stream: + stream.write(f'[environment]\ndocker_image="{pinned}"\n') + artifacts = tmp_path / "pinned" + report = run_validation(task, artifacts_dir=artifacts) + startup = result(report, "runtime.startup") + assert startup.status is CheckStatus.PASS + assert startup.measured["source_image"] == pinned + native = json.loads((artifacts / "harbor-evidence.json").read_text()) + assert native["source_image"] == pinned + # The do-nothing trial pulls the pin; later trials run the image it retained. + images = [trial.docker_image for trial in boundary.trials] + assert images == [pinned, *[startup.measured["retained_image_tag"]] * 2] + + def test_explicit_contract_and_valid_toml_required(task, boundary): (task / "validation/openenv.json").unlink() with pytest.raises(ManifestError):