From 442aa31b5656e64c77fd2b2e816bcc1b487eaa6a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 9 Nov 2026 10:57:31 -0800 Subject: [PATCH] fix(episode): mint an episode's run id in its task's namespace An episode's run id was a bare uuid4 hex, while `agent-env run` and the local runner mint theirs with new_instance_id(task_id), which keeps an @local task's namespace. A step that names what it writes after the run, such as run_container_unit_tests_verifier's stdout and stderr artifacts, got a bare id in an episode of an @local task, and the local-run write fence refused it. A container grader writing its reward file could grade the task under `agent-env run` but not as an OpenEnv episode. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/spec.py | 8 +++-- tst/unit/task/local_run_write_fence_test.py | 37 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 4f795b62f..2eafc412b 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -1,13 +1,13 @@ from __future__ import annotations import math -import uuid from collections.abc import Mapping, Sequence from typing import Any from agent_env.task.episode.rules import DEPLOYS, problems, works_in_sandbox from agent_env.task.episode.run import Episode, EpisodeError from agent_env.task.episode.steps import _EpisodeAgentStep +from agent_env.task.store import new_instance_id from agent_env.task.task import Task, substitute_seed_in_steps from agent_env.task_step.context import TaskStepContext, sandbox_overrides from agent_env.task_step.task_step import TaskStep @@ -59,8 +59,10 @@ def prompts(self) -> tuple[tuple[str, str | None], ...]: async def start(self, *, sandbox: str | None = None) -> Episode: """Start one run of the task, a new episode each call. It returns once the run's record exists, before anything is deployed; ``next_turn()`` is where the deploys and setup are waited for. ``sandbox`` is the - provider its envs, agents, sandboxes and judge deploy on, as ``agent-env run --sandbox`` sets it.""" - episode_id = uuid.uuid4().hex + provider its envs, agents, sandboxes and judge deploy on, as ``agent-env run --sandbox`` sets it. Its id is + its run's, minted as ``agent-env run`` mints one, so what the run names after it stays in its task's + namespace.""" + episode_id = new_instance_id(self._copy.id) metadata: dict[str, Any] = {"episode": {"id": episode_id, "trainer_agents": list(self.trainer_agents)}} if self._seed: metadata["seed"] = dict(self._seed) diff --git a/tst/unit/task/local_run_write_fence_test.py b/tst/unit/task/local_run_write_fence_test.py index 108709d94..6c62f3ade 100644 --- a/tst/unit/task/local_run_write_fence_test.py +++ b/tst/unit/task/local_run_write_fence_test.py @@ -8,8 +8,10 @@ from types import SimpleNamespace import pytest +from pytest_socket import enable_socket, socket_allow_hosts import agent_env.providers.sandbox_providers.sandbox_provider as sandbox_provider +import agent_env.task.episode.run as episode_run from agent_env.artifact import EnvironmentUniverseArtifact, FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.artifact.store import get_artifact_store @@ -24,7 +26,9 @@ from agent_env.store.object_store import LocalFilesystemObjectStore from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox from agent_env.store.routing import LocalRunWriteError, run_scope +from agent_env.task import EpisodeSpec from agent_env.task.task import Task +from agent_env.task.teardown import TeardownReport from agent_env.task_step.context import DeployedSandbox, TaskStepContext from agent_env.task_step.snapshot_utils.snapshot_series import SnapshotConfig, SnapshotSeries from agent_env.task_step.task_step import TaskStep @@ -349,6 +353,39 @@ async def test_an_local_runs_verifier_outputs_are_named_after_the_run(sentinels, assert artifact.load() == text +@pytest.fixture +def loopback(): + """Loopback connections and no others, for the proxy agent an episode serves in this process.""" + enable_socket() + socket_allow_hosts(["127.0.0.1"], allow_unix_socket=True) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("loopback") +async def test_an_local_episodes_verifier_outputs_are_named_after_its_run(sentinels, vm, monkeypatch): + """An episode is a run of its task, as ``agent-env run``'s is: what its steps name after the run stays local.""" + monkeypatch.setattr(episode_run, "teardown_run", lambda context, **_: asyncio.sleep(0, result=TeardownReport())) + + def box(context): + context.deployed_sandboxes.append(vm) + context.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "box"}] + + played = Task.from_dict({"id": LOCAL_TASK, "version": 1, "steps": [ + {"id": "alice", "type": "deploy_agent", "agent_name": "alice", "env_ids": []}, + {"id": "turn", "type": "prompt_agent", "agent_name": "alice", "prompt_id": "turn", "prompt": "Fix it.", + "timeout_seconds": 5, "poll_interval_seconds": 0.05}, + {"id": "verify", "type": "run_container_unit_tests_verifier", "sandbox_name": "box", "container_name": "c", + "command": "true"}, + ]}) + episode = await EpisodeSpec.from_task(Task(id=LOCAL_TASK, version=1, steps=[_Step("box", box), *played.steps])).start() + await (await episode.next_turn()).end("Fixed.") + + assert await episode.result() == {"verify": 1.0} + entry = Task.get_instance(episode.id).context["metadata"]["verifications"]["verify"] + assert FileArtifact.get(entry["stdout_artifact"]["id"]).id.startswith(derive_id(episode.id, "verifier-stdout-verify-")) + assert episode.id.startswith(f"{LOCAL_TASK}-") + + @pytest.mark.asyncio async def test_an_local_run_refuses_a_verifier_step_id_its_outputs_cant_take_before_the_tests_run(sentinels, vm, monkeypatch): monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", lambda: pytest.fail("the sandbox was read"))