Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions src/agent_env/task/episode/spec.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
from __future__ import annotations

import math
import uuid
from collections.abc import Mapping, Sequence
from typing import Any

from agent_env.task.episode.rules import DEPLOYS, problems, works_in_sandbox
from agent_env.task.episode.run import Episode, EpisodeError
from agent_env.task.episode.steps import _EpisodeAgentStep
from agent_env.task.store import new_instance_id
from agent_env.task.task import Task, substitute_seed_in_steps
from agent_env.task_step.context import TaskStepContext, sandbox_overrides
from agent_env.task_step.task_step import TaskStep
Expand Down Expand Up @@ -59,8 +59,10 @@ def prompts(self) -> tuple[tuple[str, str | None], ...]:
async def start(self, *, sandbox: str | None = None) -> Episode:
"""Start one run of the task, a new episode each call. It returns once the run's record exists, before
anything is deployed; ``next_turn()`` is where the deploys and setup are waited for. ``sandbox`` is the
provider its envs, agents, sandboxes and judge deploy on, as ``agent-env run --sandbox`` sets it."""
episode_id = uuid.uuid4().hex
provider its envs, agents, sandboxes and judge deploy on, as ``agent-env run --sandbox`` sets it. Its id is
its run's, minted as ``agent-env run`` mints one, so what the run names after it stays in its task's
namespace."""
episode_id = new_instance_id(self._copy.id)
metadata: dict[str, Any] = {"episode": {"id": episode_id, "trainer_agents": list(self.trainer_agents)}}
if self._seed:
metadata["seed"] = dict(self._seed)
Expand Down
37 changes: 37 additions & 0 deletions tst/unit/task/local_run_write_fence_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@
from types import SimpleNamespace

import pytest
from pytest_socket import enable_socket, socket_allow_hosts

import agent_env.providers.sandbox_providers.sandbox_provider as sandbox_provider
import agent_env.task.episode.run as episode_run
from agent_env.artifact import EnvironmentUniverseArtifact, FileArtifact
from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse
from agent_env.artifact.store import get_artifact_store
Expand All @@ -24,7 +26,9 @@
from agent_env.store.object_store import LocalFilesystemObjectStore
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox
from agent_env.store.routing import LocalRunWriteError, run_scope
from agent_env.task import EpisodeSpec
from agent_env.task.task import Task
from agent_env.task.teardown import TeardownReport
from agent_env.task_step.context import DeployedSandbox, TaskStepContext
from agent_env.task_step.snapshot_utils.snapshot_series import SnapshotConfig, SnapshotSeries
from agent_env.task_step.task_step import TaskStep
Expand Down Expand Up @@ -349,6 +353,39 @@ async def test_an_local_runs_verifier_outputs_are_named_after_the_run(sentinels,
assert artifact.load() == text


@pytest.fixture
def loopback():
"""Loopback connections and no others, for the proxy agent an episode serves in this process."""
enable_socket()
socket_allow_hosts(["127.0.0.1"], allow_unix_socket=True)


@pytest.mark.asyncio
@pytest.mark.usefixtures("loopback")
async def test_an_local_episodes_verifier_outputs_are_named_after_its_run(sentinels, vm, monkeypatch):
"""An episode is a run of its task, as ``agent-env run``'s is: what its steps name after the run stays local."""
monkeypatch.setattr(episode_run, "teardown_run", lambda context, **_: asyncio.sleep(0, result=TeardownReport()))

def box(context):
context.deployed_sandboxes.append(vm)
context.metadata["deployed_docker_containers"] = [{"container_name": "c", "sandbox_name": "box"}]

played = Task.from_dict({"id": LOCAL_TASK, "version": 1, "steps": [
{"id": "alice", "type": "deploy_agent", "agent_name": "alice", "env_ids": []},
{"id": "turn", "type": "prompt_agent", "agent_name": "alice", "prompt_id": "turn", "prompt": "Fix it.",
"timeout_seconds": 5, "poll_interval_seconds": 0.05},
{"id": "verify", "type": "run_container_unit_tests_verifier", "sandbox_name": "box", "container_name": "c",
"command": "true"},
]})
episode = await EpisodeSpec.from_task(Task(id=LOCAL_TASK, version=1, steps=[_Step("box", box), *played.steps])).start()
await (await episode.next_turn()).end("Fixed.")

assert await episode.result() == {"verify": 1.0}
entry = Task.get_instance(episode.id).context["metadata"]["verifications"]["verify"]
assert FileArtifact.get(entry["stdout_artifact"]["id"]).id.startswith(derive_id(episode.id, "verifier-stdout-verify-"))
assert episode.id.startswith(f"{LOCAL_TASK}-")


@pytest.mark.asyncio
async def test_an_local_run_refuses_a_verifier_step_id_its_outputs_cant_take_before_the_tests_run(sentinels, vm, monkeypatch):
monkeypatch.setattr(sandbox_provider, "get_sandbox_provider", lambda: pytest.fail("the sandbox was read"))
Expand Down
Loading