Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/agent_env/bundle/ledger.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,14 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check:
"""Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id),
the version each earlier write it needs left or, in a dry run, would leave; an env or agent is hashed with
them, so one written anew rewrites it too."""
stored = self._stored_version(write)
stored_doc = self._stored(write)
stored = stored_doc["version"] if stored_doc else None
digest = self.digest(write, needs)
if digest is None:
untracked = ("its inputs aren't tracked yet, so it is written every run",)
return Check(write, None, None, untracked, stored, needs)
orphan = self._orphan(write, stored, digest)
stale = _predates_entity_ids(write, stored_doc)
orphan = None if stale else self._orphan(write, stored, digest)
if orphan is not None:
return Check(write, digest, orphan, ("written by an interrupted run that didn't record it",), stored, needs,
adopted=True)
Expand All @@ -122,6 +124,8 @@ def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check:
reasons.append("new" if row is None else f"its v{recorded} is not in the store")
elif stored != recorded:
reasons.append(f"the store's latest, v{stored}, wasn't recorded by this bundle")
elif stale:
reasons.append("its stored version doesn't record the ids the bundle gave what it names")
if row is not None:
reasons.extend(_changes(row, digest))
return Check(write, digest, None if reasons else recorded, tuple(reasons), stored, needs)
Expand Down Expand Up @@ -214,9 +218,9 @@ def _bundle(self, write: Write) -> str:
def _latest(self, store: str, id: str) -> dict | None:
return self._find(id, self.collection, Filter.of(store=store, id=id, status="done"))

def _stored_version(self, write: Write) -> int | None:
doc = self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id))
return doc["version"] if doc else None
def _stored(self, write: Write) -> dict | None:
"""The store's latest version of ``write``'s entity."""
return self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id))

def _find(self, entity_id: str, collection: str, filter: Filter) -> dict | None:
"""The highest version matching ``filter`` in ``entity_id``'s store, read without creating a local store."""
Expand Down Expand Up @@ -350,3 +354,9 @@ def _file_sha256(path: Path) -> str:

def _now() -> str:
return datetime.now(timezone.utc).isoformat()


def _predates_entity_ids(write: Write, stored: dict | None) -> bool:
"""Whether the store's latest version of task ``write``, written before tasks recorded them, lacks the entity_ids
its bundle gives it now, so reusing it would leave its runs naming those entities as written."""
return write.kind is BundleKind.TASK and stored is not None and (stored.get("entity_ids") or {}) != write.source.entity_ids
6 changes: 4 additions & 2 deletions src/agent_env/bundle/materialize.py
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None:
raise BundleError(problems)
for write in tasks:
through_ledger(write, lambda: Task.put(id=write.id, steps=built[write.id].steps,
requires=write.requires).version)
requires=write.requires,
entity_ids=built[write.id].entity_ids).version)
Comment thread
earakely-scale marked this conversation as resolved.
Resolved
for write in evals:
through_ledger(write, lambda: _write_eval(plan, write, written))
return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left))
Expand Down Expand Up @@ -418,7 +419,8 @@ def _build(write: Write, plan: Plan, written: Mapping[tuple[str, str], Materiali
"""The task ``write`` puts: as resolved, or given what was ``written``, with each reference that takes a version
naming the one it reads."""
steps = write.source.config if written is None else pinned_steps(plan, write, written)
return Task(id=write.id, version=None, steps=[build_step(step) for step in steps])
return Task(id=write.id, version=None, steps=[build_step(step) for step in steps],
entity_ids=write.source.entity_ids)


def pinned_steps(plan: Plan, write: Write, written: Mapping[tuple[str, str], Materialized]) -> list[dict]:
Expand Down
23 changes: 23 additions & 0 deletions src/agent_env/bundle/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ class ResolvedEntry:
references: tuple[Reference, ...]
outputs: tuple[Output, ...]

@property
def entity_ids(self) -> dict[str, str]:
"""For a task, each id a field of its steps names as written that ``config`` names by another id, to that id
(``Task.entity_ids``); a reference the plan adds without rewriting a field, as ``--agent``'s, adds nothing."""
if self.entry.kind is not BundleKind.TASK:
return {}
return _rewritten_ids(self.entry.config, self.config)


@dataclass(frozen=True)
class ResolvedBundle:
Expand Down Expand Up @@ -561,3 +569,18 @@ def _strings(value: Any, path: str = "") -> Iterator[tuple[str, str]]:
elif isinstance(value, list):
for i, item in enumerate(value):
yield from _strings(item, f"{path}[{i}]")


def _rewritten_ids(written: list[dict], resolved: list[dict]) -> dict[str, str]:
"""Each id a reference field of ``written``'s steps holds that the same field of ``resolved``'s holds another id
in place of, to that id."""
registry = get_task_step_registry()
renamed = {}
for before, after in zip(written, resolved):
cls = registry.get(before.get("type"))
if cls is None or not cls.entity_refs:
continue
now = {site.path: site.value for site in ref_sites(cls.entity_refs, after)}
renamed.update((site.value, now[site.path]) for site in ref_sites(cls.entity_refs, before)
if site.path in now and now[site.path] != site.value)
return renamed
13 changes: 11 additions & 2 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Callable, ClassVar, Optional

from agentenv_protocol import client as protocol_v1
Expand All @@ -25,7 +25,7 @@
from agent_env.store.routing import run_scope
from agent_env.task_step.preflight import pinned
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import TaskStepContext, regraft_redacted_keys
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, regraft_redacted_keys
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
from agent_env.task_step.task_step import RetryConfig, TaskStep

Expand Down Expand Up @@ -349,6 +349,8 @@ class Task:
encoded in each step's depends_on. Independent steps run concurrently.
``requires`` names the plugin packages its steps and what they reference use, as pip requirements, which a
registration records (``agent-env bundle register``); None when nothing recorded it.
``entity_ids`` maps each id a bundle rewrote in the task's steps, as the task names it, to the id the bundle gave
it; ``run`` puts it in ``context.metadata`` for ``run_entity_id``. None for a task stored as written.
"""

type: ClassVar[str] = "task"
Expand All @@ -359,11 +361,13 @@ def __init__(
version: Optional[int],
steps: list[TaskStep] | None = None,
requires: Sequence[str] | None = None,
entity_ids: Mapping[str, str] | None = None,
):
self.id = id
self.version = version
self.steps: list[TaskStep] = steps or []
self.requires: list[str] | None = list(requires) if requires is not None else None
self.entity_ids: dict[str, str] | None = dict(entity_ids) if entity_ids else None
self._validate_dag(self.steps)

@classmethod
Expand Down Expand Up @@ -424,6 +428,8 @@ def to_dict(self) -> dict:
}
if self.requires is not None:
data["requires"] = list(self.requires)
if self.entity_ids:
data["entity_ids"] = dict(self.entity_ids)
return data

@classmethod
Expand All @@ -450,6 +456,7 @@ def from_dict(cls, data: dict) -> "Task":
version=data.get("version"),
steps=steps,
requires=data["requires"] if isinstance(data.get("requires"), list) else None,
entity_ids=data["entity_ids"] if isinstance(data.get("entity_ids"), dict) else None,
)

async def run(
Expand Down Expand Up @@ -495,6 +502,8 @@ async def _run(
if context is None:
context = TaskStepContext()
context.metadata.setdefault("run_group_id", uuid.uuid4().hex)
if self.entity_ids:
context.metadata.setdefault(ENTITY_IDS_KEY, dict(self.entity_ids))

# Seed substitution for EVERY step field, not just prompts. The hub's
# "Seed values" box writes context.metadata["seed"], and until now only
Expand Down
7 changes: 7 additions & 0 deletions src/agent_env/task_step/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,13 @@
from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url

HEARTBEAT_KEY = "_heartbeat_fn"
# context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run.
ENTITY_IDS_KEY = "entity_ids"


def run_entity_id(metadata: dict, entity_id: str) -> str:
"""The id the run's task gave the entity it names as ``entity_id`` (``Task.entity_ids``), else ``entity_id``."""
return (metadata.get(ENTITY_IDS_KEY) or {}).get(entity_id, entity_id)


class StepContextError(RuntimeError):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from agent_env.config import get_config
from agent_env.config.model import ModelParam
from agent_env.providers.sandbox_providers.sandbox import into_container
from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed
from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed, run_entity_id
from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import (
SCREENSHOT_GROUNDING,
JudgeOutputFormat,
Expand Down Expand Up @@ -188,10 +188,12 @@ def _bulk_verdicts_correction(eval_prompt: str, bulk: dict) -> str:
_MISSING = object()


def _lookup_dotted_path(root: object, path: str) -> object:
node = root
def _lookup_dotted_path(metadata: dict, path: str) -> object:
node = metadata
for segment in path.split("."):
if isinstance(node, dict):
if segment not in node:
segment = run_entity_id(metadata, segment)
if segment not in node:
return _MISSING
node = node[segment]
Expand All @@ -217,7 +219,9 @@ def _render_fact(value: object) -> str:
def resolve_context_facts(facts: dict[str, str], metadata: dict) -> list[tuple[str, str]]:
"""Resolve ``{label: dotted path}`` against ``metadata`` into ``[(label, rendered value)]``, in order.
A path walks dicts by key and lists by index; one that does not resolve renders as
``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises."""
``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises.
A key the run's task gave an entity another id under (``run_entity_id``, as a bundle does) is also found by the
id the task names it by, so a path naming an env as the task wrote it resolves wherever the task runs."""
rendered: list[tuple[str, str]] = []
for label, path in facts.items():
value = _lookup_dotted_path(metadata, path)
Expand Down
31 changes: 29 additions & 2 deletions tst/unit/bundle/ledger_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,13 @@ def _latest_version(collection, id):


def _written(write):
"""Stands in for the materializer's write: the next version of the entity, in the @local namespace's store."""
"""Stands in for the materializer's write: the next version of the entity, in the @local namespace's store, a
task's with the ids its bundle gave what it names, as Task.put stores them."""
collection = _COLLECTIONS[write.kind.store]
latest = local_store().find_one(collection, Filter.of(id=write.id), sort=Sort.by("version"))
version = latest["version"] + 1 if latest else 1
local_store().insert(collection, {"id": write.id, "version": version})
renamed = write.source.entity_ids if write.kind.store == "task" else {}
local_store().insert(collection, {"id": write.id, "version": version, **({"entity_ids": renamed} if renamed else {})})
return version


Expand Down Expand Up @@ -293,6 +295,18 @@ def edited_meanwhile():
assert ledger.check(write, {}).reasons == ("the ledger doesn't know what its last version was made from",)


def test_a_task_written_before_it_recorded_its_renamed_ids_is_written_again_with_them(bundle_dir, monkeypatch):
with monkeypatch.context() as earlier:
earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {}))
_run(bundle_dir)

checks = _run(bundle_dir)

assert checks[f"{ROOT}/t"].reasons == ("its stored version doesn't record the ids the bundle gave what it names",)
assert [id for id, check in checks.items() if not check.unchanged] == [f"{ROOT}/t"]
assert all(check.unchanged for check in _run(bundle_dir).values())


def test_a_new_digest_scheme_rewrites_everything_and_says_only_that(bundle_dir, monkeypatch):
_run(bundle_dir)
monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", ledger_module.SCHEME + 1)
Expand Down Expand Up @@ -524,6 +538,19 @@ def test_a_version_an_interrupted_run_wrote_is_kept_and_recorded_not_written_aga
assert [row["status"] for row in local_store().query(LEDGER_COLLECTION, Filter.of(id=GREETING))] == ["done"]


def test_a_task_an_interrupted_earlier_run_wrote_without_its_renamed_ids_is_written_again(bundle_dir, monkeypatch):
plan = plan_of(bundle_dir)
task = next(w for w in plan.writes if w.id == f"{ROOT}/t")
with monkeypatch.context() as earlier:
earlier.setattr(resolve_module.ResolvedEntry, "entity_ids", property(lambda self: {}))
_interrupted_after_its_write(Ledger.for_plan(plan), task, earlier)

check = Ledger.for_plan(plan).check(task, {})

assert not check.unchanged and not check.adopted
assert check.reasons == ("the store's latest, v1, wasn't recorded by this bundle",)


@pytest.mark.parametrize("since", ["another-write", "an-edit"])
def test_an_interrupted_runs_version_is_written_over_once_the_store_or_the_files_have_moved_on(
bundle_dir, monkeypatch, since,
Expand Down
10 changes: 10 additions & 0 deletions tst/unit/bundle/materialize_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,16 @@ def test_an_agent_is_written_from_its_agent_toml_over_a_store_image(bundle_dir):
assert _summary(_run(bundle_dir))[f"{ROOT}/solver"] == (1, True, ())


def test_the_written_task_records_the_ids_its_bundle_gave_what_it_names(bundle_dir):
_put_image("claude-image")
layout(bundle_dir, {"agents/solver/agent.toml": AGENT_TOML})
_steps(bundle_dir, [{"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "solver"}])

_run(bundle_dir)

assert Task.get(f"{ROOT}/t").entity_ids == {"solver": f"{ROOT}/solver"}


@pytest.mark.parametrize(("toml", "problem"), [
('image = "claude-image"\nowner = "me"\n',
"agent.toml: unknown key 'owner'; an a2a_agent agent takes image, default_env_vars, metadata, type and id"),
Expand Down
19 changes: 19 additions & 0 deletions tst/unit/bundle/resolve_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import json
import random
from dataclasses import replace

import pytest

Expand Down Expand Up @@ -149,6 +150,24 @@ def test_an_output_is_named_under_its_task_and_later_steps_find_it(make):
assert refs(entry) == [(EntityKind.ENV, f"{ROOT}/tickets", None, "tickets")]


def test_a_task_records_each_id_it_rewrote_as_written(make):
entry = resolved(make(tasks={"t": [
{"id": "deploy", "type": "deploy_env", "env_id": "tickets"},
{"id": "seed", "type": "load_artifact", "env_id": "tickets", "artifact_id": "shared-artifact"},
{"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"},
]}), "t")
assert entry.entity_ids == {"tickets": f"{ROOT}/tickets", "tickets-cli": f"{ROOT}/t/tickets-cli"}


def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make):
bundle = make(tasks={"t": [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}]})
entry = resolved(bundle, "t")
solver = next(e for e in bundle.entries if e.name == "solver")
swapped = replace(entry, references=(*entry.references,
Reference(EntityKind.AGENT, solver.id, None, solver, "--agent 'solver'", None)))
assert swapped.entity_ids == entry.entity_ids == {"tickets": f"{ROOT}/tickets"}


def test_an_output_id_a_step_derives_is_named_under_its_task_too(make):
entry = resolved(make(tasks={"t": [
{"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "tickets"},
Expand Down
61 changes: 61 additions & 0 deletions tst/unit/task/entity_ids_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
"""Task.entity_ids: the ids a bundle gave what a task names, stored with the task and put in each run's metadata."""

import pytest

from agent_env.config.runtime import Config
from agent_env.task import Task
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, run_entity_id
from agent_env.task_step.task_step import TaskStep

IDS = {"world": "@local/~/runs/pn/world"}


class _Seen(TaskStep):
"""Records the metadata its run reached it with."""

type = "entity_ids_test"
entity_refs = ()
seen: list[dict] = []

@classmethod
def from_dict(cls, data):
return cls(**cls._base_from_dict(data))

async def execute(self, context):
_Seen.seen.append(dict(context.metadata))
return context


@pytest.fixture(autouse=True)
def registries(monkeypatch):
steps = {**Config().task_step_registry(), _Seen.type: _Seen}
monkeypatch.setattr(Config, "task_step_registry", lambda self: steps)
monkeypatch.setattr(_Seen, "seen", [])


def _task(**kw) -> Task:
return Task(id="t", version=None, steps=[_Seen(id="s", version=None)], **kw)


def test_they_are_stored_with_the_task_and_left_out_when_there_are_none():
assert Task.from_dict(_task(entity_ids=IDS).to_dict()).entity_ids == IDS
assert "entity_ids" not in _task().to_dict()
assert "entity_ids" not in _task(entity_ids={}).to_dict()


@pytest.mark.asyncio
async def test_a_run_puts_them_in_its_metadata_for_run_entity_id(local_stores):
await _task(entity_ids=IDS).run(context=TaskStepContext())

metadata = _Seen.seen[0]
assert metadata[ENTITY_IDS_KEY] == IDS
assert run_entity_id(metadata, "world") == "@local/~/runs/pn/world"
assert run_entity_id(metadata, "elsewhere") == "elsewhere"


@pytest.mark.asyncio
async def test_a_task_stored_as_written_adds_nothing(local_stores):
await _task().run(context=TaskStepContext())

assert ENTITY_IDS_KEY not in _Seen.seen[0]
assert run_entity_id(_Seen.seen[0], "world") == "world"
Loading
Loading