From a0b37e7e64970a2a8e943a0e0bafaf9e471a9d18 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 14:38:25 -0800 Subject: [PATCH 1/4] fix(bundle): a bundle rewrites the ids a step's context paths name, as it does every other id A step type declares the fields that hold paths into a run's context (TaskStep.context_paths), and the bundle resolver renames each id such a path names to the one the bundle gives that entity, before the run. A renamed id that a dotted path can't hold is bracketed: env_trigger_state[@local//].statuses.wf-1. - rubrics_verifier and always_on_agent_verifier declare context_facts_for_judge, whose lookup is plain again. - bundle register renames such ids to registry ids, and refuses an @local id left inside a path. - A path naming a renamed id in a field its step doesn't declare is warned of when the bundle is resolved. - Run.firings(env=...) and Run.firing(..., env=...) take an env as the run's task names it. - A task written before this is written again: its resolved config, which the ledger hashes, changed. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/register.py | 15 ++++-- src/agent_env/bundle/resolve.py | 25 +++++++++- src/agent_env/entity/paths.py | 21 ++++++-- src/agent_env/task_step/context.py | 31 ++++++++++++ src/agent_env/task_step/task_step.py | 5 ++ .../verifiers/always_on_agent_verifier.py | 1 + .../task_steps/verifiers/rubrics_verifier.py | 27 +++++----- .../task_steps/verifiers/run_context.py | 15 +++--- tst/unit/bundle/ledger_test.py | 15 ++++++ tst/unit/bundle/register_test.py | 20 ++++++++ tst/unit/bundle/resolve_test.py | 28 +++++++++++ tst/unit/task_step/context_path_test.py | 38 ++++++++++++++ tst/unit/task_step/test_entity_refs.py | 26 ++++++++++ ...brics_verifier_context_facts_bundle_ids.py | 50 +++++++++++-------- .../task_step/verifiers/test_run_context.py | 14 ++++++ 15 files changed, 282 insertions(+), 49 deletions(-) create mode 100644 tst/unit/task_step/context_path_test.py diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index ca983e069..8adeab777 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -37,6 +37,7 @@ from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.image_store.oci_registry_credentials import is_loopback_host from agent_env.store.routing import configured_store, namespace_routing +from agent_env.task_step.context import context_path_segments, renamed_context_path from agent_env.task_step.registry import get_task_step_registry from ._secrets import credentials_in @@ -286,9 +287,13 @@ def _aliased(use: AliasUse, ids: Mapping[str, str]) -> AliasUse: def _renamed(value: Any, ids: Mapping[str, str]) -> Any: - """A copy of ``value`` with each string that's an ``@local`` id in ``ids``, or derived from one, its registry id.""" + """A copy of ``value`` with each string that's an ``@local`` id in ``ids``, or derived from one, its registry id, and + each such id in a path into a run's context its registry id too.""" if isinstance(value, str): - return ids.get(value) or _derived(value, ids) or value if is_local_id(value) else value + if is_local_id(value): + return ids.get(value) or _derived(value, ids) or value + return renamed_context_path(value, lambda segment: (ids.get(segment) or _derived(segment, ids)) + if is_local_id(segment) else None) if isinstance(value, dict): return {key: _renamed(item, ids) for key, item in value.items()} if isinstance(value, list): @@ -360,8 +365,10 @@ def _unregistrable(plan: Plan, shared: bool) -> Iterator[str]: continue where = plan.path(write.source.entry) for path, text in _strings(write.source.config): - if is_local_id(text): - yield f"{where}: {path or 'its value'} names {text!r}, which only this machine's stores hold" + local = text if is_local_id(text) else next( + (segment for segment in context_path_segments(text) or () if is_local_id(segment)), None) + if local is not None: + yield f"{where}: {path or 'its value'} names {local!r}, which only this machine's stores hold" elif shared and (url := _local_url(text)) is not None: yield f"{where}: {path or 'its value'} names {url}, which only this machine reaches" if shared: diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 686330e62..c0944eaad 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -3,13 +3,15 @@ References are the fields step types declare in ``entity_refs`` and the toml keys entity types declare in ``toml_refs``. The name of one of this bundle's entities becomes that entity's id, a name an output of the same task writes becomes the output's id, and anything else is a store id, left as written; -whether store ids exist is checked later, with the other checks that read a store. +whether store ids exist is checked later, with the other checks that read a store. An id renamed so is +also renamed in each path into the run's context a step type declares in ``context_paths``. """ from __future__ import annotations import copy import functools +import logging import unicodedata from collections.abc import Callable, Iterator from dataclasses import dataclass, field, replace @@ -19,6 +21,7 @@ from agent_env.a2a_agent import A2AAgent from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.registry import canonical_type, get_artifact_registry +from agent_env.entity.paths import field_sites from agent_env.entity.refs import EntityKind, EntityRef, RefRole, RefSite, parse_toml_ref, ref_sites from agent_env.env.envs.gateway_server import GatewayEnv from agent_env.env.envs.service_db import ServiceDBEnv @@ -28,6 +31,7 @@ from agent_env.plugins._requirements import unmet_problems from agent_env.store.ids import HF_PREFIX, derive_id, is_local_id, validate_local_id from agent_env.store.image_store.oci_registry_credentials import names_registry +from agent_env.task_step.context import context_path_segments, renamed_context_path from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies @@ -36,6 +40,8 @@ from .parse import BUNDLE_TOML, DOCKERFILE, NAMED_BY, Bundle, BundleEntry, BundleError, BundleKind, foreign_hub_id from .requires import UnmetRequirements +logger = logging.getLogger(__name__) + # Step keys that hold step ids, never entity ids. _STEP_ID_KEYS = ("id", "type", "depends_on") # The env types config names one of, for every deploy that needs one, by the setting that names it. @@ -258,6 +264,9 @@ def _task(self, entry: BundleEntry) -> ResolvedEntry: continue self._resolve(entry, site, f"{where}{site.path}", site.value, site.version, outputs, functools.partial(_upstream, edges, index), references) + renamed = _rewritten_ids(entry.config, steps) + for _, step, cls in declared: + self._rename_context_paths(entry, step, cls, renamed) return ResolvedEntry(entry, steps, tuple(references), tuple(outputs.values())) def _image_folder(self, entry: BundleEntry) -> bool: @@ -484,6 +493,20 @@ def _undeclared(self, entry: BundleEntry, step: dict, cls: type, outputs: dict[t self._problem(entry, f"{where}{path} = {value!r} names {named}, but {cls.type} doesn't declare its " "entity_refs, so it can't be rewritten") + def _rename_context_paths(self, entry: BundleEntry, step: dict, cls: type, renamed: dict[str, str]) -> None: + """Rename each id ``renamed`` holds in the paths into the run's context ``cls`` declares (``context_paths``). + Warn of a path naming one in a field ``cls`` doesn't declare, which the run would look up as written.""" + for path in cls.context_paths: + for _, holder, key in field_sites(step, path): + if isinstance(holder[key], str): + holder[key] = renamed_context_path(holder[key], renamed.get) + for where, value in _strings(step): + segments = context_path_segments(value) or [] + if named := next((segment for segment in segments[1:] if segment in renamed), None): + logger.warning("%s: step %r: %s = %r names %r, which this bundle writes as %r, but %s doesn't declare " + "the field in context_paths, so it isn't rewritten", relative(self.bundle.root, entry.path), + step.get("id"), where, value, named, renamed[named], cls.type) + def _problem(self, entry: BundleEntry, message: str) -> None: self.problems.append(f"{relative(self.bundle.root, entry.path)}: {message}") diff --git a/src/agent_env/entity/paths.py b/src/agent_env/entity/paths.py index f10a72376..d2f5b23a4 100644 --- a/src/agent_env/entity/paths.py +++ b/src/agent_env/entity/paths.py @@ -10,13 +10,24 @@ PATH = re.compile(r"[A-Za-z_]\w*(\[\]|\{\})?(\.[A-Za-z_]\w*(\[\]|\{\})?)*") -def field_sites(data: dict, path: str) -> Iterator[tuple[str, dict, str]]: - """Each place ``path`` holds a value that isn't None in ``data``: where, as a path with indexes, the dict holding - it, and its key there.""" +def field_sites(data: dict, path: str) -> Iterator[tuple[str, dict | list, str | int]]: + """Each place ``path`` holds a value that isn't None in ``data``: where, as a path with indexes, what holds it, and + its key there. A last segment's ``[]`` or ``{}`` is each element of its list or value of its table.""" *parents, leaf = path.split(".") + key = leaf.removesuffix("[]").removesuffix("{}") for trail, holder in descend(data, parents, ()): - if isinstance(holder, dict) and holder.get(leaf) is not None: - yield ".".join((*trail, leaf)), holder, leaf + value = holder.get(key) if isinstance(holder, dict) else None + if value is None: + continue + where = ".".join((*trail, key)) + if leaf.endswith("[]"): + items = enumerate(value) if isinstance(value, list) else () + yield from ((f"{where}[{i}]", value, i) for i, item in items if item is not None) + elif leaf.endswith("{}"): + items = value.items() if isinstance(value, dict) else () + yield from ((f"{where}.{name}", value, name) for name, item in items if item is not None) + else: + yield where, holder, key def descend(node: Any, segments: list[str], trail: tuple[str, ...]) -> Iterator[tuple[tuple[str, ...], Any]]: diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 059183bf9..b92d0fa58 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -4,6 +4,8 @@ import contextlib import dataclasses +import re +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -22,6 +24,24 @@ def run_entity_id(metadata: dict, entity_id: str) -> str: return (metadata.get(ENTITY_IDS_KEY) or {}).get(entity_id, entity_id) +def context_path_segments(path: Any) -> list[str] | None: + """The keys and list indexes a path into a run's context walks, or None when ``path`` isn't one. A segment is + letters, digits, ``_`` and ``-`` after a ``.``, or any id in brackets: ``env_trigger_state[@local/x/crm].statuses``.""" + if not isinstance(path, str) or not _CONTEXT_PATH.fullmatch(path): + return None + return [plain or bracketed for plain, bracketed in _CONTEXT_PATH_SEGMENT.findall(path)] + + +def renamed_context_path(path: str, rename: Callable[[str], str | None]) -> str: + """``path`` with each segment ``rename`` gives another id for naming that id; ``path`` itself when it isn't a path + into a run's context or ``rename`` renames none of its segments.""" + segments = context_path_segments(path) or [] + renamed = [rename(segment) or segment for segment in segments] + if renamed == segments: + return path + return "".join(_path_segment(segment, first=i == 0) for i, segment in enumerate(renamed)) + + class StepContextError(RuntimeError): """The run's context can't satisfy a step: something the step needs from an earlier step is missing, or something it would create is already there. Running the step again on the same context fails the same @@ -293,3 +313,14 @@ def from_dict(cls, data: dict[str, Any]) -> TaskStepContext: agent_harness=data.get("agent_harness"), instance_id=data.get("instance_id"), ) + + +_PLAIN_SEGMENT = re.compile(r"[\w-]+") +_CONTEXT_PATH = re.compile(r"(?:[\w-]+|\[[^\[\]]+\])(?:\.[\w-]+|\[[^\[\]]+\])*") +_CONTEXT_PATH_SEGMENT = re.compile(r"\.?([\w-]+)|\[([^\[\]]+)\]") + + +def _path_segment(segment: str, *, first: bool) -> str: + if not _PLAIN_SEGMENT.fullmatch(segment): + return f"[{segment}]" + return segment if first else f".{segment}" diff --git a/src/agent_env/task_step/task_step.py b/src/agent_env/task_step/task_step.py index 1f572e941..c26c6dc6c 100644 --- a/src/agent_env/task_step/task_step.py +++ b/src/agent_env/task_step/task_step.py @@ -91,6 +91,9 @@ class TaskStep(ABC): DEFAULT_AGENT_NAME: ClassVar[str] = "default-agent" # Fields holding env, A2A agent or artifact ids; None means undeclared. entity_refs: ClassVar[tuple[EntityRef, ...] | None] = None + # Fields holding paths into a run's context (``context_path_segments``), as dotted keys with ``[]`` / ``{}`` for each + # element or value; a bundle rewrites the ids in them as it does entity_refs'. + context_paths: ClassVar[tuple[str, ...]] = () # Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A # subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs. field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER} @@ -100,6 +103,8 @@ def __init_subclass__(cls, **kwargs: Any) -> None: # Never inherited: a subclass may add id fields its parent's declaration misses. if "entity_refs" not in vars(cls): cls.entity_refs = None + if "context_paths" not in vars(cls): + cls.context_paths = () def __init__( self, diff --git a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py index aa4765afb..372163820 100644 --- a/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/always_on_agent_verifier.py @@ -29,6 +29,7 @@ class AlwaysOnAgentVerifier(RubricsVerifierTaskStep): type: ClassVar[str] = "always_on_agent_verifier" entity_refs = RubricsVerifierTaskStep.entity_refs # the judge, as rubrics_verifier's + context_paths = RubricsVerifierTaskStep.context_paths def __init__( self, diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 006a9b0ed..12c19dfab 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -21,7 +21,13 @@ from agent_env.config import get_config from agent_env.config.model import ModelParam from agent_env.providers.sandbox_providers.sandbox import into_container -from agent_env.task_step.context import PromptResponse, StepContextError, TaskStepContext, dual_keyed, run_entity_id +from agent_env.task_step.context import ( + PromptResponse, + StepContextError, + TaskStepContext, + context_path_segments, + dual_keyed, +) from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( SCREENSHOT_GROUNDING, JudgeOutputFormat, @@ -126,9 +132,6 @@ def trajectory_mistakes_key(verifier_id: str) -> str: return f"{verifier_id}{TRAJECTORY_MISTAKES_SUFFIX}" -# A context_facts_for_judge path: dotted segments of letters, digits, `_` and `-` (dict keys, or a list index). -_CONTEXT_FACT_PATH = re.compile(r"^[\w-]+(\.[\w-]+)*$") - # A graded batch is a bulk verdict when this many of its rows share one justification (at least # _BULK_MIN_SHARED of them), or when this share of justifications is shorter than _BULK_SHORT_CHARS. _BULK_SHARED_FRACTION = 0.25 @@ -190,11 +193,11 @@ def _bulk_verdicts_correction(eval_prompt: str, bulk: dict) -> str: def _lookup_dotted_path(metadata: dict, path: str) -> object: - node = metadata - for segment in path.split("."): + node, segments = metadata, context_path_segments(path) + if segments is None: + return _MISSING + for segment in segments: if isinstance(node, dict): - if segment not in node: - segment = run_entity_id(metadata, segment) if segment not in node: return _MISSING node = node[segment] @@ -221,8 +224,7 @@ def resolve_context_facts(facts: dict[str, str], metadata: dict) -> list[tuple[s """Resolve ``{label: dotted path}`` against ``metadata`` into ``[(label, rendered value)]``, in order. A path walks dicts by key and lists by index; one that does not resolve renders as ``CONTEXT_FACT_NOT_AVAILABLE`` so the judge sees the fact was requested. Never raises. - A key the run's task gave an entity another id under (``run_entity_id``, as a bundle does) is also found by the - id the task names it by, so a path naming an env as the task wrote it resolves wherever the task runs.""" + A bundle rewrites an id a path names to the one it gives that entity (``TaskStep.context_paths``).""" rendered: list[tuple[str, str]] = [] for label, path in facts.items(): value = _lookup_dotted_path(metadata, path) @@ -401,6 +403,7 @@ def _as_of_turn(prompt_response: PromptResponse, turn: Optional[int]) -> PromptR class RubricsVerifierTaskStep(TaskStep): type: ClassVar[str] = "rubrics_verifier" entity_refs = (EntityRef.agent("judge_a2a_agent_id"),) + context_paths = ("context_facts_for_judge{}",) field_scopes = {**TaskStep.field_scopes, **dict.fromkeys( ("judge_a2a_agent_id", "default_model", "default_model_api_base", "effort", "max_thinking_tokens", "judge_sandbox_type", "judge_batch_size", "judge_batch_concurrency"), FieldScope.USER)} @@ -479,12 +482,12 @@ def __init__( # and shown as a "## Context facts" section the judge trusts over the pixels. if context_facts_for_judge is not None: if not isinstance(context_facts_for_judge, dict) or not all( - isinstance(label, str) and label and isinstance(path, str) and _CONTEXT_FACT_PATH.match(path) + isinstance(label, str) and label and context_path_segments(path) is not None for label, path in context_facts_for_judge.items() ): raise ValueError( "context_facts_for_judge must map non-empty labels to dotted paths of identifiers " - "(letters, digits, '_' and '-' per segment), e.g. " + "(letters, digits, '_' and '-' per segment, or any id in brackets), e.g. " '{"items in cart": "verifications.cart-check.count"}.' ) self.context_facts_for_judge = context_facts_for_judge diff --git a/src/agent_env/task_step/task_steps/verifiers/run_context.py b/src/agent_env/task_step/task_steps/verifiers/run_context.py index 6356f53d2..7c976e172 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_context.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_context.py @@ -10,7 +10,7 @@ from agent_env.a2a_agent import conversation_store from agent_env.config import get_config -from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.context import ENTITY_IDS_KEY, PromptResponse, TaskStepContext class RunRecordUnavailable(LookupError): @@ -83,6 +83,7 @@ class Run: _firings: tuple[Firing, ...] = () # oldest first _logs: tuple[str, ...] = () # the firing logs that were read, as ``Firing.source`` names them unavailable: Mapping[str, str] = field(default_factory=dict) # what could not be read, and why + _entity_ids: Mapping[str, str] = field(default_factory=dict, repr=False) # the run's task's ``Task.entity_ids`` @property def turns(self) -> tuple[Turn, ...]: @@ -95,8 +96,9 @@ def turn(self, number: int) -> Turn: return self._turns[number - 1] def firings(self, trigger_id: Optional[str] = None, *, env: Optional[str] = None) -> tuple[Firing, ...]: - """Oldest first, of ``trigger_id`` and ``env`` when given; raises ``RunRecordUnavailable`` for a missing log.""" - source = None if env is None else f"env:{env}" + """Oldest first, of ``trigger_id`` and ``env`` when given, ``env`` as the run's task names it; raises + ``RunRecordUnavailable`` for a missing log.""" + source = None if env is None else f"env:{self._entity_ids.get(env, env)}" _need(self.unavailable, [source] if source else self.unavailable) if source and source not in self._logs: raise RunRecordUnavailable(source, "no firing log was captured") @@ -169,7 +171,7 @@ async def load_run_context(context: TaskStepContext, prompt_response: Optional[P if prompt_response is not None: env_events = await asyncio.to_thread(_until_next_prompt, env_events, context, prompt_response, unavailable) return _run(conversation, {**{f"env:{e}": events for e, events in env_events.items()}, "user_agent": agent_events}, - unavailable) + unavailable, context.metadata.get(ENTITY_IDS_KEY) or {}) def _error(e: Exception) -> str: @@ -180,7 +182,8 @@ def _time(ts: Optional[str]) -> Optional[datetime]: return datetime.fromisoformat(ts) if ts else None -def _run(conversation: list[Message], logs: dict[str, list[dict]], unavailable: dict[str, str]) -> Run: +def _run(conversation: list[Message], logs: dict[str, list[dict]], unavailable: dict[str, str], + entity_ids: Mapping[str, str] | None = None) -> Run: """Places every firing in its turn; without the conversation the firing logs are withheld, as they can't be.""" if "conversation" in unavailable: why = "withheld: without the conversation, its firings can't be placed in turns" @@ -200,7 +203,7 @@ def _run(conversation: list[Message], logs: dict[str, list[dict]], unavailable: turns = tuple(Turn(number=m.turn, message=m, reply=replies.get(m.turn), _unavailable=unavailable, _firings=tuple(f for f in firings if f.turn == m.turn)) for m in opened) return Run(_turns=turns, _firings=tuple(firings), _logs=tuple(s for s in logs if s not in unavailable), - unavailable=unavailable) + unavailable=unavailable, _entity_ids=entity_ids or {}) def _fold(source: str, events: list[dict], starts: list[Optional[datetime]]) -> list[Firing]: diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index 99a132006..ff8f8ad71 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -307,6 +307,21 @@ def test_a_task_written_before_it_recorded_its_renamed_ids_is_written_again_with assert all(check.unchanged for check in _run(bundle_dir).values()) +def test_a_task_written_before_its_context_paths_named_the_bundle_s_ids_is_written_again(bundle_dir, monkeypatch): + judge = {"id": "judge", "type": "rubrics_verifier", "criteria": [{"id": "c", "description": "d"}], + "prompt_id": "p1", "context_facts_for_judge": {"wf-1": "env_trigger_state.tickets.statuses.wf-1"}} + (bundle_dir / "tasks" / "t.json").write_text(_steps(judge)) + with monkeypatch.context() as earlier: + earlier.setattr(resolve_module._Resolver, "_rename_context_paths", lambda *args: None) + _run(bundle_dir) + + checks = _run(bundle_dir) + + assert checks[f"{ROOT}/t"].reasons == ("config changed",) + assert [id for id, check in checks.items() if not check.unchanged] == [f"{ROOT}/t"] + assert all(check.unchanged for check in _run(bundle_dir).values()) + + def test_a_new_digest_scheme_rewrites_everything_and_says_only_that(bundle_dir, monkeypatch): _run(bundle_dir) monkeypatch.setattr("agent_env.bundle.ledger.SCHEME", ledger_module.SCHEME + 1) diff --git a/tst/unit/bundle/register_test.py b/tst/unit/bundle/register_test.py index dcf233716..80341f3c3 100644 --- a/tst/unit/bundle/register_test.py +++ b/tst/unit/bundle/register_test.py @@ -313,6 +313,26 @@ def test_an_at_local_id_left_in_what_would_be_written_is_refused(home): "tasks/t.json: [0].metadata.from names '@local/~/elsewhere/x', which only this machine's stores hold",) +def test_an_at_local_id_left_in_a_path_into_the_run_s_context_is_refused(home): + files = {"tasks/t.json": json.dumps([{**BOX, "metadata": {"from": "env_trigger_state[@local/~/elsewhere/x].s"}}])} + + with pytest.raises(BundleError) as refused: + register_bundle(layout(home / "triage", files)) + + assert refused.value.problems == ( + "tasks/t.json: [0].metadata.from names '@local/~/elsewhere/x', which only this machine's stores hold",) + + +def test_a_context_fact_naming_a_bundle_entity_names_its_registry_id(home): + judge = {"id": "judge", "type": "rubrics_verifier", "criteria": [{"id": "c", "description": "d"}], + "prompt_id": "p1", "context_facts_for_judge": {"greeting": "loaded.greeting.version"}} + steps = [BOX, _load("greet", "greeting"), _load("docs", "docs"), judge] + + register_bundle(layout(home / "triage", {**TRIAGE, "tasks/t.json": json.dumps(steps)})) + + assert _steps("triage-t")[-1]["context_facts_for_judge"] == {"greeting": "loaded.triage-greeting.version"} + + def test_a_shared_store_refuses_a_credential_or_reference_in_env_vars_and_a_url_only_this_machine_reaches( home, monkeypatch): monkeypatch.setattr(register_module, "shared_store", lambda store: True) diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 824adb98a..b729dd33c 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -1,6 +1,7 @@ """Resolving a parsed bundle: every declared reference rewritten to the id it names, before any write.""" import json +import logging import random from dataclasses import replace @@ -168,6 +169,33 @@ def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make): assert swapped.entity_ids == entry.entity_ids == {"tickets": f"{ROOT}/tickets"} +def test_a_context_fact_naming_what_the_bundle_renamed_names_it_by_its_new_id(make): + facts = {"wf-1": "env_trigger_state.tickets.statuses.wf-1", "cli": "env_changes.tickets-cli.rows", + "store env": "env_trigger_state.elsewhere.statuses.wf-1", "score": "verifications.judge.score"} + entry = resolved(make(tasks={"t": [ + {"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, + {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, + {"id": "judge", "type": "rubrics_verifier", "context_facts_for_judge": facts}, + ]}), "t") + + assert entry.config[2]["context_facts_for_judge"] == { + "wf-1": f"env_trigger_state[{ROOT}/tickets].statuses.wf-1", "cli": f"env_changes[{ROOT}/t/tickets-cli].rows", + "store env": "env_trigger_state.elsewhere.statuses.wf-1", "score": "verifications.judge.score"} + + +def test_a_path_naming_what_the_bundle_renamed_in_a_field_its_step_doesnt_declare_is_warned_of(make, caplog): + steps = [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, + {"id": "note", "type": "plugin_test", "path": "env_trigger_state.tickets.statuses", "file": "tickets.json"}] + + with caplog.at_level(logging.WARNING, logger="agent_env.bundle.resolve"): + entry = resolved(make(tasks={"t": steps}), "t") + + assert entry.config[1] == steps[1] + assert [record.getMessage() for record in caplog.records] == [ + "tasks/t.json: step 'note': path = 'env_trigger_state.tickets.statuses' names 'tickets', which this bundle " + f"writes as '{ROOT}/tickets', but plugin_test doesn't declare the field in context_paths, so it isn't rewritten"] + + def test_an_output_id_a_step_derives_is_named_under_its_task_too(make): entry = resolved(make(tasks={"t": [ {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "tickets"}, diff --git a/tst/unit/task_step/context_path_test.py b/tst/unit/task_step/context_path_test.py new file mode 100644 index 000000000..fc498b1ba --- /dev/null +++ b/tst/unit/task_step/context_path_test.py @@ -0,0 +1,38 @@ +"""Paths into a run's context: dotted keys, or any id in brackets, as a bundle writes an id it renamed.""" +from __future__ import annotations + +import pytest + +from agent_env.task_step.context import context_path_segments, renamed_context_path + +ID = "@local/~/runs/pn.v2/tickets" + + +@pytest.mark.parametrize("path, segments", [ + ("verifications.cart-check.count", ["verifications", "cart-check", "count"]), + (f"env_trigger_state[{ID}].statuses.wf-1", ["env_trigger_state", ID, "statuses", "wf-1"]), + (f"[{ID}].rows", [ID, "rows"]), + ("items.0", ["items", "0"]), +]) +def test_a_path_walks_its_keys_and_a_bracketed_id_is_one_key(path, segments): + assert context_path_segments(path) == segments + + +@pytest.mark.parametrize("path", ["", "a..b", "a.", ".a", "a/b", "a[b", "a[]", "a[[b]]", "a b", None, 3]) +def test_anything_else_is_no_path(path): + assert context_path_segments(path) is None + + +def test_a_renamed_id_is_bracketed_unless_a_dotted_path_can_hold_it(): + rename = {"tickets": ID, "crm": "triage-crm"}.get + + assert renamed_context_path("env_trigger_state.tickets.statuses.wf-1", rename) == ( + f"env_trigger_state[{ID}].statuses.wf-1") + assert renamed_context_path("env_changes.crm.rows", rename) == "env_changes.triage-crm.rows" + assert renamed_context_path("tickets.rows", rename) == f"[{ID}].rows" + assert context_path_segments(renamed_context_path("tickets.rows", rename)) == [ID, "rows"] + + +@pytest.mark.parametrize("path", ["verifications.judge.score", "a note about tickets", f"env_trigger_state[{ID}].x"]) +def test_a_path_naming_no_renamed_id_and_a_string_that_isnt_a_path_are_left_as_they_are(path): + assert renamed_context_path(path, {"tickets": ID}.get) == path diff --git a/tst/unit/task_step/test_entity_refs.py b/tst/unit/task_step/test_entity_refs.py index 77d4d7fcb..974a76f2d 100644 --- a/tst/unit/task_step/test_entity_refs.py +++ b/tst/unit/task_step/test_entity_refs.py @@ -5,6 +5,7 @@ import pytest from agent_env.artifact.registry import get_artifact_registry +from agent_env.entity.paths import PATH, field_sites from agent_env.entity.refs import EntityKind, EntityRef, default_agent_refs, parse_toml_ref, ref_sites from agent_env.task_step.registry import _builtin_registry from agent_env.task_step.snapshot_utils.snapshot_series import SnapshotConfig @@ -14,6 +15,7 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.register_agent_triggers import ( RegisterAgentTriggersStep, _env_trigger_conditions, @@ -196,6 +198,30 @@ class Declared(DeployEnvTaskStep): assert DeployEnvTaskStep.entity_refs[0].version_field == "env_version" +@pytest.mark.parametrize("step_type,cls", BUILTINS) +def test_builtin_context_paths_name_real_fields(step_type, cls): + for path in cls.context_paths: + assert PATH.fullmatch(path), f"{step_type}: {path} isn't dotted keys" + assert path.split(".")[0].removesuffix("[]").removesuffix("{}") in _params(cls), ( + f"{step_type}: {path} names no constructor parameter") + + +def test_a_subclass_declares_no_context_paths_until_it_declares(): + class Variant(RubricsVerifierTaskStep): + type = "rubrics_verifier_variant" + + assert Variant.context_paths == () + assert RubricsVerifierTaskStep.context_paths == ("context_facts_for_judge{}",) + + +def test_a_path_s_last_braces_or_brackets_reach_each_value_or_element(): + step = {"facts": {"a": "x.y", "b": None}, "ids": ["p", None, "q"], "one": "z"} + + assert [(where, key) for where, _, key in field_sites(step, "facts{}")] == [("facts.a", "a")] + assert [(where, key) for where, _, key in field_sites(step, "ids[]")] == [("ids[0]", 0), ("ids[2]", 2)] + assert [(where, key) for where, _, key in field_sites(step, "one")] == [("one", "one")] + + @pytest.mark.parametrize("bad", [ lambda: EntityRef("env id", EntityKind.ENV), lambda: EntityRef("env_id", "env"), diff --git a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py index 989555065..e836c4234 100644 --- a/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py +++ b/tst/unit/task_step/verifiers/test_rubrics_verifier_context_facts_bundle_ids.py @@ -1,5 +1,5 @@ -"""context_facts_for_judge in a bundle run: a path naming an entity as the task wrote it also finds the key under the -id the run's task gave it (``entity_ids``), as a bundle-written task's run records.""" +"""context_facts_for_judge in a bundle run: the bundle writes a path naming one of its entities with the id it gives +that entity, in brackets, and the judge's fact resolves through it as written.""" from __future__ import annotations import pytest @@ -12,7 +12,7 @@ ) ENV = "@local/~/runs/pn/world" -FACTS = {"wf-1 status": "env_trigger_state.world.statuses.wf-1"} +FACTS = {"wf-1 status": f"env_trigger_state[{ENV}].statuses.wf-1"} def _resolved(metadata: dict, facts: dict = FACTS) -> str: @@ -23,33 +23,41 @@ def _state(key: str, status: str) -> dict: return {key: {"statuses": {"wf-1": status}}} -def test_a_renamed_env_is_found_by_the_id_the_task_names_it_by(): - assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}) == '"armed"' +def _verifier(facts: dict) -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep( + id="validity", version=1, criteria=[{"id": "world", "description": "the events fired"}], prompt_id="p1", + use_agent_judge=False, use_trajectory=True, verifier_id="validity", context_facts_for_judge=facts, + ) + + +def test_a_bracketed_id_is_one_key(): + assert _resolved({"env_trigger_state": _state(ENV, "armed")}) == '"armed"' -def test_a_renamed_step_output_is_found_too(): - snapshot = "@local/~/runs/pn/task-a/snap" - metadata = {"entity_ids": {"snap": snapshot}, "env_changes": {snapshot: {"rows": 3}}} - assert _resolved(metadata, {"rows": "env_changes.snap.rows"}) == "3" +def test_a_path_resolves_as_written_only(): + metadata = {"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")} + assert _resolved(metadata, {"wf-1 status": "env_trigger_state.world.statuses.wf-1"}) == CONTEXT_FACT_NOT_AVAILABLE -def test_a_key_as_written_still_wins(): - state = {**_state("world", "fired"), **_state(ENV, "armed")} - assert _resolved({"entity_ids": {"world": ENV}, "env_trigger_state": state}) == '"fired"' +def test_a_string_that_isnt_a_path_resolves_to_nothing(): + assert _resolved({"a": 1}, {"fact": "a b"}) == CONTEXT_FACT_NOT_AVAILABLE -@pytest.mark.parametrize("entity_ids", [None, {}, {"other": ENV}], ids=["no map", "empty map", "not in the map"]) -def test_without_a_rename_a_key_resolves_only_as_written(entity_ids): - metadata = {"env_trigger_state": _state(ENV, "fired"), **({"entity_ids": entity_ids} if entity_ids is not None else {})} - assert _resolved(metadata) == CONTEXT_FACT_NOT_AVAILABLE +@pytest.mark.parametrize("path", [FACTS["wf-1 status"], "verifications.cart-check.count"]) +def test_the_step_keeps_a_path_with_or_without_a_bracketed_id(path): + step = _verifier({"fact": path}) + assert RubricsVerifierTaskStep.from_dict(step.to_dict()).context_facts_for_judge == {"fact": path} + + +@pytest.mark.parametrize("path", ["env_trigger_state.@local/x.statuses", "a..b", "a[b"]) +def test_the_step_refuses_anything_else(path): + with pytest.raises(ValueError, match="context_facts_for_judge"): + _verifier({"fact": path}) @pytest.mark.asyncio async def test_the_judge_sees_the_fact_in_a_bundle_run(monkeypatch): - v = RubricsVerifierTaskStep( - id="validity", version=1, criteria=[{"id": "world", "description": "the events fired"}], prompt_id="p1", - use_agent_judge=False, use_trajectory=True, verifier_id="validity", context_facts_for_judge=FACTS, - ) + v = _verifier(FACTS) captured = {} async def fake_run(*, eval_prompt, **kwargs): @@ -61,7 +69,7 @@ async def fake_run(*, eval_prompt, **kwargs): ctx = TaskStepContext( prompt_responses=[PromptResponse(prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json")], - metadata={"entity_ids": {"world": ENV}, "env_trigger_state": _state(ENV, "armed")}, + metadata={"env_trigger_state": _state(ENV, "armed")}, ) await v.execute(ctx) diff --git a/tst/unit/task_step/verifiers/test_run_context.py b/tst/unit/task_step/verifiers/test_run_context.py index f6f542fd4..4f3a19cb6 100644 --- a/tst/unit/task_step/verifiers/test_run_context.py +++ b/tst/unit/task_step/verifiers/test_run_context.py @@ -98,6 +98,20 @@ async def test_the_run_has_every_turn_and_one_firing_per_fire_each_in_its_turn() assert run.firings(env="calendar") == run.firings("confirm") + run.firings("late") +@pytest.mark.asyncio +async def test_an_env_the_run_s_task_renamed_is_found_by_the_id_the_task_names_it_by(): + context, prompt_response, _ = recorded_run() + renamed = "@local/~/runs/pn/calendar" + state = context.metadata["env_trigger_state"] + state[renamed] = state.pop("calendar") + context.metadata["entity_ids"] = {"calendar": renamed} + + run = await load_run_context(context, prompt_response) + + assert run.firing("confirm", env="calendar").source == f"env:{renamed}" + assert run.firings(env="calendar") == run.firings(env=renamed) == run.firings("confirm") + run.firings("late") + + def test_the_gateway_log_folds_into_one_firing_per_fire(): ts = [f"2026-10-20T10:00:0{i}+00:00" for i in range(8)] events = [ From b5be976dd4562280205fb1000739fbe411dcf3e3 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 14:51:09 -0800 Subject: [PATCH 2/4] fix(bundle): an import names an entity in a context path as the folder names it A task's import rewrites each id a field of its steps names to the name the folder gives that entity, numbered when the name is taken. A declared path into the run's context now takes the same name, so a fact naming an env the folder writes as crm-env-2 names it so. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/importer.py | 15 +++++++++++++++ tst/unit/bundle/importer_test.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 712941c95..79cad09b1 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -55,6 +55,7 @@ from agent_env.bundle.resolve import resolve_bundle from agent_env.config import get_config from agent_env.entity.export import ExportContext, Unwritable, checked_path +from agent_env.entity.paths import field_sites from agent_env.entity.refs import EntityKind, EntityRef, RefRole, RefSite, default_agent_refs, ref_sites from agent_env.entity.scopes import FieldScope, prune from agent_env.env.env import Env @@ -66,6 +67,7 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import pin_digest from agent_env.task.task import Task +from agent_env.task_step.context import renamed_context_path from agent_env.task_step.registry import get_task_step_registry from agent_env.utils.paths import os_reason @@ -238,6 +240,7 @@ def add_task(self, task: Task) -> str: if _key(site) not in outputs: # the run writes an output again, so it's no entity to write here self._ref(site, f"{where}: {site.path}", pins) self._default_agents(step, cls.entity_refs or (), where) + _rename_context_paths(found, sites) self.files[path] = json.dumps(steps, indent=2, ensure_ascii=False, allow_nan=False) + "\n" self.tasks.append(ImportedTask(path, task.id, task.version, len(steps))) self.declared += task.requires or () @@ -671,3 +674,15 @@ def _key(site: RefSite) -> tuple[EntityKind, str]: def _nfc(text: str) -> str: return unicodedata.normalize("NFC", text) + + +def _rename_context_paths(found: list[tuple[dict, type, Any, list[RefSite]]], sites: list[RefSite]) -> None: + """Rename, in the paths into the run's context each step's type declares, each id a field of the task was + rewritten from to what it names in the folder, so a path names an entity as its field does.""" + renamed = {site.value: site.owner[site.key] for site in sites + if isinstance(site.owner[site.key], str) and site.owner[site.key] != site.value} + for step, cls, *_ in found: + for path in cls.context_paths: + for _, holder, key in field_sites(step, path): + if isinstance(holder[key], str): + holder[key] = renamed_context_path(holder[key], renamed.get) diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index 32d9dc981..b892972b2 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -220,6 +220,21 @@ def test_an_id_a_bundle_named_under_its_task_is_renamed_so_the_import_names_it_u assert (steps[1]["cli_artifact_id"], steps[2]["cli_artifact_ids"]) == ("crm-cli", ["crm-cli"]) +def test_a_context_fact_names_an_entity_as_its_folder_does_when_the_folder_isnt_named_as_its_id(store): + other, old = "@local/~/elsewhere/crm-env", "@local/~/old/triage" + judge = {**STEPS[4], "context_facts_for_judge": {"mine": "env_trigger_state.crm-env.statuses.wf-1", + "other": f"env_trigger_state[{other}].statuses.wf-1"}} + with namespace_routing(): + MCPServerEnv.put(id=other, docker_image_artifact=context_image(store, "crm-image"), environment_name="crm") + _task(old, [STEPS[0], {**STEPS[0], "id": "env2", "env_id": other}, STEPS[2], judge]) + imported = import_task(old, store / "again") + steps = _steps(imported.root, "triage") + + assert (steps[0]["env_id"], steps[1]["env_id"]) == ("crm-env", "crm-env-2") + assert steps[3]["context_facts_for_judge"] == {"mine": "env_trigger_state.crm-env.statuses.wf-1", + "other": "env_trigger_state.crm-env-2.statuses.wf-1"} + + def test_a_renamed_output_is_numbered_when_another_output_or_a_store_reference_has_its_name(store): old = "@local/~/old/triage" with namespace_routing(): From 64991e7bbd2c8fa6ba56aba7e0c1e78a765db4f2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 15:27:21 -0800 Subject: [PATCH 3/4] fix(bundle): a context path's env follows every rewrite of the task's env fields, and nothing else in it changes A path is rebuilt from the one the task wrote, after each place that rewrites a task's env fields: resolving, pointing an installed alias, repointing a variant, and importing. Only the segment that names an env changes: the one after a key whose entries are keyed by env id (ENV_KEYED_METADATA). A key that only shares an entity's name, such as a verifier's id, is kept, and an env named by an installed alias is named by what the alias names. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/importer.py | 20 ++---- src/agent_env/bundle/plan.py | 4 +- src/agent_env/bundle/resolve.py | 80 +++++++++++++++------- src/agent_env/bundle/variants.py | 6 +- src/agent_env/task_step/context.py | 11 ++- tst/unit/bundle/installed_entities_test.py | 9 +++ tst/unit/bundle/ledger_test.py | 2 +- tst/unit/bundle/register_test.py | 10 ++- tst/unit/bundle/resolve_test.py | 26 +++---- tst/unit/bundle/variants_test.py | 12 ++++ 10 files changed, 115 insertions(+), 65 deletions(-) diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 79cad09b1..8fd33967d 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -4,6 +4,7 @@ from __future__ import annotations import contextlib +import copy import json import os import re @@ -52,10 +53,9 @@ ) from agent_env.bundle.plan import STORE_GETTERS, check_bundle from agent_env.bundle.requires import Providers, Use, detected -from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle.resolve import resolve_bundle, sync_context_paths from agent_env.config import get_config from agent_env.entity.export import ExportContext, Unwritable, checked_path -from agent_env.entity.paths import field_sites from agent_env.entity.refs import EntityKind, EntityRef, RefRole, RefSite, default_agent_refs, ref_sites from agent_env.entity.scopes import FieldScope, prune from agent_env.env.env import Env @@ -67,7 +67,6 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import pin_digest from agent_env.task.task import Task -from agent_env.task_step.context import renamed_context_path from agent_env.task_step.registry import get_task_step_registry from agent_env.utils.paths import os_reason @@ -214,6 +213,7 @@ def add_task(self, task: Task) -> str: name = self._name("tasks", task.id) path = f"tasks/{name}.json" steps = task.to_dict()["steps"] + stored = copy.deepcopy(steps) registry = get_task_step_registry() found = [] for step in steps: @@ -240,7 +240,7 @@ def add_task(self, task: Task) -> str: if _key(site) not in outputs: # the run writes an output again, so it's no entity to write here self._ref(site, f"{where}: {site.path}", pins) self._default_agents(step, cls.entity_refs or (), where) - _rename_context_paths(found, sites) + sync_context_paths(stored, steps) self.files[path] = json.dumps(steps, indent=2, ensure_ascii=False, allow_nan=False) + "\n" self.tasks.append(ImportedTask(path, task.id, task.version, len(steps))) self.declared += task.requires or () @@ -674,15 +674,3 @@ def _key(site: RefSite) -> tuple[EntityKind, str]: def _nfc(text: str) -> str: return unicodedata.normalize("NFC", text) - - -def _rename_context_paths(found: list[tuple[dict, type, Any, list[RefSite]]], sites: list[RefSite]) -> None: - """Rename, in the paths into the run's context each step's type declares, each id a field of the task was - rewritten from to what it names in the folder, so a path names an entity as its field does.""" - renamed = {site.value: site.owner[site.key] for site in sites - if isinstance(site.owner[site.key], str) and site.owner[site.key] != site.value} - for step, cls, *_ in found: - for path in cls.context_paths: - for _, holder, key in field_sites(step, path): - if isinstance(holder[key], str): - holder[key] = renamed_context_path(holder[key], renamed.get) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 948aeee16..55162e2e4 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -70,7 +70,7 @@ from .parse import NAMED_BY, Bundle, BundleEntry, BundleError, BundleKind, parse_bundle from .variants import Varied from .resolve import (BuiltImage, BundleImage, Reference, ResolvedBundle, ResolvedEntry, build_step, entity_class, - resolve_bundle) + resolve_bundle, sync_context_paths) if TYPE_CHECKING: from .installed import InstalledBundle @@ -1255,6 +1255,8 @@ def _renamed(entry: ResolvedEntry, ids: Mapping[tuple[EntityKind, str], str]) -> for site in sites: if site.ref.role is RefRole.INPUT and (site.ref.kind, site.value) in ids: site.rewrite(ids[site.ref.kind, site.value]) + if entry.entry.kind is BundleKind.TASK: + sync_context_paths(entry.entry.config, config) return config diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index c0944eaad..e409fbcc8 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -3,8 +3,9 @@ References are the fields step types declare in ``entity_refs`` and the toml keys entity types declare in ``toml_refs``. The name of one of this bundle's entities becomes that entity's id, a name an output of the same task writes becomes the output's id, and anything else is a store id, left as written; -whether store ids exist is checked later, with the other checks that read a store. An id renamed so is -also renamed in each path into the run's context a step type declares in ``context_paths``. +whether store ids exist is checked later, with the other checks that read a store. An env renamed so is +also renamed where a path into the run's context a step type declares in ``context_paths`` names it +(``sync_context_paths``). """ from __future__ import annotations @@ -31,7 +32,7 @@ from agent_env.plugins._requirements import unmet_problems from agent_env.store.ids import HF_PREFIX, derive_id, is_local_id, validate_local_id from agent_env.store.image_store.oci_registry_credentials import names_registry -from agent_env.task_step.context import context_path_segments, renamed_context_path +from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies @@ -264,9 +265,8 @@ def _task(self, entry: BundleEntry) -> ResolvedEntry: continue self._resolve(entry, site, f"{where}{site.path}", site.value, site.version, outputs, functools.partial(_upstream, edges, index), references) - renamed = _rewritten_ids(entry.config, steps) - for _, step, cls in declared: - self._rename_context_paths(entry, step, cls, renamed) + sync_context_paths(entry.config, steps) + self._undeclared_paths(entry, declared, _rewritten_ids(entry.config, steps, EntityKind.ENV)) return ResolvedEntry(entry, steps, tuple(references), tuple(outputs.values())) def _image_folder(self, entry: BundleEntry) -> bool: @@ -493,19 +493,17 @@ def _undeclared(self, entry: BundleEntry, step: dict, cls: type, outputs: dict[t self._problem(entry, f"{where}{path} = {value!r} names {named}, but {cls.type} doesn't declare its " "entity_refs, so it can't be rewritten") - def _rename_context_paths(self, entry: BundleEntry, step: dict, cls: type, renamed: dict[str, str]) -> None: - """Rename each id ``renamed`` holds in the paths into the run's context ``cls`` declares (``context_paths``). - Warn of a path naming one in a field ``cls`` doesn't declare, which the run would look up as written.""" - for path in cls.context_paths: - for _, holder, key in field_sites(step, path): - if isinstance(holder[key], str): - holder[key] = renamed_context_path(holder[key], renamed.get) - for where, value in _strings(step): - segments = context_path_segments(value) or [] - if named := next((segment for segment in segments[1:] if segment in renamed), None): - logger.warning("%s: step %r: %s = %r names %r, which this bundle writes as %r, but %s doesn't declare " - "the field in context_paths, so it isn't rewritten", relative(self.bundle.root, entry.path), - step.get("id"), where, value, named, renamed[named], cls.type) + def _undeclared_paths(self, entry: BundleEntry, declared: list[tuple[int, dict, type]], + renamed: dict[str, str]) -> None: + """Warn of a path naming a renamed env in a field its step's type doesn't declare (``context_paths``), which + the run would look up as written.""" + for _, step, cls in declared: + for where, value in _strings(step): + if (env := _named_env(value)) in renamed: + logger.warning("%s: step %r: %s = %r names %r, which this bundle writes as %r, but %s doesn't " + "declare the field in context_paths, so it isn't rewritten", + relative(self.bundle.root, entry.path), step.get("id"), where, value, env, + renamed[env], cls.type) def _problem(self, entry: BundleEntry, message: str) -> None: self.problems.append(f"{relative(self.bundle.root, entry.path)}: {message}") @@ -538,6 +536,25 @@ def _registry_problem(key: str, name: Any) -> str | None: return f"{key}: {name!r} doesn't name its registry; spell it out, as in docker.io/library/{name}" +def sync_context_paths(written: list[dict], steps: list[dict]) -> None: + """Point each path into the run's context a step of ``steps`` declares (``TaskStep.context_paths``) at the env its + task's env fields name now, rebuilt from the path ``written``, the task as written, gives it. Only a segment that + names an env changes, so a key that just shares an entity's name, such as a verifier id, is kept. Each place that + rewrites a task's env fields calls it after, and calling it again changes nothing.""" + renamed = _rewritten_ids(written, steps, EntityKind.ENV) + registry = get_task_step_registry() + authored = {step.get("id"): step for step in written} + for step in steps: + cls = registry.get(step.get("type")) if isinstance(step.get("type"), str) else None + if cls is None or (before := authored.get(step.get("id"))) is None: + continue + for path in cls.context_paths: + was = {where: holder[key] for where, holder, key in field_sites(before, path)} + for where, holder, key in field_sites(step, path): + if isinstance(was.get(where), str): + holder[key] = _env_renamed(was[where], renamed) + + def build_step(step: dict) -> TaskStep: """The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes never changes.""" @@ -594,16 +611,31 @@ def _strings(value: Any, path: str = "") -> Iterator[tuple[str, str]]: yield from _strings(item, f"{path}[{i}]") -def _rewritten_ids(written: list[dict], resolved: list[dict]) -> dict[str, str]: - """Each id a reference field of ``written``'s steps holds that the same field of ``resolved``'s holds another id - in place of, to that id.""" +def _rewritten_ids(written: list[dict], resolved: list[dict], kind: EntityKind | None = None) -> dict[str, str]: + """Each id a reference field of ``written``'s steps holds, of ``kind`` when given, that the same field of + ``resolved``'s holds another id in place of, to that id.""" registry = get_task_step_registry() renamed = {} for before, after in zip(written, resolved): cls = registry.get(before.get("type")) if cls is None or not cls.entity_refs: continue - now = {site.path: site.value for site in ref_sites(cls.entity_refs, after)} - renamed.update((site.value, now[site.path]) for site in ref_sites(cls.entity_refs, before) + refs = [ref for ref in cls.entity_refs if kind is None or ref.kind is kind] + now = {site.path: site.value for site in ref_sites(refs, after)} + renamed.update((site.value, now[site.path]) for site in ref_sites(refs, before) if site.path in now and now[site.path] != site.value) return renamed + + +def _named_env(path: Any) -> str | None: + """The env a path into a run's context names: the segment after a key whose entries are keyed by env id.""" + segments = context_path_segments(path) or [] + return segments[1] if len(segments) > 1 and segments[0] in ENV_KEYED_METADATA else None + + +def _env_renamed(path: str, renamed: dict[str, str]) -> str: + """``path`` naming the env ``renamed`` gives its env segment's id for by that id.""" + if (env := _named_env(path)) not in renamed: + return path + segments = context_path_segments(path) + return context_path([segments[0], renamed[env], *segments[2:]]) diff --git a/src/agent_env/bundle/variants.py b/src/agent_env/bundle/variants.py index ce13272d8..da226a174 100644 --- a/src/agent_env/bundle/variants.py +++ b/src/agent_env/bundle/variants.py @@ -15,7 +15,7 @@ from agent_env.task_step.registry import get_task_step_registry from .parse import Bundle, BundleEntry, BundleError, BundleKind, segment_problem -from .resolve import entity_class +from .resolve import entity_class, sync_context_paths _TARGET = f"{BundleKind.ENV.value}/" _FORM = "envs/NAME.KEY=VALUE, such as envs/desktop.driver=cua-driver" @@ -187,8 +187,10 @@ def _renamed(entry: BundleEntry, new_id: str, config: Any) -> BundleEntry: def _repointed(entry: BundleEntry, config: Any, renaming: dict[EntityKind, dict[str, str]]) -> Any: """``config``, ``entry``'s steps or toml, with each env or task it names by an id that changes named by its variant's id.""" - config = copy.deepcopy(config) + written, config = config, copy.deepcopy(config) for site in _sites(entry, config): if site.value in renaming.get(site.ref.kind, {}): site.rewrite(renaming[site.ref.kind][site.value], site.version) + if entry.kind is BundleKind.TASK: + sync_context_paths(written, config) return config diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index b92d0fa58..9483449bc 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -17,6 +17,8 @@ HEARTBEAT_KEY = "_heartbeat_fn" # context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run. ENTITY_IDS_KEY = "entity_ids" +# The context.metadata keys whose entries are keyed by a deployed env's id, so a path into one names an env next. +ENV_KEYED_METADATA = frozenset({"env_trigger_state", "env_changes", "env_trajectory", "live_websites"}) def run_entity_id(metadata: dict, entity_id: str) -> str: @@ -32,14 +34,17 @@ def context_path_segments(path: Any) -> list[str] | None: return [plain or bracketed for plain, bracketed in _CONTEXT_PATH_SEGMENT.findall(path)] +def context_path(segments: list[str]) -> str: + """The path into a run's context that walks ``segments``: each that a dotted path can't hold in brackets.""" + return "".join(_path_segment(segment, first=i == 0) for i, segment in enumerate(segments)) + + def renamed_context_path(path: str, rename: Callable[[str], str | None]) -> str: """``path`` with each segment ``rename`` gives another id for naming that id; ``path`` itself when it isn't a path into a run's context or ``rename`` renames none of its segments.""" segments = context_path_segments(path) or [] renamed = [rename(segment) or segment for segment in segments] - if renamed == segments: - return path - return "".join(_path_segment(segment, first=i == 0) for i, segment in enumerate(renamed)) + return path if renamed == segments else context_path(renamed) class StepContextError(RuntimeError): diff --git a/tst/unit/bundle/installed_entities_test.py b/tst/unit/bundle/installed_entities_test.py index 88d8e6dcf..1f9248e4a 100644 --- a/tst/unit/bundle/installed_entities_test.py +++ b/tst/unit/bundle/installed_entities_test.py @@ -420,6 +420,15 @@ def test_an_env_named_by_an_installed_alias_is_pulled_in_and_the_task_names_it_b assert plan.alias_uses[EntityKind.AGENT] == {} and plan.store_refs == () +def test_a_context_fact_naming_an_env_by_an_installed_alias_names_the_env_the_alias_names(bundle_dir, tmp_path): + facts = {"wf-1": "env_trigger_state.crm-x.statuses.wf-1", "verdict": "verifications.env.score"} + judge = {**JUDGE, "id": "judge", "context_facts_for_judge": facts} + + plan = _plan(_task(bundle_dir, [*CRM_STEPS, judge]), _kit(tmp_path, ENV_ALIASED)) + + assert _step(plan, "judge")["context_facts_for_judge"] == {**facts, "wf-1": f"env_trigger_state[{CRM}].statuses.wf-1"} + + def test_the_stores_env_of_an_alias_id_shadows_the_alias(bundle_dir, tmp_path): kit = _kit(tmp_path, ENV_ALIASED) with namespace_routing(): diff --git a/tst/unit/bundle/ledger_test.py b/tst/unit/bundle/ledger_test.py index ff8f8ad71..255bf886b 100644 --- a/tst/unit/bundle/ledger_test.py +++ b/tst/unit/bundle/ledger_test.py @@ -312,7 +312,7 @@ def test_a_task_written_before_its_context_paths_named_the_bundle_s_ids_is_writt "prompt_id": "p1", "context_facts_for_judge": {"wf-1": "env_trigger_state.tickets.statuses.wf-1"}} (bundle_dir / "tasks" / "t.json").write_text(_steps(judge)) with monkeypatch.context() as earlier: - earlier.setattr(resolve_module._Resolver, "_rename_context_paths", lambda *args: None) + earlier.setattr(resolve_module, "sync_context_paths", lambda *args: None) _run(bundle_dir) checks = _run(bundle_dir) diff --git a/tst/unit/bundle/register_test.py b/tst/unit/bundle/register_test.py index 80341f3c3..537136b4a 100644 --- a/tst/unit/bundle/register_test.py +++ b/tst/unit/bundle/register_test.py @@ -323,14 +323,12 @@ def test_an_at_local_id_left_in_a_path_into_the_run_s_context_is_refused(home): "tasks/t.json: [0].metadata.from names '@local/~/elsewhere/x', which only this machine's stores hold",) -def test_a_context_fact_naming_a_bundle_entity_names_its_registry_id(home): - judge = {"id": "judge", "type": "rubrics_verifier", "criteria": [{"id": "c", "description": "d"}], - "prompt_id": "p1", "context_facts_for_judge": {"greeting": "loaded.greeting.version"}} - steps = [BOX, _load("greet", "greeting"), _load("docs", "docs"), judge] +def test_an_at_local_id_in_a_path_into_the_run_s_context_takes_its_registry_id(): + facts = {"wf-1": "env_trigger_state[@local/~/triage/crm].statuses.wf-1", "score": "verifications.judge.score"} - register_bundle(layout(home / "triage", {**TRIAGE, "tasks/t.json": json.dumps(steps)})) + renamed = register_module._renamed({"context_facts_for_judge": facts}, {"@local/~/triage/crm": "triage-crm"}) - assert _steps("triage-t")[-1]["context_facts_for_judge"] == {"greeting": "loaded.triage-greeting.version"} + assert renamed == {"context_facts_for_judge": {**facts, "wf-1": "env_trigger_state.triage-crm.statuses.wf-1"}} def test_a_shared_store_refuses_a_credential_or_reference_in_env_vars_and_a_url_only_this_machine_reaches( diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index b729dd33c..3572b81c0 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -169,18 +169,20 @@ def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make): assert swapped.entity_ids == entry.entity_ids == {"tickets": f"{ROOT}/tickets"} -def test_a_context_fact_naming_what_the_bundle_renamed_names_it_by_its_new_id(make): - facts = {"wf-1": "env_trigger_state.tickets.statuses.wf-1", "cli": "env_changes.tickets-cli.rows", - "store env": "env_trigger_state.elsewhere.statuses.wf-1", "score": "verifications.judge.score"} - entry = resolved(make(tasks={"t": [ - {"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, - {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, - {"id": "judge", "type": "rubrics_verifier", "context_facts_for_judge": facts}, - ]}), "t") - - assert entry.config[2]["context_facts_for_judge"] == { - "wf-1": f"env_trigger_state[{ROOT}/tickets].statuses.wf-1", "cli": f"env_changes[{ROOT}/t/tickets-cli].rows", - "store env": "env_trigger_state.elsewhere.statuses.wf-1", "score": "verifications.judge.score"} +def test_a_context_fact_naming_a_bundle_env_names_it_by_its_new_id_and_keeps_any_other_key(make): + facts = {"wf-1": "env_trigger_state.tickets.statuses.wf-1", "store env": "env_trigger_state.elsewhere.statuses.wf-1", + "verifier named as the agent": "verifications.solver.score", "an output": "env_changes.tickets-cli.rows"} + steps = [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, + {"id": "agent", "type": "deploy_agent", "env_ids": ["tickets"], "a2a_agent_id": "solver"}, + {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, + {"id": "judge", "type": "rubrics_verifier", "context_facts_for_judge": facts}] + entry = resolved(make(tasks={"t": steps}), "t") + + assert entry.config[3]["context_facts_for_judge"] == { + **facts, "wf-1": f"env_trigger_state[{ROOT}/tickets].statuses.wf-1"} + again = json.loads(json.dumps(entry.config)) + resolve_module.sync_context_paths(steps, again) + assert again == entry.config def test_a_path_naming_what_the_bundle_renamed_in_a_field_its_step_doesnt_declare_is_warned_of(make, caplog): diff --git a/tst/unit/bundle/variants_test.py b/tst/unit/bundle/variants_test.py index adb6159e3..af68396ed 100644 --- a/tst/unit/bundle/variants_test.py +++ b/tst/unit/bundle/variants_test.py @@ -96,6 +96,18 @@ def test_a_reference_by_id_is_repointed_at_the_variant(tmp_path): assert _entry(bundle, "desktop").config["tools"] == "standard" +def test_a_context_fact_naming_the_env_by_id_is_repointed_at_the_variant(tmp_path): + judge = {"id": "judge", "type": "rubrics_verifier", "prompt_id": "ask", "verifier_id": "v", + "criteria": [{"id": "c", "description": "ok"}], + "context_facts_for_judge": {"wf-1": f"env_trigger_state[{ROOT}/desktop].statuses.wf-1"}} + steps = [{**DEPLOY, "env_id": f"{ROOT}/desktop"}, {**AGENT, "env_ids": [f"{ROOT}/desktop"]}, PROMPT, judge] + + bundle = _varied(_bundle(tmp_path / "desk", steps), "envs/desktop.driver=cua-driver") + + assert _entry(bundle, "t").config[3]["context_facts_for_judge"] == { + "wf-1": f"env_trigger_state[{ROOT}/desktop(desktop.driver cua-driver)].statuses.wf-1"} + + @pytest.mark.parametrize("named", ["t", f"{ROOT}/t"], ids=["name", "id"]) def test_an_eval_naming_its_task_in_a_table_is_renamed_and_repointed_too(tmp_path, named): root = _bundle(tmp_path / "desk", files={"evals/e.toml": f'tasks = [{{ task = "{named}" }}]\n'}) From 299681fc15f3300392d9d29382fcf51fc5e5b3a5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 12 Nov 2026 16:01:41 -0800 Subject: [PATCH 4/4] fix(bundle): env_skills paths name their env, and an import never reads a path back from before masking - ENV_KEYED_METADATA records where each key's entries name an env: the segment after env_trigger_state, env_changes, env_trajectory and live_websites, and env_skills' third segment, after the agent's name. - A path is written only when its env is renamed. - An import leaves out scoped fields and masks credentials before it keeps the steps a path is rebuilt from, so a credential masked in a path stays masked when the path's env is renamed. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/importer.py | 13 +++++++------ src/agent_env/bundle/resolve.py | 25 ++++++++++++++++--------- src/agent_env/task_step/context.py | 6 ++++-- tst/unit/bundle/importer_test.py | 15 +++++++++++++++ tst/unit/bundle/resolve_test.py | 6 ++++-- 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 8fd33967d..562e30b47 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -213,14 +213,15 @@ def add_task(self, task: Task) -> str: name = self._name("tasks", task.id) path = f"tasks/{name}.json" steps = task.to_dict()["steps"] - stored = copy.deepcopy(steps) registry = get_task_step_registry() found = [] for step in steps: cls = registry[step["type"]] - pruned = prune(step, cls.field_scopes, self.include_scopes) # first, so no pruned field is written - found.append((step, cls, pruned, [site for site in ref_sites(cls.entity_refs or (), step) - if isinstance(site.value, str) and site.value])) + # First, so no pruned field or secret is written, nor read back when paths are synced below. + dropped = prune(step, cls.field_scopes, self.include_scopes) + scrub(step) + found.append((step, cls, dropped, [site for site in ref_sites(cls.entity_refs or (), step) + if isinstance(site.value, str) and site.value])) + stored = copy.deepcopy(steps) # as the store holds it, less what's left out, before any id is renamed sites = [site for *_, step_sites in found for site in step_sites] outputs = _output_names(sites) for site in sites: # each output, and each step that reads it, takes the output's name @@ -230,9 +231,9 @@ def add_task(self, task: Task) -> str: for site in sites: if site.ref.kind is EntityKind.ENV and site.version is not None: pins.setdefault(_key(site)[1], set()).add(site.version) - for step, cls, pruned, step_sites in found: + for step, cls, dropped, step_sites in found: where = f"{path}: step {step.get('id')!r}" - self.dropped += [Dropped(f"{where}: {at}", why) for at, why in pruned + scrub(step)] + self.dropped += [Dropped(f"{where}: {at}", why) for at, why in dropped] self._need(where, "step type", TASK_STEPS, cls.type, cls) if cls.entity_refs is None: self.undeclared.append(f"{where} ({cls.type})") diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index e409fbcc8..cd1f06b60 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -538,9 +538,10 @@ def _registry_problem(key: str, name: Any) -> str | None: def sync_context_paths(written: list[dict], steps: list[dict]) -> None: """Point each path into the run's context a step of ``steps`` declares (``TaskStep.context_paths``) at the env its - task's env fields name now, rebuilt from the path ``written``, the task as written, gives it. Only a segment that - names an env changes, so a key that just shares an entity's name, such as a verifier id, is kept. Each place that - rewrites a task's env fields calls it after, and calling it again changes nothing.""" + task's env fields name now, rebuilt from the path ``written``, the task as written, gives it. Only a path whose env + is renamed is written, and only its env segment changes, so a key that just shares an entity's name, such as a + verifier id, is kept. Each place that rewrites a task's env fields calls it after, and calling it again changes + nothing.""" renamed = _rewritten_ids(written, steps, EntityKind.ENV) registry = get_task_step_registry() authored = {step.get("id"): step for step in written} @@ -551,7 +552,7 @@ def sync_context_paths(written: list[dict], steps: list[dict]) -> None: for path in cls.context_paths: was = {where: holder[key] for where, holder, key in field_sites(before, path)} for where, holder, key in field_sites(step, path): - if isinstance(was.get(where), str): + if _named_env(was.get(where)) in renamed: holder[key] = _env_renamed(was[where], renamed) @@ -627,15 +628,21 @@ def _rewritten_ids(written: list[dict], resolved: list[dict], kind: EntityKind | return renamed -def _named_env(path: Any) -> str | None: - """The env a path into a run's context names: the segment after a key whose entries are keyed by env id.""" +def _env_segment(path: Any) -> tuple[list[str], int] | None: + """A path into a run's context that names an env, and the index of its segment that does (``ENV_KEYED_METADATA``).""" segments = context_path_segments(path) or [] - return segments[1] if len(segments) > 1 and segments[0] in ENV_KEYED_METADATA else None + index = ENV_KEYED_METADATA.get(segments[0]) if segments else None + return (segments, index) if index is not None and index < len(segments) else None + + +def _named_env(path: Any) -> str | None: + found = _env_segment(path) + return found[0][found[1]] if found else None def _env_renamed(path: str, renamed: dict[str, str]) -> str: """``path`` naming the env ``renamed`` gives its env segment's id for by that id.""" if (env := _named_env(path)) not in renamed: return path - segments = context_path_segments(path) - return context_path([segments[0], renamed[env], *segments[2:]]) + segments, index = _env_segment(path) + return context_path([*segments[:index], renamed[env], *segments[index + 1:]]) diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 9483449bc..df454be3a 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -17,8 +17,10 @@ HEARTBEAT_KEY = "_heartbeat_fn" # context.metadata[ENTITY_IDS_KEY]: the run's task's Task.entity_ids, set by Task.run. ENTITY_IDS_KEY = "entity_ids" -# The context.metadata keys whose entries are keyed by a deployed env's id, so a path into one names an env next. -ENV_KEYED_METADATA = frozenset({"env_trigger_state", "env_changes", "env_trajectory", "live_websites"}) +# The context.metadata keys whose entries name a deployed env by its id, each with the index of the path segment that +# names it: env_skills is keyed by agent name, then env id. +ENV_KEYED_METADATA = {"env_trigger_state": 1, "env_changes": 1, "env_trajectory": 1, "live_websites": 1, + "env_skills": 2} def run_entity_id(metadata: dict, entity_id: str) -> str: diff --git a/tst/unit/bundle/importer_test.py b/tst/unit/bundle/importer_test.py index b892972b2..f2a2e0789 100644 --- a/tst/unit/bundle/importer_test.py +++ b/tst/unit/bundle/importer_test.py @@ -235,6 +235,21 @@ def test_a_context_fact_names_an_entity_as_its_folder_does_when_the_folder_isnt_ "other": "env_trigger_state.crm-env-2.statuses.wf-1"} +def test_a_credential_masked_in_a_context_path_stays_masked_when_the_paths_env_is_renamed(store): + other, old = "@local/~/elsewhere/crm-env", "@local/~/old/triage" + leak = f"env_trigger_state[{other}][https://user:hunter2@wiki.example.com/x]" + judge = {**STEPS[4], "context_facts_for_judge": {"leak": leak}} + with namespace_routing(): + MCPServerEnv.put(id=other, docker_image_artifact=context_image(store, "crm-image"), environment_name="crm") + _task(old, [STEPS[0], {**STEPS[0], "id": "env2", "env_id": other}, STEPS[2], judge]) + imported = import_task(old, store / "again") + + written = (imported.root / "tasks" / "triage.json").read_text() + assert "hunter2" not in written + assert _steps(imported.root, "triage")[3]["context_facts_for_judge"]["leak"].startswith( + "env_trigger_state.crm-env-2[") + + def test_a_renamed_output_is_numbered_when_another_output_or_a_store_reference_has_its_name(store): old = "@local/~/old/triage" with namespace_routing(): diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 3572b81c0..66880b98a 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -171,7 +171,8 @@ def test_a_reference_added_without_rewriting_a_field_is_not_a_renamed_id(make): def test_a_context_fact_naming_a_bundle_env_names_it_by_its_new_id_and_keeps_any_other_key(make): facts = {"wf-1": "env_trigger_state.tickets.statuses.wf-1", "store env": "env_trigger_state.elsewhere.statuses.wf-1", - "verifier named as the agent": "verifications.solver.score", "an output": "env_changes.tickets-cli.rows"} + "verifier named as the agent": "verifications.solver.score", "an output": "env_changes.tickets-cli.rows", + "skill": "env_skills.default-agent.tickets.crm", "agent named as the env": "env_skills.tickets.other.crm"} steps = [{"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, {"id": "agent", "type": "deploy_agent", "env_ids": ["tickets"], "a2a_agent_id": "solver"}, {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "cli_artifact_id": "tickets-cli"}, @@ -179,7 +180,8 @@ def test_a_context_fact_naming_a_bundle_env_names_it_by_its_new_id_and_keeps_any entry = resolved(make(tasks={"t": steps}), "t") assert entry.config[3]["context_facts_for_judge"] == { - **facts, "wf-1": f"env_trigger_state[{ROOT}/tickets].statuses.wf-1"} + **facts, "wf-1": f"env_trigger_state[{ROOT}/tickets].statuses.wf-1", + "skill": f"env_skills.default-agent[{ROOT}/tickets].crm"} again = json.loads(json.dumps(entry.config)) resolve_module.sync_context_paths(steps, again) assert again == entry.config