diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index 948aeee16..89aba28d2 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -63,7 +63,7 @@ from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from ._fs import fold, relative, with_article from .authoring import AuthoringContext, entry_files, staged_build_context @@ -1219,11 +1219,10 @@ def folder_files(bundle: Bundle, entry: BundleEntry) -> Mapping[str, Path]: def solver_steps(steps: Any) -> tuple[str, ...]: - """The ids of the steps in ``steps``, a task's config, that deploy or install an agent a ``prompt_agent`` step - prompts, and not as its user simulator: the solver's.""" - prompts = [step for step in steps if isinstance(step, dict) and step.get("type") == PromptAgentTaskStep.type] - prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts} - prompted -= {step.get("user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME for step in prompts} + """The ids of the steps in ``steps``, a task's config, that deploy or install an agent a step prompts as the agent + under test (``is_prompt_step``), and not as its user simulator: the solver's.""" + prompts = [step for step in steps if isinstance(step, dict) and is_prompt_step(step)] + prompted = {step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME for step in prompts} - user_sim_agents(steps) return tuple(step["id"] for step in steps if isinstance(step, dict) and isinstance(step.get("id"), str) and step.get("type") in (DeployAgentTaskStep.type, InstallAgentTaskStep.type) and (step.get("agent_name") or TaskStep.DEFAULT_AGENT_NAME) in prompted) diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 686330e62..f43547bdb 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -30,6 +30,7 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies +from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from ._fs import fold, relative, show from .authoring import AuthoringContext, default_dockerfile @@ -125,11 +126,12 @@ def resolve_bundle(bundle: Bundle) -> ResolvedBundle: def played_deploys(entry: ResolvedEntry) -> frozenset[str]: - """The ``deploy_agent`` steps of a task's only prompted agent, which a caller playing that agent itself - (``EpisodeSpec``) stands in for; none when the entry isn't a task, or prompts no agent or several.""" + """The ``deploy_agent`` steps of a task's only agent under test, the one a step prompts as it and no step talks to + as its user (``user_sim_agents``), which a caller playing that agent itself (``EpisodeSpec``) stands in for; none + when the entry isn't a task, or prompts no agent or several.""" if entry.entry.kind is not BundleKind.TASK: return frozenset() - prompted = {_agent_name(step) for step in entry.config if step.get("type") == "prompt_agent"} + prompted = {_agent_name(step) for step in entry.config if is_prompt_step(step)} - user_sim_agents(entry.config) if len(prompted) != 1: return frozenset() return frozenset(step.get("id") for step in entry.config diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index c29418b48..41753d304 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -30,6 +30,7 @@ from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.context_ops import build_context_update_ops from agent_env.task_step.task_steps.collect_artifacts import CollectArtifactsTaskStep +from agent_env.task_step.task_steps.prompt_agent import is_prompt_step def _indent(text: str, prefix: str = " ") -> str: @@ -176,8 +177,11 @@ def _step_output_lines(step, context) -> list[str]: if match: lines.append(_format_loaded_snapshot(match)) return lines - if step.type == "prompt_agent" and context.prompt_responses: - return [_format_prompt_response(context.prompt_responses[-1])] + if is_prompt_step(step): + # This step's own response: a step running alongside it may have added a later one. + match = next((r for r in reversed(context.prompt_responses) if r.step_id == step.id), None) + if match: + return [_format_prompt_response(match)] if step.type == "snapshot_agent_state": snapshots = context.metadata.get("agent_snapshots") or [] match = next((s for s in reversed(snapshots) if s.get("capture_step_id") == step.id), None) @@ -359,7 +363,8 @@ def _stored(task, context: TaskStepContext) -> str: @click.option("--litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY for this run; a secret:KEY or env:NAME reference is resolved where it is used") @click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used") @click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)") -@click.option("--a2a-agent-id", default=None, type=str, help="Override A2A agent id for deploy_agent steps") +@click.option("--a2a-agent-id", default=None, type=str, + help="Override A2A agent id for deploy_agent steps; a user sim's agent keeps its own") @click.option("--agent-sandbox", default=None, help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')") @click.option("--env-sandbox", default=None, diff --git a/src/agent_env/task/episode/spec.py b/src/agent_env/task/episode/spec.py index 2eafc412b..0b9dc6593 100644 --- a/src/agent_env/task/episode/spec.py +++ b/src/agent_env/task/episode/spec.py @@ -11,7 +11,7 @@ from agent_env.task.task import Task, substitute_seed_in_steps from agent_env.task_step.context import TaskStepContext, sandbox_overrides from agent_env.task_step.task_step import TaskStep -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, is_prompt_step class EpisodeSpec: @@ -74,7 +74,7 @@ async def start(self, *, sandbox: str | None = None) -> Episode: def _trainer_agents(task: Task, steps: list[TaskStep], named: Sequence[str] | None) -> list[str]: - prompted = list(dict.fromkeys(step.agent_name for step in steps if step.type == "prompt_agent")) + prompted = list(dict.fromkeys(step.agent_name for step in steps if is_prompt_step(step))) if named is None: if len(prompted) != 1: raise EpisodeError( diff --git a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py index eca3c1dc0..5b222fbd8 100644 --- a/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py +++ b/src/agent_env/task_step/task_steps/agent/deploy_user_sim_agent.py @@ -168,7 +168,6 @@ class DeployUserSimAgentTaskStep(DeployAgentTaskStep): ``agent_name`` in ``metadata["user_sims"]``.""" type: ClassVar[str] = "deploy_user_sim_agent" - plays_user: ClassVar[bool] = True system_prompt_required: ClassVar[bool] = True entity_refs = DeployAgentTaskStep.entity_refs # Which agent plays the user is part of the task, not the user's setup: written out for elsewhere, the task keeps it. diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index bba9f0995..8098f1a4b 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -1350,3 +1350,25 @@ def _trigger_answer(resp: dict) -> Optional[dict]: if resp.get("fired"): return {"message": resp.get("parts") or []} return None + + +def is_prompt_step(step: Any) -> bool: + """Whether ``step`` prompts an agent: a prompt_agent step or a subclass of one, such as always_on. A step dict, or + anything else with a ``type``, is looked up by that type in the step registry.""" + if isinstance(step, PromptAgentTaskStep): + return True + from agent_env.task_step.registry import get_task_step_registry # its built-ins import this module + + kind = step.get("type") if isinstance(step, dict) else getattr(step, "type", None) + cls = get_task_step_registry().get(kind) if isinstance(kind, str) else None + return isinstance(cls, type) and issubclass(cls, PromptAgentTaskStep) + + +def user_sim_agents(steps: Any) -> set[str]: + """The agents a task's prompting steps talk to as their user: an always_on step's ``user_sim``, a prompt_agent + step's ``user_agent_name`` (its default when unset). ``steps`` are steps or step dicts.""" + def field(step: Any, name: str) -> Any: + return step.get(name) if isinstance(step, dict) else getattr(step, name, None) + + return {field(step, "user_sim") or field(step, "user_agent_name") or PromptAgentTaskStep.DEFAULT_USER_AGENT_NAME + for step in steps if is_prompt_step(step)} diff --git a/src/agent_env/training/harness.py b/src/agent_env/training/harness.py index 807eddde0..00ca38851 100644 --- a/src/agent_env/training/harness.py +++ b/src/agent_env/training/harness.py @@ -36,6 +36,7 @@ from agent_env.task.teardown import teardown_run from agent_env.task_step.context import TaskStepContext, sandbox_overrides from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.training._openenv import on_session_loop @@ -231,7 +232,20 @@ def _index_named(prompt: Any, ids: list[str]) -> int: def _first_prompt(task: Task) -> str: - return next((step.prompt for step in task.steps if step.type == "prompt_agent"), "") + """The user's first message to the agent under test: its step's prompt, else the opening of the user sim the step + names (an always_on step's ``user_sim``, a prompt_agent step's ``user_agent_name``), as it was written. A step that + prompts an agent another step talks to as its user (the user sim, say a briefing) isn't the agent under test's.""" + users = user_sim_agents(task.steps) + step = next((each for each in task.steps + if is_prompt_step(each) and getattr(each, "agent_name", None) not in users), None) + if step is None: + return "" + if getattr(step, "prompt", None): + return step.prompt + user = getattr(step, "user_sim", None) or getattr(step, "user_agent_name", None) + sim = next((each for each in task.steps + if user and getattr(each, "agent_name", None) == user and getattr(each, "opening_prompt", None)), None) + return getattr(sim, "opening_prompt", None) or "" def _free_port() -> int: diff --git a/tst/unit/bundle/agent_override_test.py b/tst/unit/bundle/agent_override_test.py index f4c670791..5159db696 100644 --- a/tst/unit/bundle/agent_override_test.py +++ b/tst/unit/bundle/agent_override_test.py @@ -17,6 +17,7 @@ from agent_env.task.task import Task from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep +from agent_env.task_step.task_steps.prompt_agent import user_sim_agents from tst.unit.bundle._support import layout, store_agent ROOT = "@local/~/triage" @@ -119,6 +120,41 @@ def test_only_the_steps_deploying_or_installing_the_agent_a_step_prompts_are_the assert solver_steps(steps) == ("solver", "installed") +def test_the_agent_an_always_on_step_wakes_is_the_solver(): + steps = [ + {"id": "solver", "type": "deploy_agent"}, + {"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"}, + {"id": "loop", "type": "always_on", "user_sim": "linda"}, + ] + + assert solver_steps(steps) == ("solver",) + + +def test_the_agent_an_always_on_step_talks_to_as_its_user_is_never_the_solvers(): + """Even when a step prompts it, as a briefing would, and another step installs it.""" + steps = [ + {"id": "solver", "type": "deploy_agent"}, + {"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"}, + {"id": "installed", "type": "install_agent", "agent_name": "linda", "sandbox_name": "box"}, + {"id": "brief", "type": "prompt_agent", "agent_name": "linda"}, + {"id": "loop", "type": "always_on", "user_sim": "linda"}, + ] + + assert solver_steps(steps) == ("solver",) + assert user_sim_agents(steps) == {"linda", "human_agent"} + + +def test_a_prompt_agent_tasks_solver_and_user_agents_are_as_on_main(): + steps = [ + {"id": "solver", "type": "deploy_agent"}, + {"id": "user", "type": "deploy_agent", "agent_name": "human_agent"}, + {"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3}, + ] + + assert solver_steps(steps) == ("solver",) + assert user_sim_agents(steps) == {"human_agent"} + + def test_preflight_checks_the_agent_the_solver_deploys(bundle_dir, monkeypatch): checked = [] original = preflight_module._Walk._agent_deploy diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 824adb98a..218da2775 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -3,12 +3,13 @@ import json import random from dataclasses import replace +from types import SimpleNamespace import pytest from agent_env.bundle import BundleError, BundleKind, parse_bundle from agent_env.bundle import resolve as resolve_module -from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, resolve_bundle +from agent_env.bundle.resolve import BuiltImage, Output, Reference, ReferencedImage, played_deploys, resolve_bundle from agent_env.entity.refs import EntityKind, EntityRef from agent_env.env.env import Env from agent_env.task import Task @@ -594,3 +595,24 @@ def test_a_bad_depends_on_on_a_step_deriving_its_output_is_named_once(make, depe assert problems(make(tasks={"t": [ {"id": "cli", "type": "build_mcp_cli", "env_id": "tickets", "command_name": "c", "depends_on": depends_on}, ]})) == (f"tasks/t.json: step 'cli': {problem}",) + + +def test_the_agent_an_always_on_step_wakes_is_the_one_a_caller_plays(): + task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[ + {"id": "agent", "type": "deploy_agent"}, + {"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda"}, + {"id": "brief", "type": "prompt_agent", "agent_name": "linda"}, + {"id": "loop", "type": "always_on", "user_sim": "linda"}, + ]) + + assert played_deploys(task) == frozenset({"agent"}) + + +def test_a_prompt_agent_tasks_played_deploy_is_as_on_main(): + task = SimpleNamespace(entry=SimpleNamespace(kind=BundleKind.TASK), config=[ + {"id": "agent", "type": "deploy_agent"}, + {"id": "user", "type": "deploy_agent", "agent_name": "human_agent"}, + {"id": "ask", "type": "prompt_agent", "max_conversation_turns": 3}, + ]) + + assert played_deploys(task) == frozenset({"agent"}) diff --git a/tst/unit/cli/always_on_step_output_test.py b/tst/unit/cli/always_on_step_output_test.py new file mode 100644 index 000000000..a6d50b52a --- /dev/null +++ b/tst/unit/cli/always_on_step_output_test.py @@ -0,0 +1,30 @@ +"""`task run` shows an always_on step its response as it shows a prompt_agent step's.""" + +from __future__ import annotations + +from types import SimpleNamespace + +from agent_env.cli.task.run import _step_output_lines +from agent_env.task_step.context import PromptResponse, TaskStepContext + + +def test_an_always_on_steps_response_is_shown(): + context = TaskStepContext(prompt_responses=[PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop")]) + + [lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context) + + assert "prompt_id: loop" in lines and "Recap sent." in lines + + +def test_an_always_on_step_without_a_response_shows_nothing(): + assert _step_output_lines(SimpleNamespace(type="always_on", id="loop"), TaskStepContext()) == [] + + +def test_an_always_on_step_shows_its_own_response_when_a_parallel_step_added_a_later_one(): + context = TaskStepContext(prompt_responses=[ + PromptResponse(prompt_id="loop", response="Recap sent.", step_id="loop"), + PromptResponse(prompt_id="side", response="Side answer.", step_id="side")]) + + [lines] = _step_output_lines(SimpleNamespace(type="always_on", id="loop"), context) + + assert "Recap sent." in lines and "Side answer." not in lines diff --git a/tst/unit/cli/task_run_agent_pick_test.py b/tst/unit/cli/task_run_agent_pick_test.py new file mode 100644 index 000000000..c2b557162 --- /dev/null +++ b/tst/unit/cli/task_run_agent_pick_test.py @@ -0,0 +1,64 @@ +"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, at the deploy steps' stored versions, for +each step to apply (tst/unit/task_step/run_wide_agent_pick_test.py); a step that plays the user keeps its own agent by +its class, as it does for a hub run's pick.""" + +import importlib +import logging + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli, submitted_runs +from agent_env.runner import local_runner +from agent_env.task import Task +from agent_env.task.teardown import TeardownReport + +task_run = importlib.import_module("agent_env.cli.task.run") # the package's ``run`` is the click command +pytestmark = pytest.mark.usefixtures("sigint_handled", "local_stores") + +AGENT = {"id": "agent", "type": "deploy_agent", "env_ids": [], "a2a_agent_id": "theirs", "a2a_agent_version": 4} +USER = {"id": "user", "type": "deploy_agent", "env_ids": [], "agent_name": "human_agent", "a2a_agent_id": "sim"} +ASK = {"id": "ask", "type": "prompt_agent", "prompt_id": "ask", "prompt": "hi", "max_conversation_turns": 3} + + +class _Recorded: + """A stored task that records each context it runs with.""" + + id, version, requires = "t", 1, None + + def __init__(self, steps): + self.steps = Task.from_dict({"id": "t", "version": 1, "steps": steps}).steps + self.contexts = [] + + async def run(self, context, **_): + self.contexts.append(context) + return context + + +@pytest.fixture +def stored(monkeypatch): + async def teardown_run(context): + return TeardownReport() + + monkeypatch.setattr(task_run, "teardown_run", teardown_run) + monkeypatch.setattr(local_runner, "teardown_run", teardown_run) + monkeypatch.setattr(submitted_runs, "POLL_SECONDS", 0.02) + + def store(*steps): + recorded = _Recorded(list(steps)) + monkeypatch.setattr(Task, "get", classmethod(lambda cls, id, version=None: recorded)) + return recorded + + logging.disable(logging.CRITICAL) + yield store + logging.disable(logging.NOTSET) + + +def test_the_pick_stays_run_wide_with_a_user_agent_deployed(stored): + recorded = stored(AGENT, USER, ASK) + result = CliRunner().invoke(cli, ["task", "run", "--id", "t", "--a2a-agent-id", "mine"]) + assert result.exit_code == 0, result.output + + [context] = recorded.contexts + overrides = context.metadata["user_overrides"] + assert overrides["a2a_agent_id"] == "mine" and "step_params" not in overrides diff --git a/tst/unit/task/episode/spec_test.py b/tst/unit/task/episode/spec_test.py index c85d90a61..62641e54c 100644 --- a/tst/unit/task/episode/spec_test.py +++ b/tst/unit/task/episode/spec_test.py @@ -21,6 +21,9 @@ "verifier_id": "berth"} BOB = {"id": "bob", "type": "deploy_agent", "agent_name": "bob", "a2a_agent_id": "echo"} ASK_BOB = {"id": "ask-bob", "type": "prompt_agent", "agent_name": "bob", "prompt": "Check the plan."} +USER_SIM = {"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda", "model": "sim-model", + "multi_day": False, "opening_prompt": "Plan week .", + "persona": {"display_name": "Linda Murphy", "description": {"who_you_are": ["Runs the port's berths."]}}} SEED = {"week": "3", "port": "Valencia"} @@ -265,13 +268,41 @@ def test_steps_that_need_only_what_an_outside_player_leaves_are_allowed(replace) EpisodeSpec.from_task(_portsim(**replace)) def test_every_built_in_step_type_is_checked_or_unaffected(): - registered, checked = set(_builtin_registry()), set(CHECKS) + registry = _builtin_registry() + registered, checked = set(registry), set(CHECKS) assert sorted(registered - checked - UNAFFECTED) == [] assert sorted((checked | UNAFFECTED) - registered) == [] assert sorted(checked & UNAFFECTED) == [] +class _AlwaysOn(PromptAgentTaskStep): + """An always_on step as these rules read one: the agent it wakes, by name.""" + + type = "always_on" + + +def test_the_agent_an_always_on_step_wakes_is_the_one_played_and_is_refused_for_now(): + loop = _AlwaysOn.from_dict({**PLAY, "id": "loop"}) + task = Task(id="dock", version=4, steps=[*_portsim(play=None).steps, loop]) + + assert ("step 'loop': always_on needs 'default-agent' as a real agent, and it is played from outside" + in _refusal(task)) + + +def test_the_user_sims_deploy_is_refused_when_its_agent_is_played_from_outside(): + task = _portsim({**USER_SIM, "agent_name": "default-agent"}) + + assert ("step 'user': deploy_user_sim_agent deploys 'default-agent', which is played from outside" + in _refusal(task)) + + +def test_the_user_sims_deploy_with_its_own_agent_is_allowed(): + task = _portsim(USER_SIM) + + assert EpisodeSpec.from_task(task, seed=SEED).trainer_agents == ("default-agent",) + + def test_the_seed_pass_rebuilds_each_step_as_its_own_class(): class _Unregistered(PromptAgentTaskStep): pass diff --git a/tst/unit/task_step/run_wide_agent_pick_test.py b/tst/unit/task_step/run_wide_agent_pick_test.py new file mode 100644 index 000000000..eecdf6e3b --- /dev/null +++ b/tst/unit/task_step/run_wide_agent_pick_test.py @@ -0,0 +1,38 @@ +"""A run-wide agent pick (``user_overrides.a2a_agent_id``: ``task run --a2a-agent-id``, a hub run's agent pick) reaches +every ``deploy_agent`` and ``install_agent`` step, a ``prompt_agent`` task's user agent included, exactly as on main. +A ``deploy_user_sim_agent`` step keeps its own agent through its own ``_resolve_agent_ref``, not anything these steps +read.""" + +from __future__ import annotations + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep +from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep + +SOLVER = {"id": "deploy-solver", "type": "deploy_agent", "env_ids": [], "agent_name": "solver", + "a2a_agent_id": "theirs", "a2a_agent_version": 4} +HUMAN = {"id": "deploy-human", "type": "deploy_agent", "env_ids": [], "agent_name": "human_agent", + "a2a_agent_id": "sim", "a2a_agent_version": 2} + + +def _picked(**overrides) -> TaskStepContext: + return TaskStepContext(instance_id="run-1", metadata={"user_overrides": {"a2a_agent_id": "mine", **overrides}}) + + +def test_the_pick_reaches_a_prompt_agent_tasks_user_agent_too_at_each_steps_stored_version(): + ctx = _picked() + assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 4) + assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 2) + + +def test_a_picked_version_applies_to_every_deploy(): + ctx = _picked(a2a_agent_version=9) + assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 9) + assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 9) + + +def test_install_agent_takes_the_pick_at_its_stored_version(): + installed = InstallAgentTaskStep.from_dict({"id": "install-human", "type": "install_agent", + "agent_name": "human_agent", "a2a_agent_id": "sim", + "a2a_agent_version": 2, "sandbox_name": "world"}) + assert installed._agent_ref(_picked()) == ("mine", 2) diff --git a/tst/unit/task_step/test_deploy_user_sim_agent.py b/tst/unit/task_step/test_deploy_user_sim_agent.py index 2a25c50b0..2992a85ba 100644 --- a/tst/unit/task_step/test_deploy_user_sim_agent.py +++ b/tst/unit/task_step/test_deploy_user_sim_agent.py @@ -16,12 +16,10 @@ from agent_env.a2a_agent import A2AAgent from agent_env.a2a_agent import protocol from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep from agent_env.task_step.task_steps import always_on as ao from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY, DeployUserSimAgentTaskStep -from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep from tst.unit.task_step.always_on_harness import USER_SIM, Rig DATA = Path(__file__).parents[2] / "data" / "user_sim" @@ -184,12 +182,6 @@ def test_the_tested_schedule_loads(): # ---- it plays the user, never the agent under test ----------------------------------------------------------------- -def test_only_the_user_sim_step_plays_the_user(): - assert DeployUserSimAgentTaskStep.plays_user is True - assert not hasattr(TaskStep, "plays_user") and not hasattr(DeployAgentTaskStep, "plays_user") - assert not hasattr(PromptAgentTaskStep, "plays_user") and _step().plays_user is True - - @pytest.mark.asyncio async def test_its_agent_is_deployed_and_recorded_exactly_as_deploy_agent_deploys_one(monkeypatch): """Everything deploy_agent gives an agent, the user sim's gets: the same deployed_agents entry and the same posts, diff --git a/tst/unit/task_step/test_prompt_agent.py b/tst/unit/task_step/test_prompt_agent.py index 6f821ae72..7eb108e67 100644 --- a/tst/unit/task_step/test_prompt_agent.py +++ b/tst/unit/task_step/test_prompt_agent.py @@ -31,6 +31,7 @@ PromptAgentTaskStep, _duplicates_prompt_text, _parse_structured_output, + is_prompt_step, ) from tst.util.fake_a2a import FakeA2AAgents @@ -269,3 +270,15 @@ def trajectory(request: httpx.Request) -> httpx.Response: assert json.loads(store.get(uri)) == [{"type": "echo", "output": "Echo: hi"}] +def test_a_prompt_step_is_a_prompt_agent_or_a_subclass_whether_a_step_or_its_config(): + class _Unregistered(PromptAgentTaskStep): + type = "unregistered_prompt" + + assert is_prompt_step({"type": "prompt_agent"}) and is_prompt_step({"type": "always_on"}) + assert is_prompt_step(SimpleNamespace(type="always_on")) + assert is_prompt_step(_Unregistered(id="p", version=None, prompt="hi")) + assert not is_prompt_step({"type": "deploy_agent"}) + assert not is_prompt_step({"type": "deploy_user_sim_agent"}) + assert not is_prompt_step({"type": "no_such_step"}) and not is_prompt_step({}) + + diff --git a/tst/unit/training/harness_test.py b/tst/unit/training/harness_test.py index defddc230..d0999762e 100644 --- a/tst/unit/training/harness_test.py +++ b/tst/unit/training/harness_test.py @@ -8,6 +8,7 @@ import logging import threading from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace import httpx import pytest @@ -246,3 +247,46 @@ def eval_only(llm_url, model, **_): with pytest.raises(RuntimeError, match="eval-only"): AgentEnvSessionFactory(bundle, llm_url="http://127.0.0.1:9", model=MODEL) + + +def test_a_rows_prompt_is_an_always_on_steps_as_it_is_a_prompt_agent_steps(): + task = SimpleNamespace(steps=[SimpleNamespace(type="deploy_agent"), + SimpleNamespace(type="always_on", prompt="Watch Tess's shop this month.")]) + + assert harness._first_prompt(task) == "Watch Tess's shop this month." + + +def test_a_rows_prompt_with_no_step_prompt_is_the_named_user_sims_opening(): + def user_sim(name, opening): + return SimpleNamespace(type="deploy_user_sim_agent", agent_name=name, opening_prompt=opening) + + always_on = SimpleNamespace(type="always_on", prompt=None, user_sim="nora") + task = SimpleNamespace(steps=[user_sim("other", "Not this one."), user_sim("nora", "Find me a sofa."), always_on]) + assert harness._first_prompt(task) == "Find me a sofa." + + prompt_agent = SimpleNamespace(type="prompt_agent", prompt="", user_agent_name="nora") + assert harness._first_prompt(SimpleNamespace(steps=[user_sim("nora", "Find me a sofa."), prompt_agent])) \ + == "Find me a sofa." + + unnamed = SimpleNamespace(type="always_on", prompt=None, user_sim=None) + assert harness._first_prompt(SimpleNamespace(steps=[user_sim("nora", "Find me a sofa."), unnamed])) == "" + + +def test_a_rows_prompt_skips_a_briefing_sent_to_the_user_sim(): + sim = SimpleNamespace(type="deploy_user_sim_agent", agent_name="nora", opening_prompt="Find a sofa.") + briefing = SimpleNamespace(type="prompt_agent", agent_name="nora", prompt="You are Nora; stay in character.") + always_on = SimpleNamespace(type="always_on", agent_name="solver", prompt=None, user_sim="nora") + + assert harness._first_prompt(SimpleNamespace(steps=[sim, briefing, always_on])) == "Find a sofa." + + +def test_a_task_whose_user_sim_step_deploys_the_user_trains_only_its_solver(): + user = {"id": "user", "type": "deploy_user_sim_agent", "agent_name": "linda", "a2a_agent_id": "sim", + "model": "sim-model", "multi_day": False, "opening_prompt": "Help Linda.", + "persona": {"display_name": "Linda Murphy", "description": {"who_you_are": ["Plans her family's week."]}}} + config = [{"id": "solver", "type": "deploy_agent", "a2a_agent_id": "theirs"}, user, + {"id": "brief", "type": "prompt_agent", "agent_name": "linda", "prompt": "You are Linda."}, + {"id": "ask", "type": "prompt_agent", "prompt": "Help Linda.", "user_agent_name": "linda"}] + + assert harness._untrainable(config) is None + assert harness.solver_steps(config) == ("solver",)