Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/agent_env/bundle/resolve.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from agent_env.store.image_store.oci_registry_credentials import names_registry
from agent_env.task_step.context import ENV_KEYED_METADATA, context_path, context_path_segments
from agent_env.task_step.registry import get_task_step_registry
from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies
from agent_env.task_step.task_step import TaskStep, attach_base_fields, dependencies
from agent_env.task_step.task_steps.prompt_agent import is_prompt_step, user_sim_agents

from ._fs import fold, relative, show
Expand Down Expand Up @@ -562,7 +562,7 @@ def build_step(step: dict) -> TaskStep:
"""The step a task's write puts, built from a copy of ``step``, so the resolved config the ledger hashes
never changes."""
step = copy.deepcopy(step)
return attach_retry_config(get_task_step_registry()[step["type"]].from_dict(step), step)
return attach_base_fields(get_task_step_registry()[step["type"]].from_dict(step), step)


def _nfc(name: str) -> str:
Expand Down
15 changes: 7 additions & 8 deletions src/agent_env/cli/task/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,13 @@ def _build_steps(steps: list[dict]) -> list[tuple[dict, object]]:
raise SystemExit(1)
kwargs = {k: v for k, v in step_dict.items() if k != "type"}
kwargs.setdefault("version", None)
# retry_config lives on the TaskStep base but most step __init__s don't
# accept it; construct without it and hydrate (as Task.from_dict does),
# so retry_config on any step type doesn't crash create. put() below
# handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_retry_config

ctor_kwargs = {k: v for k, v in kwargs.items() if k != "retry_config"}
step = attach_retry_config(step_cls(**ctor_kwargs), step_dict)
# retry_config and step_role live on the TaskStep base but most step
# __init__s don't accept them; construct without them and hydrate (as
# Task.from_dict does). put() below handles the kwargs form the same way.
from agent_env.task_step.task_step import attach_base_fields

ctor_kwargs = {k: v for k, v in kwargs.items() if k not in ("retry_config", "step_role")}
step = attach_base_fields(step_cls(**ctor_kwargs), step_dict)
built.append((kwargs, step))
return built

Expand Down
129 changes: 129 additions & 0 deletions src/agent_env/task/outcome.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""What a step failure means for its run: the role the failed step played and why it failed.

A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what
the run lost. Each ``context.metadata.failed_steps`` entry carries them.
"""

from __future__ import annotations

import re
from collections.abc import Iterator
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING, Any

from agent_env.task_step.task_step import StepRole

if TYPE_CHECKING:
from agent_env.task_step.task_step import TaskStep

__all__ = ["FailedStep", "FailureCause", "StepRole", "classify_failure"]


class FailureCause(StrEnum):
KEY_BUDGET = "key_budget"
KEY_AUTH = "key_auth"
TRANSIENT = "transient"
UNKNOWN = "unknown"


# Checked in order: a key that is out of budget or rejected fails every retry, so it wins over a timeout that wraps it.
_CAUSE_PATTERNS: tuple[tuple[FailureCause, re.Pattern[str]], ...] = (
(FailureCause.KEY_BUDGET, re.compile(
r"budget\s+(?:has\s+been\s+)?exceeded|exceeded\s+(?:the\s+|your\s+)?(?:max\s+)?budget|ExceededBudget|BudgetExceeded",
re.IGNORECASE,
)),
(FailureCause.KEY_AUTH, re.compile(
r"\b401\b|unauthori[sz]ed|authentication\s*error|token_not_found_in_db|incorrect\s+api\s+key"
r"|invalid[\s_-]+(?:proxy[\s_-]+server[\s_-]+)?(?:api[\s_-]+)?(?:key|token)"
r"|(?:key|token)\s+(?:has\s+)?(?:been\s+)?(?:revoked|expired)|(?:revoked|expired)\s+(?:api[\s_-]+)?(?:key|token)",
re.IGNORECASE,
)),
(FailureCause.TRANSIENT, re.compile(
r"timed?[\s_-]*out|disconnect|connection\s*(?:reset|refused|aborted|closed|error)|broken\s*pipe|remote\s+end\s+closed"
r"|internal\s+server\s+error|bad\s+gateway|service\s+unavailable|gateway\s+time-?out"
r"|(?:HTTP|status(?:[\s_]code)?|error)[\s:=]*5\d\d\b"
r"|\b429\b|rate[\s_-]?limit|too\s+many\s+requests"
r"|OOM[\s_-]?kill|out\s+of\s+memory|\bOOM\b|exit(?:ed)?(?:\s+with)?\s+code\s+137",
re.IGNORECASE,
)),
)


def _chain(exc: BaseException) -> Iterator[BaseException]:
"""``exc`` and every exception it was raised from or during, each once."""
seen: set[int] = set()
stack: list[BaseException | None] = [exc]
while stack:
current = stack.pop()
if current is None or id(current) in seen:
continue
seen.add(id(current))
yield current
stack.extend([current.__cause__, current.__context__])
if isinstance(current, BaseExceptionGroup):
stack.extend(current.exceptions)


def classify_failure(exc: BaseException) -> FailureCause:
"""Why ``exc`` happened, read from the type names and messages of its whole chain.

Matching text is the only option: a judge or agent error crosses A2A and the model proxy as a message string, so its
original type is gone by the time a step raises.
"""
text = "\n".join(f"{type(e).__name__}: {e}" for e in _chain(exc))
for cause, pattern in _CAUSE_PATTERNS:
if pattern.search(text):
return cause
return FailureCause.UNKNOWN


@dataclass(frozen=True)
class FailedStep:
"""One ``context.metadata.failed_steps`` entry."""

step_id: str
step_type: str
error: str
error_type: str
started_at_utc: str
duration_seconds: float
is_fatal: bool
retried: bool = False
step_role: StepRole = StepRole.OTHER
cause: FailureCause = FailureCause.UNKNOWN
# Set once a later action (e.g. a regrade) repaired what this failure lost.
resolved_by: str | None = None

@classmethod
def from_exception(
cls, step: TaskStep, exc: BaseException, *, started_at_utc: str, duration_seconds: float, is_fatal: bool,
) -> FailedStep:
return cls(
step_id=step.id,
step_type=step.type,
error=str(exc),
error_type=type(exc).__name__,
started_at_utc=started_at_utc,
duration_seconds=duration_seconds,
is_fatal=is_fatal,
step_role=getattr(step, "step_role", StepRole.OTHER),
cause=classify_failure(exc),
)

def to_dict(self) -> dict[str, Any]:
"""The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and
``resolved_by`` appear only once set, as ``retried`` always has."""
out: dict[str, Any] = {
"step_id": self.step_id, "step_type": self.step_type,
"error": self.error, "error_type": self.error_type,
"started_at_utc": self.started_at_utc, "duration_seconds": self.duration_seconds,
"is_fatal": self.is_fatal,
"step_role": self.step_role.value, "cause": self.cause.value,
}
if self.retried:
out["retried"] = True
if self.resolved_by is not None:
out["resolved_by"] = self.resolved_by
return out

18 changes: 8 additions & 10 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from agent_env.store.document_store import UnknownDocumentTypeError
from agent_env.store.routing import run_scope
from agent_env.task_step.preflight import pinned
from agent_env.task.outcome import FailedStep
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
Expand Down Expand Up @@ -130,12 +131,12 @@ def _sub(value):
return {k: _sub(v) for k, v in value.items()}
return value

from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

out: list["TaskStep"] = []
for step in steps:
data = _sub(step.to_dict())
out.append(attach_retry_config(type(step).from_dict(data), data))
out.append(attach_base_fields(type(step).from_dict(data), data))
return out


Expand Down Expand Up @@ -460,9 +461,9 @@ def from_dict(cls, data: dict) -> "Task":
f"{_registration.failure_note(_registration.TASK_STEPS, step_type)}", *problems]),
unmet,
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

steps.append(attach_retry_config(step_cls.from_dict(step_dict), step_dict))
steps.append(attach_base_fields(step_cls.from_dict(step_dict), step_dict))
return cls(
id=data["id"],
version=data.get("version"),
Expand Down Expand Up @@ -681,12 +682,9 @@ async def _run_one(step: TaskStep, idx: int, dispatch_epoch: int) -> None:
except BaseException as exc:
duration = time.monotonic() - started_mono
fatal = getattr(step, "fail_task_on_error", True)
context.metadata.setdefault("failed_steps", []).append({
"step_id": step.id, "step_type": step.type,
"error": str(exc), "error_type": type(exc).__name__,
"started_at_utc": started_utc, "duration_seconds": duration,
"is_fatal": fatal,
})
context.metadata.setdefault("failed_steps", []).append(FailedStep.from_exception(
step, exc, started_at_utc=started_utc, duration_seconds=duration, is_fatal=fatal,
).to_dict())
if not fatal:
await _heartbeat_step(step, pre_context, _store.TaskStepStatus.FAILURE, dispatch_epoch)
raise
Expand Down
4 changes: 2 additions & 2 deletions src/agent_env/task_step/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,9 @@ def _deserialize(self, doc: dict) -> TaskStep:
f"Unknown task step type: {task_step_type}"
f"{_registration.failure_note(_registration.TASK_STEPS, task_step_type)}"
)
from agent_env.task_step.task_step import attach_retry_config
from agent_env.task_step.task_step import attach_base_fields

return attach_retry_config(cls.from_dict(doc), doc)
return attach_base_fields(cls.from_dict(doc), doc)

def execute_query(self, query: TaskStepQuery) -> list[TaskStep]:
filt, sort = to_document_query(query)
Expand Down
34 changes: 24 additions & 10 deletions src/agent_env/task_step/task_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass
from enum import StrEnum
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self

from agent_env.task_step.context import TaskStepContext
Expand Down Expand Up @@ -40,13 +41,23 @@ def dependencies(raw: Any) -> list[TaskStepDependency] | None:
return [dep if isinstance(dep, TaskStepDependency) else TaskStepDependency.from_dict(dep) for dep in raw]


def attach_retry_config(step: "TaskStep", data: dict[str, Any]) -> "TaskStep":
"""Hydrate ``retry_config`` after ``step_cls.from_dict`` at the generic
parse sites, so the field survives round-trips on step types whose
``__init__`` doesn't accept it. No-op if already set or absent."""
class StepRole(StrEnum):
"""What a step contributes to a run, so a tolerated failure says what the run lost."""

GRADER = "grader"
EVIDENCE = "evidence"
AGENT = "agent"
OTHER = "other"


def attach_base_fields(step: "TaskStep", data: dict[str, Any]) -> "TaskStep":
"""Hydrate the base fields most step ``__init__``s don't accept (``retry_config``, ``step_role``) after
``step_cls.from_dict`` at the generic parse sites, so they survive round-trips on every step type."""
raw = data.get("retry_config")
if raw is not None and getattr(step, "retry_config", None) is None:
step.retry_config = RetryConfig.from_dict(raw)
if data.get("step_role") is not None:
step.step_role = StepRole(data["step_role"])
return step


Expand Down Expand Up @@ -97,6 +108,9 @@ class TaskStep(ABC):
# Each field of a scope other than the task's, by its path in the stored step; any other field is task-scoped. A
# subclass merges its parent's. Such a field is optional, so a step written out without it still builds and runs.
field_scopes: ClassVar[Mapping[str, FieldScope]] = {"metadata.attribution": FieldScope.USER}
# A step type declares its role; one step overrides it in config, e.g. a run_code step that grades. Not
# ``role``: deploy_agent and modify_env_tool_access already use that name for their own config.
step_role: StepRole = StepRole.OTHER

def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
Expand Down Expand Up @@ -163,6 +177,8 @@ def to_dict(self) -> dict[str, Any]:
base["depends_on"] = [dataclasses.asdict(d) for d in self.depends_on]
if self.retry_config is not None:
base["retry_config"] = dataclasses.asdict(self.retry_config)
if self.step_role != type(self).step_role:
base["step_role"] = self.step_role.value
return base

@classmethod
Expand All @@ -188,13 +204,11 @@ def put(cls, **kwargs: Any) -> Self:
from .store import get_task_step_store

kwargs.setdefault("version", None)
# Most step __init__s don't accept retry_config (it lives on the base but
# subclasses don't forward it); construct without it, then hydrate so a
# retry_config on any step type round-trips through the step store.
retry_config_raw = kwargs.pop("retry_config", None)
# Most step __init__s don't accept the base-only fields; construct without
# them, then hydrate so they round-trip through the step store on any type.
base_fields = {k: kwargs.pop(k) for k in ("retry_config", "step_role") if k in kwargs}
instance = cls(**kwargs)
if retry_config_raw is not None:
attach_retry_config(instance, {"retry_config": retry_config_raw})
attach_base_fields(instance, base_fields)
return get_task_step_store().put_document(instance)

@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)

Expand All @@ -17,6 +17,7 @@

class VerifyA2AAgentCardStep(TaskStep):
type: ClassVar[str] = "verify_a2a_agent_card"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),)

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@
from agent_env.a2a_agent import A2AAgent
from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)


class VerifyA2AAgentConfigIdentityStep(TaskStep):
type: ClassVar[str] = "verify_a2a_agent_config_identity"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),)

def __init__(self, id: str, version: Optional[int], a2a_agent_id: str, a2a_agent_version: Optional[int] = None, depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)

Expand All @@ -25,6 +25,7 @@

class VerifyA2AAgentMCPStep(TaskStep):
type: ClassVar[str] = "verify_a2a_agent_mcp"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id"),)

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from agent_env.a2a_agent import A2AAgent
from agent_env.entity.refs import EntityRef
from agent_env.task_step.context import TaskStepContext
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep

Expand All @@ -32,6 +32,7 @@

class VerifyA2ABuiltinToolsStep(TaskStep):
type: ClassVar[str] = "verify_a2a_builtin_tools"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id", version_field="a2a_agent_version"),)

def __init__(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@

from agent_env.task_step.context import TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_step import StepRole, TaskStep, TaskStepDependency

logger = logging.getLogger(__name__)


class VerifyCoreA2AProtocolStep(TaskStep):
type: ClassVar[str] = "verify_a2a_core_protocol"
step_role = StepRole.GRADER
entity_refs = (EntityRef.agent("a2a_agent_id"),)

def __init__(
Expand Down
Loading
Loading