Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/agent_env/task/task.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@
from agent_env.store.routing import run_scope
from agent_env.task_step.preflight import pinned
from agent_env.task.teardown import env_record_ids
from agent_env.task_step.context import ENTITY_IDS_KEY, TaskStepContext, VerifierNotPassingError, regraft_redacted_keys
from agent_env.task_step.context import (
ENTITY_IDS_KEY,
ModelKeyUnusable,
TaskStepContext,
VerifierNotPassingError,
regraft_redacted_keys,
)
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
from agent_env.task_step.task_step import RetryConfig, TaskStep

Expand Down Expand Up @@ -753,7 +759,7 @@ async def _drain_in_flight() -> set[str]:
continue
retry_cfg = getattr(step_obj, "retry_config", None)
if (retry_cfg is not None and retry_counts.get(step_id, 0) < retry_cfg.max_retries
and not isinstance(exc, VerifierNotPassingError)):
and not isinstance(exc, (VerifierNotPassingError, ModelKeyUnusable))):
retryable.append((step_obj, retry_cfg, exc))
continue
if not getattr(step_obj, "fail_task_on_error", True):
Expand Down
17 changes: 16 additions & 1 deletion src/agent_env/task_step/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import re
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
from typing import Any, Literal

from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX
from agent_env.env.env import DeployedEnv
Expand Down Expand Up @@ -55,6 +55,21 @@ class StepContextError(RuntimeError):
way, so a runner should not retry it."""


ModelKeyCause = Literal["budget", "auth"]

MODEL_KEY_ERROR_TYPES: dict[str, ModelKeyCause] = {"model_key_budget": "budget", "model_key_auth": "auth"}


class ModelKeyUnusable(RuntimeError):
"""The model endpoint refused the run's key as out of budget or not authorized. The same key fails the same
way on every attempt, so a runner should not retry it. Carries no key material."""

def __init__(self, cause: ModelKeyCause, where: str) -> None:
super().__init__(f"The model key is unusable ({cause}) for the {where}; retrying with it cannot succeed")
self.cause = cause
self.where = where


class VerifierNotPassingError(RuntimeError):
"""A verifier set to fail its task when its verdict isn't passing recorded one that isn't. Grading the same run
again could pass it, so a runner should not retry it."""
Expand Down
11 changes: 10 additions & 1 deletion src/agent_env/task_step/task_steps/prompt_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,14 @@
)
from agent_env.store import DuplicateKeyError, get_config
from agent_env.store.object_store import ObjectStore
from agent_env.task_step.context import DeployedAgent, PromptResponse, StepContextError, TaskStepContext
from agent_env.task_step.context import (
MODEL_KEY_ERROR_TYPES,
DeployedAgent,
ModelKeyUnusable,
PromptResponse,
StepContextError,
TaskStepContext,
)
from agent_env.entity.refs import EntityRef
from agent_env.entity.scopes import FieldScope
from agent_env.task_step.task_step import RetryConfig, TaskStep, TaskStepDependency
Expand Down Expand Up @@ -826,6 +833,8 @@ def record_turn() -> None:
))

if final_state == TaskState.failed:
if final_terminal.error_type in MODEL_KEY_ERROR_TYPES:
raise ModelKeyUnusable(MODEL_KEY_ERROR_TYPES[final_terminal.error_type], "agent")
detail_parts = []
if final_terminal.error_type:
detail_parts.append(f"error_type={final_terminal.error_type}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
from agent_env.config.model import ModelParam
from agent_env.providers.sandbox_providers.sandbox import into_container
from agent_env.task_step.context import (
MODEL_KEY_ERROR_TYPES,
ModelKeyCause,
ModelKeyUnusable,
PromptResponse,
StepContextError,
TaskStepContext,
Expand Down Expand Up @@ -271,6 +274,21 @@ def _validated_run_context_parts(parts: Optional[list[str]]) -> Optional[list[st
)


_BUDGET_MARKERS = ("budget has been exceeded", "exceededbudget", "budget_exceeded")


def _model_key_refusal(exc: Exception) -> Optional[ModelKeyCause]:
"""Whether a LiteLLM error says the endpoint refused the key itself. Parsed here, once, so the retry loop
branches on a cause and never on message text. Any other failure, including a 429 rate limit, returns None."""
status = getattr(exc, "status_code", None)
text = str(exc).lower()
if type(exc).__name__ == "BudgetExceededError" or any(marker in text for marker in _BUDGET_MARKERS):
return "budget"
if status == 401 or type(exc).__name__ == "AuthenticationError":
return "auth"
return None


def _looks_like_response_format_error(exc: Exception) -> bool:
msg = str(exc).lower()
return any(s in msg for s in _SCHEMA_TERMS) and any(u in msg for u in _UNSUPPORTED_TERMS)
Expand Down Expand Up @@ -1252,6 +1270,12 @@ async def _prompt_llm_judge(
return {"response": content, "trajectory_object_url": None}
except Exception as exc:
last_exc = exc
if (cause := _model_key_refusal(exc)) is not None:
logger.error(
f"LLM judge for verifier '{self.verifier_id}' refused its model key "
f"({cause}, {type(exc).__name__}); not retrying"
)
raise ModelKeyUnusable(cause, "judge") from None
if image_blocks and use_response_format and _looks_like_response_format_error(exc):
logger.warning(
f"Multimodal judge call with json_schema failed for verifier "
Expand Down Expand Up @@ -1334,6 +1358,8 @@ async def _invoke_judge_a2a(
status_msg = (result.get("status") or {}).get("message") or {}
tr = protocol.TerminalResponse.from_message(status_msg)
if state == "failed":
if tr.error_type in MODEL_KEY_ERROR_TYPES:
raise ModelKeyUnusable(MODEL_KEY_ERROR_TYPES[tr.error_type], "judge")
sandbox_id = getattr(judge_agent, "sandbox_id", None)
logger.error(
"Judge A2A task %s failed (verifier=%s sandbox=%s error_type=%s exception=%s). "
Expand Down
28 changes: 27 additions & 1 deletion tst/unit/task_step/retry_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,13 @@
set_task_instance_store,
)
from agent_env.task.task import Task, _rolled_back_sandboxes
from agent_env.task_step.context import DeployedAgent, DeployedSandbox, PromptResponse, TaskStepContext
from agent_env.task_step.context import (
DeployedAgent,
DeployedSandbox,
ModelKeyUnusable,
PromptResponse,
TaskStepContext,
)
from agent_env.task_step.context_ops import ContextUpdateOps, build_context_update_ops
from agent_env.task_step.task_step import RetryConfig
from agent_env.config import set_document_store
Expand Down Expand Up @@ -1110,3 +1116,23 @@ def test_validate_dag_accepts_ancestor_resume_point():
c = _FlakyPrompt("c", retry_config=RetryConfig(retry_from_step_id="a"), depends_on=["a"])
# Should not raise.
Task(id="t", version=1, steps=[a, c])


class _DeadKeyPrompt(_FlakyPrompt):
async def execute(self, context: TaskStepContext) -> TaskStepContext:
self.attempts += 1
raise ModelKeyUnusable("budget", "agent")


@pytest.mark.asyncio
async def test_a_step_whose_model_key_is_unusable_is_not_retried(store):
deploy = _FakeDeployEnv("s0")
dead = _DeadKeyPrompt("s1", retry_config=RetryConfig(retry_from_step_id="s0", max_retries=3))
task = Task(id="t", version=1, steps=[deploy, dead])

ctx = TaskStepContext()
with pytest.raises(ModelKeyUnusable):
await task.run(context=ctx)

assert dead.attempts == 1
assert store.get(ctx.instance_id).status == "failed"
65 changes: 65 additions & 0 deletions tst/unit/task_step/test_prompt_agent_model_key.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
"""prompt_agent fails the step with ModelKeyUnusable when the agent turn was refused for its model key."""

from unittest.mock import MagicMock

import pytest
from a2a.types import TaskState

from agent_env.task_step.context import DeployedAgent, ModelKeyUnusable, TaskStepContext
from agent_env.task_step.task_steps import prompt_agent as pa_mod
from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep


def _failed(error_type):
data = {"error_type": error_type, "error_message": "refused"}
return {"status": {"state": TaskState.failed, "message": {"parts": [{"kind": "data", "data": data}]}}}


def _wire(monkeypatch, result):
cfg = MagicMock()
cfg.get_model_params.return_value = {}
monkeypatch.setattr(pa_mod, "get_config", lambda: cfg)
for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"):
monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None)

async def send(url, parts, message_id, context_id, timeout):
return f"{url}#task", None

async def poll(url, task_id, timeout, interval, *, sandbox_id=None):
return result

monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send)
monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll)


def _context():
ctx = TaskStepContext(instance_id="ti-1")
ctx.deployed_agents.append(DeployedAgent(
agent_name="solver", api_url="http://solver", a2a_url="http://solver", a2a_card={}, sandbox_id="sb-solver",
))
return ctx


@pytest.mark.asyncio
@pytest.mark.parametrize("error_type, cause", [("model_key_budget", "budget"), ("model_key_auth", "auth")])
async def test_a_turn_refused_for_its_model_key_raises_model_key_unusable(monkeypatch, error_type, cause):
_wire(monkeypatch, _failed(error_type))
step = PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver")
ctx = _context()

with pytest.raises(ModelKeyUnusable) as raised:
await step.execute(ctx)

assert raised.value.cause == cause
assert ctx.prompt_responses[-1].error_type == error_type


@pytest.mark.asyncio
async def test_any_other_failed_turn_still_raises_the_plain_error(monkeypatch):
_wire(monkeypatch, _failed("agent_error"))
step = PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver")

with pytest.raises(RuntimeError, match="A2A task failed") as raised:
await step.execute(_context())

assert not isinstance(raised.value, ModelKeyUnusable)
100 changes: 100 additions & 0 deletions tst/unit/task_step/verifiers/test_model_key_unusable.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""A model key the endpoint refuses fails the judge on its first call; a transient failure still retries."""
from __future__ import annotations

import json

import litellm
import pytest

from agent_env.a2a_agent import protocol
from agent_env.config.model import ModelConfig
from agent_env.store import LocalSecretStore
from agent_env.task_step.context import ModelKeyUnusable, TaskStepContext
from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep

_VERDICT = {"results": [{"criterion_id": "c1", "passed": True}]}
_BUDGET = litellm.BadRequestError(
message="Budget has been exceeded! Current cost: 51.0, Max budget: 50.0", model="m", llm_provider="openai",
)
_AUTH = litellm.AuthenticationError(
message="Incorrect API key provided: sk-secret123", model="m", llm_provider="openai",
)


@pytest.fixture
def judge(monkeypatch):
import agent_env.config as cfg_mod

config = cfg_mod.Config()
config.set_secret_store(LocalSecretStore(values={"litellm_api_key": "k"}, use_env=False))
monkeypatch.setattr(config, "_model_cfg", ModelConfig())
monkeypatch.setattr(cfg_mod, "get_config", lambda: config)
monkeypatch.delenv("LITELLM_BASE_URL", raising=False)
monkeypatch.delenv("LITELLM_API_KEY", raising=False)
return RubricsVerifierTaskStep(
id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=False,
default_model_api_base="https://judge.invalid/v1", verifier_id="vt",
)


def _answer_with(monkeypatch, *answers):
real, calls, pending = litellm.acompletion, [], list(answers)

async def acompletion(**kwargs):
calls.append(kwargs)
return await real(**kwargs, mock_response=pending.pop(0))

monkeypatch.setattr(litellm, "acompletion", acompletion)
return calls


@pytest.mark.asyncio
@pytest.mark.parametrize("refusal, cause", [(_BUDGET, "budget"), (_AUTH, "auth")], ids=["budget", "auth"])
async def test_a_judge_whose_key_is_refused_is_called_once_and_raises_model_key_unusable(
judge, monkeypatch, refusal, cause
):
calls = _answer_with(monkeypatch, *[refusal] * judge.DEFAULT_MAX_RETRIES)

with pytest.raises(ModelKeyUnusable) as raised:
await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext())

assert len(calls) == 1
assert raised.value.cause == cause
assert "sk-secret123" not in str(raised.value)
assert raised.value.__cause__ is None


@pytest.mark.asyncio
async def test_a_transient_judge_error_is_still_retried(judge, monkeypatch):
outage = litellm.ServiceUnavailableError(message="upstream down", model="m", llm_provider="openai")
rate_limit = litellm.RateLimitError(message="slow down", model="m", llm_provider="openai")
calls = _answer_with(monkeypatch, outage, rate_limit, json.dumps(_VERDICT))

out = await judge._prompt_llm_judge("grade it", model="openai/gpt-4o", context=TaskStepContext())

assert json.loads(out["response"]) == _VERDICT
assert len(calls) == 3


@pytest.mark.asyncio
async def test_an_agent_judge_task_failed_for_its_key_raises_model_key_unusable(monkeypatch):
async def send(*args, **kwargs):
return "task-1", "ctx-1"

async def poll(*args, **kwargs):
return {"status": {"state": "failed", "message": {"parts": [
{"kind": "data", "data": {"error_type": "model_key_budget", "error_message": "Budget has been exceeded"}},
]}}}

monkeypatch.setattr(protocol, "send_a2a_message", send)
monkeypatch.setattr(protocol, "poll_a2a_task", poll)
verifier = RubricsVerifierTaskStep(
id="v", version=1, criteria=[], prompt_id="p", use_agent_judge=True, verifier_id="vt",
)

with pytest.raises(ModelKeyUnusable) as raised:
await verifier._invoke_judge_a2a(
eval_prompt="grade it", judge_a2a_url="http://judge.example", judge_agent_card={}, judge_agent=None,
)

assert raised.value.cause == "budget"
Loading