From ae0bd517a6a03df22ba5995c125040294166f9c2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 07:48:13 +0000 Subject: [PATCH 1/3] test(verifiers): an errored agent turn under AGENT_ENV_ERRORED_TURN_UNGRADED is left ungraded Co-authored-by: Claude Opus 5.5 --- .../verifiers/test_errored_turn_ungraded.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 tst/unit/task_step/verifiers/test_errored_turn_ungraded.py diff --git a/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py b/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py new file mode 100644 index 000000000..1e17bf92f --- /dev/null +++ b/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py @@ -0,0 +1,79 @@ +"""AGENT_ENV_ERRORED_TURN_UNGRADED: a verifier whose agent turn errored leaves the run ungraded, failing its own step +with AgentTurnError, instead of recording a score of 0 that counts as the agent failing the task.""" +from __future__ import annotations + +import pytest + +from agent_env.task import Task +from agent_env.task_step.context import ERRORED_TURN_UNGRADED_ENV_VAR, PromptResponse, TaskStepContext +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep + + +def _rubrics(monkeypatch) -> RubricsVerifierTaskStep: + step = RubricsVerifierTaskStep( + id="grade", version=1, criteria=[{"id": "done", "description": "it is done"}], prompt_id="p1", + use_agent_judge=False, use_trajectory=True, verifier_id="grade", fail_task_on_error=False) + + async def judge(**kwargs): + return [{"id": "done", "description": "it is done", "score": 1.0, "result": True}], 0, [], None + + monkeypatch.setattr(step, "_read_trajectory_text", lambda uri: "EVENTS") + monkeypatch.setattr(step, "_run_judge_with_output_retries", judge) + return step + + +def _response(monkeypatch) -> AgentPromptResponseVerifierTaskStep: + return AgentPromptResponseVerifierTaskStep( + id="grade", version=None, prompt_id="p1", criteria=[{"type": "response_contains", "needles": ["done"]}], + verifier_id="grade", fail_task_on_error=False) + + +VERIFIERS = pytest.mark.parametrize("make_verifier", [_rubrics, _response], ids=["rubrics", "prompt-response"]) + + +def _context(error_type: str | None) -> TaskStepContext: + return TaskStepContext(prompt_responses=[PromptResponse( + prompt_id="p1", response="done", prompt_text="do it", agent_trajectory_object_url="s3://t/1.json", + error_type=error_type)]) + + +@VERIFIERS +@pytest.mark.asyncio +async def test_off_an_errored_turn_scores_zero(monkeypatch, local_stores, make_verifier): + monkeypatch.delenv(ERRORED_TURN_UNGRADED_ENV_VAR, raising=False) + ctx = _context("model_key_budget") + + await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx) + + entry = ctx.metadata["verifications"]["grade"] + assert (entry["score"], entry["results"][0]["id"]) == (0, "prompt_error") + assert "failed_steps" not in ctx.metadata + + +@VERIFIERS +@pytest.mark.asyncio +async def test_on_an_errored_turn_is_left_ungraded_and_its_verifier_fails(monkeypatch, local_stores, make_verifier): + monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1") + ctx = _context("model_key_budget") + + await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx) + + assert "grade" not in ctx.metadata.get("verifications", {}) + [failed] = ctx.metadata["failed_steps"] + assert (failed["step_id"], failed["error_type"]) == ("grade", "AgentTurnError") + assert "error_type=model_key_budget" in failed["error"] + + +@VERIFIERS +@pytest.mark.asyncio +async def test_on_a_turn_without_an_error_is_graded(monkeypatch, local_stores, make_verifier): + monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1") + ctx = _context(None) + + await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx) + + assert ctx.metadata["verifications"]["grade"]["score"] == 1.0 + assert "failed_steps" not in ctx.metadata From 317ebed080a37bfefa4344c87baffdd477ab0c61 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 07:48:29 +0000 Subject: [PATCH 2/3] feat(verifiers): leave an errored agent turn ungraded behind AGENT_ENV_ERRORED_TURN_UNGRADED With the flag set, the rubrics and prompt-response verifiers raise AgentTurnError for a turn whose prompt response carries an error_type, instead of writing a score-0 entry that counts as the agent failing. Off by default. Co-authored-by: Claude Opus 5.5 --- src/agent_env/task_step/context.py | 19 +++++++++++++++++++ .../agent_prompt_response_verifier.py | 4 +++- .../task_steps/verifiers/rubrics_verifier.py | 4 ++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 3343042d5..5e99b7f04 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -4,6 +4,7 @@ import contextlib import dataclasses +import os import re from collections.abc import Callable from dataclasses import dataclass, field @@ -59,6 +60,24 @@ class VerifierNotPassingError(RuntimeError): """A verifier set to fail its task when its verdict isn't passing recorded one that isn't. Grading the same run again could pass it, so a runner should not retry it.""" + +ERRORED_TURN_UNGRADED_ENV_VAR = "AGENT_ENV_ERRORED_TURN_UNGRADED" + + +def errored_turn_ungraded() -> bool: + """Whether a verifier leaves an errored agent turn ungraded, raising ``AgentTurnError``, instead of scoring it 0.""" + return os.environ.get(ERRORED_TURN_UNGRADED_ENV_VAR, "").strip().lower() in ("1", "true", "yes") + + +class AgentTurnError(RuntimeError): + """The agent turn a verifier grades ended in an error, so it has no answer to grade. The run is ungraded, not + failed by the agent: a 0 would count against the agent for a failure ``turn_error_type`` may say isn't its own.""" + + def __init__(self, verifier_id: str, prompt_id: str, turn_error_type: str) -> None: + super().__init__(f"Verifier '{verifier_id}' can't grade prompt '{prompt_id}': its agent turn failed with " + f"error_type={turn_error_type}") + self.turn_error_type = turn_error_type + _REDACTED_KEYS = { "litellm_api_key", "judge_litellm_api_key", "usersim_api_key", "remote_tokens", "cf_access_client_secret", HEARTBEAT_KEY, diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 7d868f330..8f112c098 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -21,7 +21,7 @@ import uuid from typing import ClassVar, Optional -from agent_env.task_step.context import StepContextError, TaskStepContext +from agent_env.task_step.context import AgentTurnError, StepContextError, TaskStepContext, errored_turn_ungraded from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score @@ -84,6 +84,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") if prompt_response.error_type: + if errored_turn_ungraded(): + raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type) logger.warning( f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}" ) diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a7796c22c..a3dc8899c 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -22,12 +22,14 @@ from agent_env.config.model import ModelParam from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import ( + AgentTurnError, PromptResponse, StepContextError, TaskStepContext, VerifierNotPassingError, context_path_segments, dual_keyed, + errored_turn_ungraded, ) from agent_env.task_step.task_steps.verifiers.judge_utils.judge_output_format import ( SCREENSHOT_GROUNDING, @@ -683,6 +685,8 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: stored_response, prompt_response = prompt_response, as_of if prompt_response.error_type: + if errored_turn_ungraded(): + raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type) logger.warning(f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}") self._write_skipped_entries( context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") From 6110b3b89f9aa59189a99a6696ba8f8758871b77 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 07:58:11 +0000 Subject: [PATCH 3/3] verifiers: leave only platform turn failures ungraded; an agent_error is still scored 0 --- src/agent_env/task_step/context.py | 13 ++++++--- .../agent_prompt_response_verifier.py | 2 +- .../task_steps/verifiers/rubrics_verifier.py | 2 +- .../verifiers/test_errored_turn_ungraded.py | 27 +++++++++++++++++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 5e99b7f04..8c5d2db4e 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -64,9 +64,16 @@ class VerifierNotPassingError(RuntimeError): ERRORED_TURN_UNGRADED_ENV_VAR = "AGENT_ENV_ERRORED_TURN_UNGRADED" -def errored_turn_ungraded() -> bool: - """Whether a verifier leaves an errored agent turn ungraded, raising ``AgentTurnError``, instead of scoring it 0.""" - return os.environ.get(ERRORED_TURN_UNGRADED_ENV_VAR, "").strip().lower() in ("1", "true", "yes") +# Turn failures the agent did not cause: the platform or its model key. An ``agent_error`` stays a scored 0, since +# the agent made the attempt and failed it. +PLATFORM_TURN_ERROR_TYPES = frozenset({"infra_error", "model_key_budget", "model_key_auth"}) + + +def errored_turn_ungraded(turn_error_type: str) -> bool: + """Whether a verifier leaves this errored agent turn ungraded, raising ``AgentTurnError``, instead of scoring it + 0: only behind the flag, and only for failures the agent did not cause.""" + flag_on = os.environ.get(ERRORED_TURN_UNGRADED_ENV_VAR, "").strip().lower() in ("1", "true", "yes") + return flag_on and turn_error_type in PLATFORM_TURN_ERROR_TYPES class AgentTurnError(RuntimeError): diff --git a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py index 8f112c098..842fd20de 100644 --- a/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/agent_prompt_response_verifier.py @@ -84,7 +84,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: raise StepContextError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") if prompt_response.error_type: - if errored_turn_ungraded(): + if errored_turn_ungraded(prompt_response.error_type): raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type) logger.warning( f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}" diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index a3dc8899c..6e2530f21 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -685,7 +685,7 @@ async def _grade(self, context: TaskStepContext) -> TaskStepContext: stored_response, prompt_response = prompt_response, as_of if prompt_response.error_type: - if errored_turn_ungraded(): + if errored_turn_ungraded(prompt_response.error_type): raise AgentTurnError(self.verifier_id, self.prompt_id, prompt_response.error_type) logger.warning(f"Skipping verification '{self.verifier_id}': prompt had error_type={prompt_response.error_type}") self._write_skipped_entries( diff --git a/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py b/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py index 1e17bf92f..f59a94bd4 100644 --- a/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py +++ b/tst/unit/task_step/verifiers/test_errored_turn_ungraded.py @@ -67,6 +67,33 @@ async def test_on_an_errored_turn_is_left_ungraded_and_its_verifier_fails(monkey assert "error_type=model_key_budget" in failed["error"] +@VERIFIERS +@pytest.mark.parametrize("platform_error", ["infra_error", "model_key_auth"]) +@pytest.mark.asyncio +async def test_on_other_platform_failures_are_left_ungraded(monkeypatch, local_stores, make_verifier, platform_error): + monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1") + ctx = _context(platform_error) + + await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx) + + assert "grade" not in ctx.metadata.get("verifications", {}) + [failed] = ctx.metadata["failed_steps"] + assert f"error_type={platform_error}" in failed["error"] + + +@VERIFIERS +@pytest.mark.asyncio +async def test_on_an_agent_error_still_scores_zero(monkeypatch, local_stores, make_verifier): + monkeypatch.setenv(ERRORED_TURN_UNGRADED_ENV_VAR, "1") + ctx = _context("agent_error") + + await Task(id="t", version=None, steps=[make_verifier(monkeypatch)]).run(context=ctx) + + entry = ctx.metadata["verifications"]["grade"] + assert (entry["score"], entry["results"][0]["id"]) == (0, "prompt_error") + assert "failed_steps" not in ctx.metadata + + @VERIFIERS @pytest.mark.asyncio async def test_on_a_turn_without_an_error_is_graded(monkeypatch, local_stores, make_verifier):