From f628698310e96678a6ea4e47f3ab59f44569c79a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 07:33:51 +0000 Subject: [PATCH] Stamp a completed run's quality in the same write as its status A run that tolerated a step failure completes like any other, so a reader of status cannot tell it lost its grade, its snapshot or its agent turn. The completion CAS now also writes quality = {degraded, reasons}, derived from failed_steps by run_quality. Reasons are the roles of the tolerated failures that were not retried or resolved, most severe first. TaskInstance.quality reads it back. A run completed before this change has no quality and reads as clean, which is what every reader assumes today. No behavior changes: status keeps its values and quality is a new field. --- src/agent_env/task/outcome.py | 63 ++++++++++++++++-- src/agent_env/task/store.py | 11 ++++ tst/unit/task/run_quality_test.py | 102 ++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 tst/unit/task/run_quality_test.py diff --git a/src/agent_env/task/outcome.py b/src/agent_env/task/outcome.py index eb9192eca..f31a32945 100644 --- a/src/agent_env/task/outcome.py +++ b/src/agent_env/task/outcome.py @@ -1,14 +1,15 @@ -"""What a step failure means for its run: the role the failed step played and why it failed. +"""What a step failure means for its run: the role the failed step played, why it failed, and whether the completed +run is degraded. A tolerated failure (``fail_task_on_error=False``) lets the run complete, so these facts are the only record of what -the run lost. Each ``context.metadata.failed_steps`` entry carries them. +the run lost. Each ``context.metadata.failed_steps`` entry carries them, and a completed run's ``quality`` sums them up. """ from __future__ import annotations import re -from collections.abc import Iterator -from dataclasses import dataclass +from collections.abc import Iterable, Iterator, Mapping +from dataclasses import dataclass, field from enum import StrEnum from typing import TYPE_CHECKING, Any @@ -17,7 +18,7 @@ if TYPE_CHECKING: from agent_env.task_step.task_step import TaskStep -__all__ = ["FailedStep", "FailureCause", "StepRole", "classify_failure"] +__all__ = ["FailedStep", "FailureCause", "RunQuality", "StepRole", "classify_failure", "run_quality"] class FailureCause(StrEnum): @@ -111,6 +112,24 @@ def from_exception( cause=classify_failure(exc), ) + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> FailedStep: + """An entry as stored. One written before ``step_role`` and ``cause`` existed, or holding a value this version + does not know, reads as ``other`` and ``unknown``.""" + return cls( + step_id=data.get("step_id", ""), + step_type=data.get("step_type", ""), + error=data.get("error", ""), + error_type=data.get("error_type", ""), + started_at_utc=data.get("started_at_utc", ""), + duration_seconds=data.get("duration_seconds", 0.0), + is_fatal=bool(data.get("is_fatal", True)), + retried=bool(data.get("retried", False)), + step_role=StepRole(data["step_role"]) if data.get("step_role") in _ROLES else StepRole.OTHER, + cause=FailureCause(data["cause"]) if data.get("cause") in _CAUSES else FailureCause.UNKNOWN, + resolved_by=data.get("resolved_by"), + ) + def to_dict(self) -> dict[str, Any]: """The stored form: the keys every reader already knows, plus ``step_role`` and ``cause``. ``retried`` and ``resolved_by`` appear only once set, as ``retried`` always has.""" @@ -127,3 +146,37 @@ def to_dict(self) -> dict[str, Any]: out["resolved_by"] = self.resolved_by return out + +_ROLES = frozenset(StepRole) +_CAUSES = frozenset(FailureCause) +# Most severe first: a run with no grade is worse than one missing its agent's turn, which is worse than one missing +# evidence. +_SEVERITY = (StepRole.GRADER, StepRole.AGENT, StepRole.EVIDENCE, StepRole.OTHER) + + +@dataclass(frozen=True) +class RunQuality: + """How much of a completed run is usable. ``reasons`` holds the roles of the steps whose failure the run tolerated + and nothing has repaired, most severe first; a run with none is clean.""" + + reasons: tuple[StepRole, ...] = () + # Derived, never passed, so it cannot disagree with ``reasons``; a field so ``asdict`` matches the stored form. + degraded: bool = field(init=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "degraded", bool(self.reasons)) + + @classmethod + def from_dict(cls, data: Mapping[str, Any] | None) -> RunQuality: + """``quality`` as stored. A run completed before ``quality`` existed has none and reads as clean.""" + roles = {StepRole(r) for r in (data or {}).get("reasons", ()) if r in _ROLES} + return cls(tuple(r for r in _SEVERITY if r in roles)) + + def to_dict(self) -> dict[str, Any]: + return {"degraded": self.degraded, "reasons": [r.value for r in self.reasons]} + + +def run_quality(failed_steps: Iterable[FailedStep]) -> RunQuality: + """A fatal failure fails the run instead, and a retried or resolved one lost nothing.""" + open_roles = {f.step_role for f in failed_steps if not f.is_fatal and not f.retried and f.resolved_by is None} + return RunQuality(tuple(r for r in _SEVERITY if r in open_roles)) diff --git a/src/agent_env/task/store.py b/src/agent_env/task/store.py index 72a1b1fbb..17acc0140 100644 --- a/src/agent_env/task/store.py +++ b/src/agent_env/task/store.py @@ -28,6 +28,7 @@ commit_ordered, replay_context, ) +from agent_env.task.outcome import FailedStep, RunQuality, run_quality from agent_env.task_step.context import TaskStepContext if TYPE_CHECKING: @@ -184,6 +185,8 @@ class TaskInstance: # carries the epoch it was dispatched under and no-ops when older than the stored one, # fencing a writer from a rolled-back attempt. Missing (pre-field docs) reads as 0. attempt_epoch: int = 0 + # Stamped with ``status="completed"``; clean until then, and on a run completed before the field existed. + quality: RunQuality = field(default_factory=RunQuality) @classmethod def from_dict(cls, data: dict) -> TaskInstance: @@ -202,6 +205,7 @@ def from_dict(cls, data: dict) -> TaskInstance: completed_steps=[TaskStepResult.from_dict(e) for e in data.get("completed_steps", [])], step_attempt_failures=data.get("step_attempt_failures", []), attempt_epoch=data.get("attempt_epoch", 0), + quality=RunQuality.from_dict(data.get("quality")), ) @@ -717,6 +721,13 @@ def _mutate(doc: dict) -> UpdateSpec | None: if len(completed) >= total_steps and doc.get("status") not in ("failed", "cancelled"): set_fields["status"] = "completed" set_fields["completed_at_utc"] = completed_at_utc + # In the same write as the status, so no reader sees a completed run without its quality. + after = {"context": copy.deepcopy(doc.get("context"))} + _apply_forward(after, ops, move_to_end=rerecorded) + failed_steps = _get_path(after, "context.metadata.failed_steps") or [] + set_fields["quality"] = run_quality( + FailedStep.from_dict(f) for f in failed_steps if isinstance(f, dict) + ).to_dict() return UpdateSpec(set=set_fields, unset=set(ops.unsets)) result = compare_and_swap( diff --git a/tst/unit/task/run_quality_test.py b/tst/unit/task/run_quality_test.py new file mode 100644 index 000000000..330a8ff0c --- /dev/null +++ b/tst/unit/task/run_quality_test.py @@ -0,0 +1,102 @@ +"""A completed run carries ``quality``: degraded when it tolerated a step failure that nothing repaired.""" + +from __future__ import annotations + +import pytest + +from agent_env.config import set_document_store +from agent_env.store import LocalSqliteDocumentStore +from agent_env.store.document_store import Filter +from agent_env.task.outcome import FailedStep, RunQuality, StepRole, run_quality +from agent_env.task.store import TASK_INSTANCES_COLLECTION, TaskInstance, TaskInstanceStore, set_task_instance_store +from agent_env.task.task import Task +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import RetryConfig +from agent_env.task_step.task_steps.verifiers import AgentPromptResponseVerifierTaskStep + + +def _failed(role: StepRole, **fields) -> FailedStep: + return FailedStep(**{ + "step_id": role.value, "step_type": "t", "error": "boom", "error_type": "RuntimeError", + "started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False, + "step_role": role, **fields, + }) + + +def test_a_run_with_no_failures_is_clean(): + assert run_quality([]) == RunQuality() + assert RunQuality().to_dict() == {"degraded": False, "reasons": []} + + +def test_open_tolerated_failures_degrade_the_run_most_severe_first(): + quality = run_quality([_failed(StepRole.EVIDENCE), _failed(StepRole.GRADER), _failed(StepRole.EVIDENCE)]) + assert quality.to_dict() == {"degraded": True, "reasons": ["grader", "evidence"]} + + +@pytest.mark.parametrize("fields", [{"is_fatal": True}, {"retried": True}, {"resolved_by": "regrade:2026-01-02"}]) +def test_a_fatal_retried_or_resolved_failure_does_not_degrade_the_run(fields): + assert not run_quality([_failed(StepRole.GRADER, **fields)]).degraded + + +def test_an_entry_written_before_roles_existed_degrades_as_other(): + legacy = {"step_id": "s", "step_type": "run_code", "error": "boom", "error_type": "RuntimeError", + "started_at_utc": "2026-01-01T00:00:00+00:00", "duration_seconds": 1.0, "is_fatal": False} + assert run_quality([FailedStep.from_dict(legacy)]).to_dict() == {"degraded": True, "reasons": ["other"]} + + +def test_a_run_completed_before_quality_existed_reads_as_clean(): + doc = {"instance_id": "i", "task_id": "t", "task_version": 1, "status": "completed", + "current_step": 1, "total_steps": 1} + assert TaskInstance.from_dict(doc).quality == RunQuality() + + +class _FailingVerifier(AgentPromptResponseVerifierTaskStep): + def __init__(self, *args, fail_times: int, **kwargs): + super().__init__(*args, **kwargs) + self.fail_times = fail_times + + async def execute(self, context): + if self.fail_times: + self.fail_times -= 1 + raise RuntimeError("Judge A2A task failed: API Error: 503 Service Unavailable") + return context + + +@pytest.fixture +def store(tmp_path): + doc = LocalSqliteDocumentStore(str(tmp_path / "instances.db")) + doc.ensure_index(TASK_INSTANCES_COLLECTION, ["instance_id"], unique=True) + set_document_store(doc) + s = TaskInstanceStore() + set_task_instance_store(s) + try: + yield s + finally: + set_task_instance_store(None) + + +def _stored_quality(store, instance_id): + return store._doc_store.find_one(TASK_INSTANCES_COLLECTION, Filter.of(instance_id=instance_id))["quality"] + + +@pytest.mark.asyncio +async def test_completing_with_a_tolerated_grader_failure_stamps_a_degraded_quality(store): + grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1) + ctx = TaskStepContext() + await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + assert _stored_quality(store, ctx.instance_id) == {"degraded": True, "reasons": ["grader"]} + instance = store.get(ctx.instance_id) + assert instance.status == "completed" + assert instance.quality == RunQuality((StepRole.GRADER,)) + + +@pytest.mark.asyncio +async def test_completing_after_a_successful_retry_stamps_a_clean_quality(store): + grader = _FailingVerifier("grade", None, prompt_id="p", fail_task_on_error=False, fail_times=1) + grader.retry_config = RetryConfig(retry_from_step_id="grade", max_retries=1) + ctx = TaskStepContext() + result = await Task(id="t", version=1, steps=[grader]).run(context=ctx) + + assert [f.get("retried") for f in result.metadata["failed_steps"]] == [True] + assert _stored_quality(store, ctx.instance_id) == {"degraded": False, "reasons": []}