From 6339f588f681ff992b7b605ea4a7c21f783d97f7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 15 Nov 2026 07:13:35 +0000 Subject: [PATCH] fix(a2a_agent): retry a failed trajectory read a bounded number of times after the turn ends, and make read failures alertable --- .../a2a_agent/trajectory_follower.py | 35 ++++++++--- .../a2a_agent/test_trajectory_follower.py | 59 ++++++++++++++++++- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/agent_env/a2a_agent/trajectory_follower.py b/src/agent_env/a2a_agent/trajectory_follower.py index 01ffbdf02..cd3cd9a27 100644 --- a/src/agent_env/a2a_agent/trajectory_follower.py +++ b/src/agent_env/a2a_agent/trajectory_follower.py @@ -30,6 +30,11 @@ _MAX_BACKOFF_SECONDS = 60 # Once stopped, the task has ended and its log is sealed, so the next read is normally the last. _STOPPED_INTERVAL_SECONDS = 0.2 +# After the task has ended, a failing read gets a short, bounded drain: enough to ride out a blip +# without losing the trajectory's tail, short enough that a gone agent frees the step. +_STOPPED_RETRY_SECONDS = 1.0 +_STOPPED_MAX_FAILURES = 3 +_ERROR_AFTER_FAILURES = 10 @dataclass(frozen=True) @@ -59,6 +64,16 @@ def live_trajectory_endpoint(card: dict, a2a_url: str) -> str | None: return a2a_url + path +def _log_read_failure(task_id: str, failures: int, exc: Exception) -> None: + logger.log( + logging.ERROR if failures >= _ERROR_AFTER_FAILURES else logging.WARNING, + "Live trajectory read failed (consec=%d, will retry): %r", + failures, + exc, + extra={"event": "live_trajectory_read_failed", "consec": failures, "task_id": task_id}, + ) + + async def follow_trajectory( endpoint: str, task_id: str, @@ -72,8 +87,8 @@ async def follow_trajectory( events and the last one, until the task has ended and every event is read. Between reads it waits ``poll_interval_seconds``, or less once ``stop`` is set. Transport errors - and 5xx answers are retried with backoff until ``stop`` is set, and end following after it; a - task the agent no longer knows (404) ends following.""" + and 5xx answers are retried with backoff. Once ``stop`` is set they are retried a few more times, + then end following; a task the agent no longer knows (404) ends following.""" after = 0 failures = 0 async with nullcontext(client) if client is not None else httpx.AsyncClient() as http: @@ -90,10 +105,10 @@ async def follow_trajectory( if exc.response.status_code < 500: raise failures += 1 - logger.warning("Live trajectory read failed (consec=%d, will retry): %s", failures, exc) + _log_read_failure(task_id, failures, exc) except httpx.TransportError as exc: failures += 1 - logger.warning("Live trajectory read failed (consec=%d, will retry): %r", failures, exc) + _log_read_failure(task_id, failures, exc) else: failures = 0 page = parse_response(TrajectoryEventsResponse, response.json(), operation="trajectory get") @@ -115,12 +130,14 @@ async def follow_trajectory( if page.has_more: continue if stop.is_set(): - if failures: - # The turn has ended, so an agent that stops answering now is gone, not busy: retrying - # would poll a dead address until the step's drain gives up. The step marks the turn's end. - logger.warning("Agent stopped answering after task %s ended; stopped following its trajectory", task_id) + if failures >= _STOPPED_MAX_FAILURES: + logger.warning( + "Agent stopped answering after task %s ended; stopped following its trajectory", + task_id, + extra={"event": "live_trajectory_follow_abandoned", "consec": failures, "task_id": task_id}, + ) return - await asyncio.sleep(_STOPPED_INTERVAL_SECONDS) + await asyncio.sleep(_STOPPED_RETRY_SECONDS * failures or _STOPPED_INTERVAL_SECONDS) continue # Not `wait_for`: on 3.11 it drops a cancel that lands as `stop` is set, the step's own # teardown, and the follower then outlives its step. diff --git a/tst/unit/a2a_agent/test_trajectory_follower.py b/tst/unit/a2a_agent/test_trajectory_follower.py index 8aac86070..a3a6848dc 100644 --- a/tst/unit/a2a_agent/test_trajectory_follower.py +++ b/tst/unit/a2a_agent/test_trajectory_follower.py @@ -1,6 +1,7 @@ import asyncio import contextlib import json +import logging import time import httpx @@ -167,7 +168,8 @@ async def test_a_follower_cancelled_as_it_is_stopped_ends(): @pytest.mark.asyncio -async def test_an_agent_that_fails_after_the_turn_ends_ends_following(): +async def test_an_agent_that_fails_after_the_turn_ends_is_retried_a_bounded_number_of_times(monkeypatch): + monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._STOPPED_RETRY_SECONDS", 0.01) agent = _Agent(httpx.Response(200, json=_page(0, [], "running")), *[httpx.Response(503)] * 100) stop = asyncio.Event() async with _following(agent, stop, poll_interval_seconds=30) as follower: @@ -176,7 +178,60 @@ async def test_an_agent_that_fails_after_the_turn_ends_ends_following(): async with asyncio.timeout(2): await follower - assert agent.afters == [0, 0] + assert agent.afters == [0] * 4 + + +@pytest.mark.asyncio +async def test_an_agent_that_recovers_after_the_turn_ends_still_gets_its_tail_read(monkeypatch): + monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._STOPPED_RETRY_SECONDS", 0.01) + agent = _Agent( + httpx.Response(200, json=_page(0, [], "running")), + httpx.Response(503), + httpx.Response(200, json=_page(0, [{"i": 0}], "completed")), + ) + batches: list[TrajectoryBatch] = [] + + async def sink(batch: TrajectoryBatch) -> None: + batches.append(batch) + + stop = asyncio.Event() + async with agent.client() as client: + follower = asyncio.create_task( + follow_trajectory(_ENDPOINT, "task-1", sink, poll_interval_seconds=30, stop=stop, client=client) + ) + try: + async with asyncio.timeout(5): + while not agent.afters: + await asyncio.sleep(0.01) + stop.set() + async with asyncio.timeout(2): + await follower + finally: + follower.cancel() + await asyncio.wait({follower}) + + assert batches[-1].events == [{"i": 0}] + + +@pytest.mark.asyncio +async def test_a_failing_read_is_logged_with_its_count_and_escalates_to_error(monkeypatch, caplog): + monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._MAX_BACKOFF_SECONDS", 0) + monkeypatch.setattr("agent_env.a2a_agent.trajectory_follower._ERROR_AFTER_FAILURES", 3) + agent = _Agent( + *[httpx.Response(503)] * 4, + httpx.Response(200, json=_page(0, [], "completed")), + ) + + with caplog.at_level(logging.WARNING, logger="agent_env.a2a_agent.trajectory_follower"): + await _follow(agent) + + failed = [r for r in caplog.records if getattr(r, "event", None) == "live_trajectory_read_failed"] + assert [(r.consec, r.levelno, r.task_id) for r in failed] == [ + (1, logging.WARNING, "task-1"), + (2, logging.WARNING, "task-1"), + (3, logging.ERROR, "task-1"), + (4, logging.ERROR, "task-1"), + ] @pytest.mark.asyncio