Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/reference/protocols/loopx-turn-v0.md
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,12 @@ check. Retryable Host failures also carry a content-free
Host invocation, and the TypeScript recovery decision rejects another
invocation after the declared bounded budget. Backoff is an outer-scheduler
hint; `run-once` never sleeps in-process or silently changes the selected model.
The default `run-once` Markdown readback shows every recorded Host failure's
kind, retryability and attempt, plus the retry ceiling and backoff when present.
These are the existing journal facts, not permission to launch another attempt:
even a retryable class may have exhausted its budget. Replaying the original
Turn exposes the same diagnostics without re-running the Host. JSON and the
admission policy are unchanged; raw provider messages are not added to readback.
A dangling prepared effect remains owned by the existing provider
readback protocol; the recovery decision only records that the readback is the
next required check and does not claim general exactly-once execution.
Expand Down
20 changes: 13 additions & 7 deletions loopx/cli_commands/turn_rendering.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,10 @@ def render_loopx_turn_execution_markdown(payload: dict[str, object]) -> str:
if isinstance(managed_executor.get("output_token_budget"), dict)
else {}
)
host_failure = (
payload.get("host_failure")
if isinstance(payload.get("host_failure"), dict)
else {}
)
raw_host_failure = payload.get("host_failure")
host_failure: dict[str, Any] = raw_host_failure if isinstance(raw_host_failure, dict) else {}
retry = host_failure.get("retry")
retry = retry if isinstance(retry, dict) else {}
journal_observation = (
payload.get("journal_observation")
if isinstance(payload.get("journal_observation"), dict) else {}
Expand All @@ -239,8 +238,15 @@ def render_loopx_turn_execution_markdown(payload: dict[str, object]) -> str:
*(
[f"- host_failure_kind: {host_failure.get('kind')}",
f"- host_failure_retryable: {host_failure.get('retryable')}",
f"- failure_reason: {payload.get('reason')}"]
if host_failure.get("kind") == "output_budget_exhausted" else []
*([f"- host_failure_attempt: {host_failure['attempt']}"]
if "attempt" in host_failure else []),
*([f"- host_failure_max_attempts: {retry.get('max_attempts')}",
f"- host_failure_backoff_seconds: {retry.get('backoff_seconds')}",
"Retry policy is a scheduling hint. Recovery must still be admitted against the original Turn journal."]
if retry else []),
*([f"- failure_reason: {payload.get('reason')}"]
if host_failure.get("kind") == "output_budget_exhausted" else [])]
if host_failure else []
),
f"- validation: {validation.get('status')}",
f"- recovery_kind: {validation.get('recovery_kind')}",
Expand Down
71 changes: 71 additions & 0 deletions tests/cli_commands/test_turn_failure_readback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""Failure readback must not require another execution or expose provider prose."""
from __future__ import annotations

import contextlib
import io
import json

import pytest

from loopx.cli import main
from loopx.cli_commands.turn_rendering import render_loopx_turn_execution_markdown
from loopx.control_plane.turn_driver.host_failure import BuiltInHostError, build_host_failure_record
from tests.test_loopx_turn_driver import _write_live_fixture


@pytest.mark.parametrize("kind", [
"auth_failed", "contract_rejected", "executor_timeout", "output_budget_exhausted",
"provider_capacity", "provider_overloaded", "quota_exhausted", "rate_limited",
"session_missing", "transport_lost", "unknown",
])
def test_every_classified_failure_is_visible_without_raw_provider_details(kind):
failure = build_host_failure_record(kind, attempt=1)
payload = {"status": "failed", "result_kind": "host_failure", "host_failure": failure,
"provider_message": "private provider detail"}
before = json.dumps(payload, sort_keys=True)
rendered = render_loopx_turn_execution_markdown(payload)
assert f"- host_failure_kind: {kind}" in rendered
assert f"- host_failure_retryable: {failure['retryable']}" in rendered
assert "- host_failure_attempt: 1" in rendered
assert "private provider detail" not in rendered
assert json.dumps(payload, sort_keys=True) == before


def test_exhausted_retry_policy_is_a_hint_not_permission_to_execute():
rendered = render_loopx_turn_execution_markdown({
"status": "failed", "host_failure": build_host_failure_record("rate_limited", attempt=3),
})
assert "- host_failure_attempt: 3" in rendered
assert "- host_failure_max_attempts: 3" in rendered
assert "- host_failure_backoff_seconds: 240" in rendered
assert "Recovery must still be admitted against the original Turn journal." in rendered


@pytest.mark.parametrize("kind", ["auth_failed", "rate_limited", "executor_timeout"])
def test_cli_replays_failed_journal_as_markdown_without_another_host_call(tmp_path, monkeypatch, kind):
project, runtime, registry = _write_live_fixture(tmp_path)
calls = []

def unavailable_host(request, **kwargs):
calls.append(request["turn_key"])
raise BuiltInHostError(f"codex_cli_{kind}", failure_kind=kind)

monkeypatch.setattr("loopx.cli_commands.turn_run_once.run_codex_cli_host", unavailable_host)
args = ["--registry", str(registry), "--runtime-root", str(runtime), "turn", "run-once",
"--goal-id", "loopx-turn-fixture", "--agent-id", "codex-fixture",
"--host", "codex-cli", "--project", str(project), "--no-global-sync", "--execute"]
output = io.StringIO()
with contextlib.redirect_stdout(output):
code = main([*args, "--format", "json"])
first = json.loads(output.getvalue())
assert code == 1 and first["host_failure"]["kind"] == kind
journals = {path: path.read_bytes() for path in (runtime / "goals").rglob("turns/*.json")}
assert journals
replay = io.StringIO()
with contextlib.redirect_stdout(replay):
replay_code = main([*args, "--resume-turn-key", first["resume_turn_key"], "--format", "markdown"])
assert replay_code == 1
assert f"- host_failure_kind: {kind}" in replay.getvalue()
assert "- host_failure_attempt: 1" in replay.getvalue()
assert len(calls) == 1
assert all(path.read_bytes() == original for path, original in journals.items())
3 changes: 1 addition & 2 deletions tests/test_loopx_turn_journal_inspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,12 @@ def test_existing_run_once_markdown_renderer_remains_intact() -> None:
assert "- failure_reason: dsh_output_budget_exhausted_no_final" in rendered


def test_non_budget_failure_keeps_the_existing_compact_rendering() -> None:
def test_execution_without_host_failure_keeps_the_existing_compact_rendering() -> None:
from loopx.cli_commands.turn_rendering import render_loopx_turn_execution_markdown

rendered = render_loopx_turn_execution_markdown({
"status": "failed", "result_kind": "host_failure",
"reason": "provider_failed",
"host_failure": {"kind": "auth_failed", "retryable": False},
})
assert rendered == "\n".join([
"# LoopX Turn Run Once", "- status: failed", "- result_kind: host_failure",
Expand Down
Loading