diff --git a/tests/e2e_cuj/README.md b/tests/e2e_cuj/README.md index 5a7828808..b77bfaad2 100644 --- a/tests/e2e_cuj/README.md +++ b/tests/e2e_cuj/README.md @@ -291,3 +291,9 @@ inference, successful paired responses, and a selected offered model applied to These fixtures have small catalogs. The allow-all Bedrock startup and oversized-catalog cases remain separate staging validation and are not established by these CUJs. CI pins Codex 0.165.0 for CUJ7, matching CUJ4's routing-capable version. + +Saved-provider routing checks retain each matched request's router name, offered model/harness +pairs, response status and gateway request ID in `routing-request-.json`, including +failed responses. Task prompts, arbitrary headers and response bodies are excluded. +Capture runs after task completion, as each routing response reaches validation. A failure +before validation or on an earlier response can prevent later routing artifacts. diff --git a/tests/e2e_cuj/test_cuj7_model_discovery.py b/tests/e2e_cuj/test_cuj7_model_discovery.py index 3ccce08a9..976a15e31 100644 --- a/tests/e2e_cuj/test_cuj7_model_discovery.py +++ b/tests/e2e_cuj/test_cuj7_model_discovery.py @@ -474,6 +474,17 @@ def _assert_saved_provider_routing(session, recorder, checkpoint, agent, provide options = route.payload["route_options"] assert 1 <= len(options) <= 128 response = recorder.response_for(route) + session.record( + f"routing-request-{route.sequence}.json", + { + "router_name": route.payload.get("route_selector", {}).get("router_name"), + "route_options": [ + {key: option.get(key) for key in ("model", "harness")} for option in options + ], + "status_code": response.status_code, + "request_id": response.headers.get("x-request-id"), + }, + ) assert response.status_code == 200, response.payload selections = response.payload["route_selection"] assert len(selections) == 1, selections diff --git a/tests/test_cuj_evidence.py b/tests/test_cuj_evidence.py index af5ce0185..9c37786b9 100644 --- a/tests/test_cuj_evidence.py +++ b/tests/test_cuj_evidence.py @@ -6,7 +6,7 @@ import pytest -from tests.e2e_cuj.helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS +from tests.e2e_cuj.helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS, PROVIDER_HEADER from tests.e2e_cuj.helpers.evidence import ( BaseCujHelper, ClaudeCujHelper, @@ -25,10 +25,49 @@ _smart_router_hint_shown, _task_inference_request, ) +from tests.e2e_cuj.test_cuj7_model_discovery import _assert_saved_provider_routing from tests.integration.utils.evidence import FileTask, read_jsonl +from tests.integration.utils.harness import UserSession from tests.integration.utils.provider_catalog import MODEL_SERVICE_PARENT_SCHEMA_HEADER +def test_failed_saved_provider_route_retains_metadata_without_prompt_or_credentials(tmp_path): + session = UserSession(tmp_path, tmp_path, "ug", tmp_path / "artifacts") + session.env["DATABRICKS_BEARER"] = "private-bearer" + task = SimpleNamespace(filename="task.txt") + route = RecordedRequest( + 7, + "POST", + "/ai-gateway/routing/v1/routes:select", + {PROVIDER_HEADER: "provider"}, + json.dumps( + { + "task": {"prompt": "private prompt: task.txt"}, + "route_selector": {"router_name": "recipe", "extra": "private-selector"}, + "route_options": [{"model": "model", "harness": "codex", "extra": "private"}], + } + ).encode(), + ) + response = RecordedResponse( + 400, + {"x-request-id": "request-id", "authorization": "private-bearer"}, + b'{"error_code":"BAD_REQUEST","message":"private-error"}', + ) + recorder = SimpleNamespace(requests_after=lambda _: [route], response_for=lambda _: response) + + with pytest.raises(AssertionError, match="BAD_REQUEST"): + _assert_saved_provider_routing(session, recorder, 0, "codex", "provider", "model", task) + + evidence = json.loads((session.artifacts / "routing-request-7.json").read_text()) + assert evidence == { + "router_name": "recipe", + "route_options": [{"model": "model", "harness": "codex"}], + "status_code": 400, + "request_id": "request-id", + } + assert "private" not in json.dumps(evidence) + + @pytest.mark.parametrize("enabled, first_prompt", [(True, True), (True, False), (False, False)]) def test_cuj4_status_band_requires_current_state_and_scope(enabled, first_prompt): state = "ON" if enabled else "OFF"