Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions tests/e2e_cuj/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -291,3 +291,9 @@ inference, successful paired responses, and a selected offered model applied to
These fixtures have small catalogs. The allow-all Bedrock startup and oversized-catalog cases
remain separate staging validation and are not established by these CUJs.
CI pins Codex 0.165.0 for CUJ7, matching CUJ4's routing-capable version.

Saved-provider routing checks retain each matched request's router name, offered model/harness
pairs, response status and gateway request ID in `routing-request-<sequence>.json`, including
failed responses. Task prompts, arbitrary headers and response bodies are excluded.
Capture runs after task completion, as each routing response reaches validation. A failure
before validation or on an earlier response can prevent later routing artifacts.
11 changes: 11 additions & 0 deletions tests/e2e_cuj/test_cuj7_model_discovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,17 @@ def _assert_saved_provider_routing(session, recorder, checkpoint, agent, provide
options = route.payload["route_options"]
assert 1 <= len(options) <= 128
response = recorder.response_for(route)
session.record(
f"routing-request-{route.sequence}.json",
{
"router_name": route.payload.get("route_selector", {}).get("router_name"),
"route_options": [
{key: option.get(key) for key in ("model", "harness")} for option in options
],
"status_code": response.status_code,
"request_id": response.headers.get("x-request-id"),
},
)
assert response.status_code == 200, response.payload
selections = response.payload["route_selection"]
assert len(selections) == 1, selections
Expand Down
41 changes: 40 additions & 1 deletion tests/test_cuj_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import pytest

from tests.e2e_cuj.helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS
from tests.e2e_cuj.helpers.constants import CLAUDE, CODEX, INFERENCE_PATHS, PROVIDER_HEADER
from tests.e2e_cuj.helpers.evidence import (
BaseCujHelper,
ClaudeCujHelper,
Expand All @@ -25,10 +25,49 @@
_smart_router_hint_shown,
_task_inference_request,
)
from tests.e2e_cuj.test_cuj7_model_discovery import _assert_saved_provider_routing
from tests.integration.utils.evidence import FileTask, read_jsonl
from tests.integration.utils.harness import UserSession
from tests.integration.utils.provider_catalog import MODEL_SERVICE_PARENT_SCHEMA_HEADER


def test_failed_saved_provider_route_retains_metadata_without_prompt_or_credentials(tmp_path):
session = UserSession(tmp_path, tmp_path, "ug", tmp_path / "artifacts")
session.env["DATABRICKS_BEARER"] = "private-bearer"
task = SimpleNamespace(filename="task.txt")
route = RecordedRequest(
7,
"POST",
"/ai-gateway/routing/v1/routes:select",
{PROVIDER_HEADER: "provider"},
json.dumps(
{
"task": {"prompt": "private prompt: task.txt"},
"route_selector": {"router_name": "recipe", "extra": "private-selector"},
"route_options": [{"model": "model", "harness": "codex", "extra": "private"}],
}
).encode(),
)
response = RecordedResponse(
400,
{"x-request-id": "request-id", "authorization": "private-bearer"},
b'{"error_code":"BAD_REQUEST","message":"private-error"}',
)
recorder = SimpleNamespace(requests_after=lambda _: [route], response_for=lambda _: response)

with pytest.raises(AssertionError, match="BAD_REQUEST"):
_assert_saved_provider_routing(session, recorder, 0, "codex", "provider", "model", task)

evidence = json.loads((session.artifacts / "routing-request-7.json").read_text())
assert evidence == {
"router_name": "recipe",
"route_options": [{"model": "model", "harness": "codex"}],
"status_code": 400,
"request_id": "request-id",
}
assert "private" not in json.dumps(evidence)


@pytest.mark.parametrize("enabled, first_prompt", [(True, True), (True, False), (False, False)])
def test_cuj4_status_band_requires_current_state_and_scope(enabled, first_prompt):
state = "ON" if enabled else "OFF"
Expand Down
Loading