From a887e6f24b57925d04e7448de1390f37549e6f5e Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:54:18 +0800 Subject: [PATCH 1/2] fix: return from exhausted API fallback Treat API readiness as current key eligibility instead of configuration presence, and stop projecting an exhausted pool as active. Add recovery coverage for returning to a subscription while api_fallback remains enabled. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- .../loopx-codex-provider-routing/NATIVE.md | 7 +++++++ .../extension.toml | 2 +- .../pyproject.toml | 2 +- .../smoke/native-policy.test.mjs | 7 +++++++ .../native-extension.toml | 2 +- .../loopx_codex_provider_routing/native.py | 5 ++++- .../native_policy.ts | 8 +++++++- tests/presentation/test_openai_fallback.py | 20 +++++++++++++++++++ 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/loopx-codex-provider-routing/NATIVE.md b/packages/loopx-codex-provider-routing/NATIVE.md index 79f5039036..a688e3fab6 100644 --- a/packages/loopx-codex-provider-routing/NATIVE.md +++ b/packages/loopx-codex-provider-routing/NATIVE.md @@ -163,6 +163,13 @@ eligible and a subsequent accepted request clears its preference penalty. Selection reuses the existing private runtime observations, adds no probes, and never retries an admitted stream or overrides the subscription/API role. +Version 0.15.4 treats an API route as ready only while at least one configured +key is currently eligible after authentication, rate-limit and billing +observations. If the active API pool becomes unavailable, an available +subscription is selected again; when none is available, status reports +`all_unavailable` instead of claiming `api_active`. `api_fallback` stays enabled +through this transition. + **Check connection** only requests the official [`GET /v1/models`](https://developers.openai.com/api/reference/resources/models/methods/list). It makes no inference call and does not establish spendable quota. Its 401, 403, diff --git a/packages/loopx-codex-provider-routing/extension.toml b/packages/loopx-codex-provider-routing/extension.toml index 19c4d0d7de..d4ef58c8b1 100644 --- a/packages/loopx-codex-provider-routing/extension.toml +++ b/packages/loopx-codex-provider-routing/extension.toml @@ -1,6 +1,6 @@ schema_version = "loopx_extension_manifest_v0" id = "loopx-codex-provider-routing" -version = "0.15.3" +version = "0.15.4" requires_loopx_api = ">=1,<2" permissions = [] diff --git a/packages/loopx-codex-provider-routing/pyproject.toml b/packages/loopx-codex-provider-routing/pyproject.toml index 43727653c7..945ac14367 100644 --- a/packages/loopx-codex-provider-routing/pyproject.toml +++ b/packages/loopx-codex-provider-routing/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "loopx-codex-provider-routing" -version = "0.15.3" +version = "0.15.4" description = "Native Codex account slots and exhausted-account handoff for LoopX" readme = "README.md" requires-python = ">=3.11" diff --git a/packages/loopx-codex-provider-routing/smoke/native-policy.test.mjs b/packages/loopx-codex-provider-routing/smoke/native-policy.test.mjs index 17428be2dd..e5adb08525 100644 --- a/packages/loopx-codex-provider-routing/smoke/native-policy.test.mjs +++ b/packages/loopx-codex-provider-routing/smoke/native-policy.test.mjs @@ -159,4 +159,11 @@ test("API is opt-in and strictly after every measured subscription; recovery ret assert.equal(decide(input({policy: enabled, api, observations: [sample("a", 100), sample("b", 100), {...sample("c", 100), observed_at: now - 121}]})).target, null); assert.equal(decide(input({policy: enabled, api, active: api.slot, observations: exhausted})).action, "api_active"); assert.equal(decide(input({policy: enabled, api, active: api.slot, observations: [sample("a", 80), sample("b", 20), sample("c", 100)]})).target, "b"); + const exhaustedApi = {...api, ready: false}; + assert.equal(decide(input({policy: enabled, api: exhaustedApi, active: api.slot, + observations: [sample("a", 80), sample("b", 20), sample("c", 100)]})).target, "b", + "An exhausted API pool returns to an available subscription"); + assert.equal(decide(input({policy: enabled, api: exhaustedApi, active: api.slot, + observations: exhausted})).action, "all_unavailable", + "An exhausted API pool is never projected as actively callable"); }); diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml index b6706285c9..67d6d014c9 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native-extension.toml @@ -1,6 +1,6 @@ schema_version = "loopx_extension_manifest_v0" id = "loopx-native-account-provider" -version = "0.15.3" +version = "0.15.4" requires_loopx_api = ">=1,<2" permissions = ["model-provider.local-read", "model-provider.transport-manage"] diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py index d7750a2343..e9979843b5 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py @@ -456,8 +456,11 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None, # destination. Candidates stay strictly inside the allowlist. api = None if state["policy"].get("api_fallback") or native.get("api_slot"): + # A saved key pool is not proof that a request can proceed: + # authentication, rate-limit, and billing exhaustion all + # remove keys from the current candidate set. ready = bool(state["policy"].get("api_fallback") and native.get("api_fallback_supported") - and self.api_keys().status()["keys"]) + and self.api_keys().candidates("")) api = {"slot": native.get("api_slot") or "openai-api", "ready": ready} slots = list(dict.fromkeys([native["active_slot"], *state["policy"]["slots"]])) slots = [slot for slot in slots if slot != native.get("api_slot")] diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts index f5a540d660..d5595c9018 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts @@ -205,7 +205,13 @@ export function decide(input: { ? [input.manual_slot].find(slot => slot !== input.active && allowed.includes(slot)) : candidates.sort((a, b) => b.remaining - a.remaining)[0]?.slot; if (target) return output("switch", target); - if (!input.manual_slot && onApi) return output(input.policy.api_fallback ? "api_active" : "api_disabled_active", null, input.now + 60); + if (!input.manual_slot && onApi) { + if (input.policy.api_fallback !== true) return output("api_disabled_active", null, input.now + 60); + // No subscription was selectable above. Keep an actually callable API + // route active, but do not claim availability when every key is cooling or + // exhausted; the normal all-unavailable wait owns that state. + if (input.api?.ready === true) return output("api_active", null, input.now + 60); + } // A failed/stale subscription read is not permission to incur API costs. // Every authorized subscription must have fresh, measured exhaustion. if (!input.manual_slot && input.policy.api_fallback === true && input.api?.ready diff --git a/tests/presentation/test_openai_fallback.py b/tests/presentation/test_openai_fallback.py index 32b49ee495..7f14b33cca 100644 --- a/tests/presentation/test_openai_fallback.py +++ b/tests/presentation/test_openai_fallback.py @@ -60,6 +60,26 @@ def test_automatic_api_handoff_then_recovered_subscription_adoption(setup): assert state["calls"].count("switch") == 2 +def test_exhausted_api_pool_returns_to_a_recovered_subscription(setup): + provider, state = setup + provider.configure(expected_revision=0, enabled=True, slots=["a", "b"], api_fallback=True) + assert provider.tick(refresh=True)["active_slot"] == "openai-api" + + key = provider.api_keys().status()["keys"][0] + provider.api_keys().record(key["id"], status=429, code="insufficient_quota", + model="selected-model", headers={"retry-after": "3600"}) + state["now"] = 1301 # Past the account-switch cooldown but not the key cooldown. + blocked = provider.tick(refresh=True) + assert blocked["active_slot"] == "openai-api" + assert blocked["action"] == "all_unavailable" + assert state["calls"].count("switch") == 1, "No fallback may claim readiness while every key is exhausted" + + state["b_used"] = 10 + recovered = provider.tick(refresh=True) + assert recovered["active_slot"] == "b" and recovered["api_active"] is False + assert state["calls"].count("switch") == 2 + + def test_available_or_unknown_subscription_never_starts_a_paid_handoff(setup): provider, state = setup provider.configure(expected_revision=0, enabled=True, slots=["a", "b"], api_fallback=True) From c158b438e98546c51c59f60a15f4629825619aef Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Tue, 10 Nov 2026 12:17:20 +0800 Subject: [PATCH 2/2] fix(provider-routing): sample live key eligibility and bound recovery waits Signed-off-by: huangruiteng --- .../loopx_codex_provider_routing/native.py | 19 ++++--- .../native_policy.ts | 3 +- tests/presentation/test_openai_fallback.py | 55 ++++++++++++++++++- 3 files changed, 66 insertions(+), 11 deletions(-) diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py index 54b7ad9661..3545161af3 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native.py @@ -371,7 +371,11 @@ def _validate_api(self, native: dict) -> None: def _prepare_api(self, state: dict, native: dict) -> dict: self._validate_api(native) from .openai_relay import ensure + if not self.api_keys().candidates(""): + raise HostError("api_relay_unavailable") endpoint = ensure(self) + if not self.api_keys().candidates(""): + raise HostError("api_relay_unavailable") if native.get("api_slot") is None: return self.transport(state["adapter"], "enroll_api", **endpoint) return native @@ -459,14 +463,6 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None, state["failure_count"] = 0 # Inspect the active account even when it is not an authorized # destination. Candidates stay strictly inside the allowlist. - api = None - if state["policy"].get("api_fallback") or native.get("api_slot"): - # A saved key pool is not proof that a request can proceed: - # authentication, rate-limit, and billing exhaustion all - # remove keys from the current candidate set. - ready = bool(state["policy"].get("api_fallback") and native.get("api_fallback_supported") - and self.api_keys().candidates("")) - api = {"slot": native.get("api_slot") or "openai-api", "ready": ready} slots = list(dict.fromkeys([native["active_slot"], *state["policy"]["slots"]])) slots = [slot for slot in slots if slot != native.get("api_slot")] failures = {} if refresh or manual_slot else { @@ -476,6 +472,13 @@ def tick(self, *, refresh: bool = False, manual_slot: str | None = None, } observed = self.transport(state["adapter"], "observe", slots=slots, expected_revision=native["selection_revision"], auth_failures=failures) + api = None + if state["policy"].get("api_fallback") or native.get("api_slot"): + # External observation can outlive a key's eligibility. + # Sample candidates afterwards, before the typed decision. + ready = bool(state["policy"].get("api_fallback") and native.get("api_fallback_supported") + and self.api_keys().candidates("")) + api = {"slot": native.get("api_slot") or "openai-api", "ready": ready} public_observations = [{k: v for k, v in row.items() if k != "credential_revision"} for row in observed["observations"]] decision = policy({"now": self.clock(), "policy": state["policy"], "active": native["active_slot"], diff --git a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts index d5595c9018..76b0d79222 100644 --- a/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts +++ b/packages/loopx-codex-provider-routing/src/loopx_codex_provider_routing/native_policy.ts @@ -222,5 +222,6 @@ export function decide(input: { .flatMap(a => a.reset_at && a.reset_at > input.now ? [a.reset_at] : []); const uncertain = accounts.some(a => allowed.includes(a.slot) && a.availability === "unknown"); return output(input.manual_slot ? "target_unavailable" : "all_unavailable", null, - Math.min(input.now + (uncertain ? 60 : 300), ...(resets.length ? resets : [Infinity]))); + Math.min(input.now + (uncertain || (onApi && input.policy.api_fallback === true) ? 60 : 300), + ...(resets.length ? resets : [Infinity]))); } diff --git a/tests/presentation/test_openai_fallback.py b/tests/presentation/test_openai_fallback.py index 7f14b33cca..d69ef5095c 100644 --- a/tests/presentation/test_openai_fallback.py +++ b/tests/presentation/test_openai_fallback.py @@ -69,17 +69,68 @@ def test_exhausted_api_pool_returns_to_a_recovered_subscription(setup): provider.api_keys().record(key["id"], status=429, code="insufficient_quota", model="selected-model", headers={"retry-after": "3600"}) state["now"] = 1301 # Past the account-switch cooldown but not the key cooldown. - blocked = provider.tick(refresh=True) + blocked = provider.tick() assert blocked["active_slot"] == "openai-api" assert blocked["action"] == "all_unavailable" + assert blocked["retry_at"] <= state["now"] + 60 assert state["calls"].count("switch") == 1, "No fallback may claim readiness while every key is exhausted" state["b_used"] = 10 - recovered = provider.tick(refresh=True) + state["now"] += 60 + recovered = provider.tick() assert recovered["active_slot"] == "b" and recovered["api_active"] is False assert state["calls"].count("switch") == 2 +def test_temporary_api_cooldown_is_rechecked_without_manual_refresh(setup): + provider, state = setup + provider.configure(expected_revision=0, enabled=True, slots=["a", "b"], api_fallback=True) + assert provider.tick(refresh=True)["active_slot"] == "openai-api" + state["now"] = 1301 + key = provider.api_keys().status()["keys"][0] + provider.api_keys().record(key["id"], status=429, code="rate_limit_exceeded", + model="selected-model", headers={"retry-after": "30"}) + assert provider.tick()["action"] == "all_unavailable" + state["now"] += 60 + assert provider.tick()["action"] == "api_active" + assert state["calls"].count("switch") == 1 + + +@pytest.mark.parametrize("during", ["observe", "prepare"]) +def test_last_key_invalidation_cannot_prepare_or_switch_a_paid_route(setup, monkeypatch, during): + provider, state = setup + provider.configure(expected_revision=0, enabled=True, slots=["a", "b"], api_fallback=True) + original_transport = provider.transport + + def invalidate(): + key = provider.api_keys().status()["keys"][0] + provider.api_keys().record(key["id"], status=429, code="insufficient_quota", + model="selected-model", headers={"retry-after": "3600"}) + + def transport(adapter, operation, **parameters): + result = original_transport(adapter, operation, **parameters) + if operation == "observe" and during == "observe": + invalidate() + return result + + provider.transport = transport + if during == "prepare": + relay = importlib.import_module("loopx_codex_provider_routing.openai_relay") + original_ensure = relay.ensure + + def ensure(owner): + result = original_ensure(owner) + invalidate() + return result + + monkeypatch.setattr(relay, "ensure", ensure) + provider.tick() + assert not provider.api_keys().candidates("") + assert state["active"] == "a" + assert "switch" not in state["calls"] and "enroll_api" not in state["calls"] + assert state["relay_starts"] == (1 if during == "prepare" else 0) + + def test_available_or_unknown_subscription_never_starts_a_paid_handoff(setup): provider, state = setup provider.configure(expected_revision=0, enabled=True, slots=["a", "b"], api_fallback=True)