diff --git a/scripts/test-layout/layout-providers.json b/scripts/test-layout/layout-providers.json index c095709ad6d..7efbec68460 100644 --- a/scripts/test-layout/layout-providers.json +++ b/scripts/test-layout/layout-providers.json @@ -3,6 +3,7 @@ "xai-additional-tools.test.ts": "providers/xai", "xai-replay-provenance.test.ts": "providers/xai", "xai-loaded-tool-provenance.test.ts": "providers/xai", + "free-directory-llmtr.test.ts": "providers", "free-directory-runcrate.test.ts": "providers", "cc-switch-import.test.ts": "providers", "kiro-completion-with-real-tools.test.ts": "providers/kiro" diff --git a/src/providers/free-directory.ts b/src/providers/free-directory.ts index 29e88303db0..3d8c74f1ab7 100644 --- a/src/providers/free-directory.ts +++ b/src/providers/free-directory.ts @@ -1,4 +1,4 @@ -import { getProviderRegistryEntry } from "./registry"; +import { getProviderRegistryEntry, normalizedProviderEndpoint, type ProviderModelDiscoverySpec } from "./registry"; export type ProviderAccessGroup = | "recurring-or-keyless" @@ -11,7 +11,7 @@ export const FREE_PROVIDER_ACCESS_GROUPS = { "agy", "aihorde", "api-airforce", "arcee-ai", "bazaarlink", "blackbox", "bluesminds", "cerebras", "cloudflare-ai", "cohere", "coze", "duckduckgo-web", "felo-web", "friendliai", "gemini", "github-models", "groq", "hackclub", "huggingchat", "huggingface", "iflytek", "inference-net", "kiro", "liquid", "llm7", - "mistral", "morph", "muse-spark-web", "nara", "navy", "nlpcloud", "ollama-cloud", "opencode", "openrouter", + "llmtr", "mistral", "morph", "muse-spark-web", "nara", "navy", "nlpcloud", "ollama-cloud", "opencode", "openrouter", "ovhcloud", "perch", "pollinations", "puter", "qwen-web", "reka", "runcrate", "sambanova", "sparkdesk", "t3-web", "uncloseai", ], "recurring-uncapped": [ @@ -51,6 +51,11 @@ export interface FreeDirectoryProvider { anthropicEofTolerance?: boolean; note?: string; googleMode?: "ai-studio" | "vertex"; + /** + * Live-discovery policy for this exact fixed key destination. Resolved by base URL and adapter, + * never copied into saved config and never read from PROVIDER_REGISTRY. + */ + modelDiscovery?: ProviderModelDiscoverySpec; } type ConnectableOverride = Partial>; @@ -111,6 +116,13 @@ const CONNECTABLE: Record = { // WebSocket protocol and does not describe this endpoint. iflytek: openAi("https://spark-api-open.xf-yun.com/v1", "https://console.xfyun.cn", { verification: "official", documentationUrl: "https://www.xfyun.cn/doc/spark/HTTP%E8%B0%83%E7%94%A8%E6%96%87%E6%A1%A3.html" }), liquid: openAi("https://inference.liquid.ai/v1", "https://playground.liquid.ai", { modelsUrl: "https://inference.liquid.ai/v1/models" }), + // Checked 2026-10-29. Docs: https://llmtr.com/docs/gateway/chat-completions/. + // Operator: Knowhy İleri Teknoloji Ticaret Limited Şirketi (operator, per https://llmtr.com/terms). GET /v1/models is public and returned + // 382 rows. Chat requires a Bearer key. Six chat rows are the standing zero-price set; other + // zero-price rows are dated promotions or EVREN bring-your-own-key routes. Discovery is hybrid: + // those six ids are seeded, and live discovery still lists paid chat models, including + // Turkey-hosted llmtr/qwen3-8-27b. A model that accepts `reasoning` takes a boolean, not an object. + llmtr: openAi("https://llmtr.com/v1", "https://llmtr.com/dashboard/api-keys", { verification: "official", documentationUrl: "https://llmtr.com/docs/gateway/chat-completions/", modelsUrl: "https://llmtr.com/v1/models", lastVerified: "2026-10-29", discovery: "hybrid", models: ["nvidia/nemotron-3-ultra-550b-a55b", "openai/gpt-oss-safeguard-20b", "qwen/qwen3.8-27b-free", "nvidia/nemotron-3.5-super-vl-free", "agnes/agnes-3.0-flash", "agnes/agnes-2.5-flash"], modelDiscovery: { filter: { allOf: [{ path: ["supported_operations"], containsAll: ["CHAT_COMPLETIONS"] }, { path: ["supported_endpoints"], containsAll: ["/v1/chat/completions"] }] } }, note: "The six listed chat models had a zero list price on 2026-10-29. qwen/qwen3.8-27b-free, nvidia/nemotron-3-ultra-550b-a55b, and nvidia/nemotron-3.5-super-vl-free reset on a daily quota; openai/gpt-oss-safeguard-20b, agnes/agnes-2.5-flash, and agnes/agnes-3.0-flash were listed free with no end date. A key is still required. When a daily quota is used up, those requests stop until the next day. Other zero-price rows are dated promotions or EVREN bring-your-own-key routes and stay on live discovery only. Models that are neither listed above nor covered by those promotions or routes are paid at published per-token rates, including Turkey-hosted ids such as llmtr/qwen3-8-27b. Pricing: https://llmtr.com/pricing" }), mistral: openAi("https://api.mistral.ai/v1", "https://console.mistral.ai/api-keys", { supportLevel: "supported", verification: "official", documentationUrl: "https://docs.mistral.ai/api/endpoint/models" }), morph: openAi("https://api.morphllm.com/v1", "https://morphllm.com", { verification: "official", documentationUrl: "https://docs.morphllm.com/api-reference/introduction" }), "ollama-cloud": openAi("https://ollama.com/v1", "https://ollama.com/settings/keys", { supportLevel: "supported", verification: "official", documentationUrl: "https://docs.ollama.com/cloud", modelsUrl: "https://ollama.com/api/tags" }), @@ -202,7 +214,7 @@ const LABELS: Record = { bazaarlink: "BazaarLink", blackbox: "Blackbox", bluesminds: "Bluesminds", "cloudflare-ai": "Cloudflare AI", coze: "Coze", "duckduckgo-web": "DuckDuckGo Web", "felo-web": "Felo Web", friendliai: "FriendliAI", gemini: "Google Gemini", "github-models": "GitHub Models", huggingchat: "HuggingChat", huggingface: "Hugging Face", - iflytek: "iFlytek", "inference-net": "Inference.net", llm7: "LLM7", "muse-spark-web": "Muse Spark Web", + iflytek: "iFlytek", "inference-net": "Inference.net", llm7: "LLM7", llmtr: "LLMTR", "muse-spark-web": "Muse Spark Web", nlpcloud: "NLP Cloud", "ollama-cloud": "Ollama Cloud", ovhcloud: "OVHcloud", "qwen-web": "Qwen Web", "t3-web": "T3 Web", uncloseai: "UncloseAI", perch: "Perch", ainative: "AI Native", baidu: "Baidu Qianfan", glm: "Z.AI GLM", "glm-cn": "BigModel GLM (CN)", "kilo-gateway": "Kilo Gateway", "opencode-zen": "OpenCode Zen", @@ -238,3 +250,26 @@ export const FREE_PROVIDER_DIRECTORY: readonly FreeDirectoryProvider[] = Object. ...override, } satisfies FreeDirectoryProvider; }); + +/** + * Discovery policy for a directory row's exact fixed key destination. + * + * A saved provider is matched by adapter and base URL, including when it was saved under another + * name. Templated rows, OAuth rows, and a base URL shared by more than one filtered row do not + * match. The spec stays in this module: it is not a registry preset and it is not written to config. + */ +export function directoryModelDiscoveryForDestination( + provider: { baseUrl?: string; adapter?: string; authMode?: string }, +): ProviderModelDiscoverySpec | undefined { + if (typeof provider.baseUrl !== "string" || !provider.baseUrl.trim()) return undefined; + if (provider.authMode !== undefined && provider.authMode !== "key") return undefined; + const endpoint = normalizedProviderEndpoint(provider.baseUrl); + const matches = FREE_PROVIDER_DIRECTORY.filter(row => + row.modelDiscovery + && row.authKind === "key" + && row.adapter === provider.adapter + && row.baseUrl + && !/\{[^}]*\}/.test(row.baseUrl) + && normalizedProviderEndpoint(row.baseUrl) === endpoint); + return matches.length === 1 ? matches[0]!.modelDiscovery : undefined; +} diff --git a/src/providers/model-discovery.ts b/src/providers/model-discovery.ts index 49065c4d74b..82d903982b2 100644 --- a/src/providers/model-discovery.ts +++ b/src/providers/model-discovery.ts @@ -12,6 +12,7 @@ export { MODEL_DISCOVERY_MAX_MODELS, MODEL_DISCOVERY_MAX_RESPONSE_BYTES, } from "./model-discovery-limits"; +import { directoryModelDiscoveryForDestination } from "./free-directory"; import { getProviderRegistryEntry, providerMatchesRegistryTransport, @@ -172,7 +173,11 @@ export function resolveProviderModelDiscovery( const entry = namedEntry ? (providerMatchesRegistryTransport(providerName, provider) ? namedEntry : undefined) : registryEntryForProviderDestination(provider); - const spec = entry?.modelDiscovery; + // A directory row may own a filter for its exact fixed key destination. A named registry row, + // including one whose transport no longer matches, does not fall through to that filter. + const spec = entry?.modelDiscovery ?? ( + entry || namedEntry ? undefined : directoryModelDiscoveryForDestination(provider) + ); return { ...(spec ? { spec } : {}), maxResponseBytes: positiveIntegerAtMost(spec?.maxResponseBytes, MODEL_DISCOVERY_MAX_RESPONSE_BYTES), diff --git a/tests/fixtures/llmtr-model-discovery.json b/tests/fixtures/llmtr-model-discovery.json new file mode 100644 index 00000000000..de6dec8bf43 --- /dev/null +++ b/tests/fixtures/llmtr-model-discovery.json @@ -0,0 +1,55 @@ +{ + "object": "list", + "data": [ + { + "id": "qwen/qwen3.8-27b-free", + "supported_operations": ["CHAT_COMPLETIONS"], + "supported_endpoints": ["/v1/chat/completions"] + }, + { + "id": "llmtr/qwen3-8-27b", + "supported_operations": ["CHAT_COMPLETIONS"], + "supported_endpoints": ["/v1/chat/completions"] + }, + { + "id": "google/gemini-3.8-flash", + "supported_operations": ["CHAT_COMPLETIONS", "RESPONSES"], + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + }, + { + "id": "openai/gpt-image-2", + "supported_operations": ["IMAGES_GENERATIONS"], + "supported_endpoints": ["/v1/images/generations"] + }, + { + "id": "llmtr/embeddingmagibu-200m", + "supported_operations": ["EMBEDDINGS"], + "supported_endpoints": ["/v1/embeddings"] + }, + { + "id": "jina/rerank", + "supported_operations": ["RERANK"], + "supported_endpoints": ["/v1/rerank"] + }, + { + "id": "google/veo-3.1-generate-001", + "supported_operations": ["VIDEO_GENERATIONS"], + "supported_endpoints": ["/v1/video/generations"] + }, + { + "id": "openai/gpt-6-luna", + "supported_operations": ["RESPONSES"], + "supported_endpoints": ["/v1/responses"] + }, + { + "id": "vendor/chat-without-endpoint", + "supported_operations": ["CHAT_COMPLETIONS"], + "supported_endpoints": ["/v1/responses"] + }, + { + "id": "vendor/endpoint-without-operation", + "supported_operations": ["RESPONSES"], + "supported_endpoints": ["/v1/chat/completions"] + } + ] +} diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 0c4ea5401be..a5e22524093 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1187,7 +1187,7 @@ "provider-outbound.test.ts": "providers", "provider-runtime-fetch.test.ts": "providers", "provider-tls-profile.test.ts": "providers", "provider-payload.test.ts": "gui", "provider-quota-label-sanitize.test.ts": "providers", "provider-quota-observed-marker.test.ts": "providers", - "provider-quota-abliteration.test.ts": "providers", "provider-quota-ollama-cloud.test.ts": "providers", "provider-quota.test.ts": "providers", "provider-registry-parity.test.ts": "providers", "free-directory-runcrate.test.ts": "providers", "haiku-5-5-catalog.test.ts": "providers", + "provider-quota-abliteration.test.ts": "providers", "provider-quota-ollama-cloud.test.ts": "providers", "provider-quota.test.ts": "providers", "provider-registry-parity.test.ts": "providers", "free-directory-llmtr.test.ts": "providers", "free-directory-runcrate.test.ts": "providers", "haiku-5-5-catalog.test.ts": "providers", "provider-send-path-import.test.ts": "server", "provider-static-model-discovery.test.ts": "providers", "provider-subscription-cli.test.ts": "gui", "provider-workspace-auth.test.ts": "gui", "provider-workspace-data.test.ts": "gui", "provider-workspace-rail.test.ts": "gui", diff --git a/tests/providers/free-directory-llmtr.test.ts b/tests/providers/free-directory-llmtr.test.ts new file mode 100644 index 00000000000..cffdbd2c13f --- /dev/null +++ b/tests/providers/free-directory-llmtr.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { fixturePath } from "../helpers/repo-root"; +import { FREE_PROVIDER_DIRECTORY } from "../../src/providers/free-directory"; +import { extractProviderModelItems, providerModelDiscoverySpecError, resolveProviderModelDiscovery } from "../../src/providers/model-discovery"; +import { PROVIDER_REGISTRY } from "../../src/providers/registry"; + +// Sibling of provider-registry-parity.test.ts, which sits at the file-size ratchet threshold. +describe("free-provider directory: LLMTR", () => { + test("the LLMTR row bounds its free models and keeps the paid catalog discoverable", () => { + // The row accepts a key. Live discovery keeps chat rows from /v1/models and drops the rest. + // Six chat ids were the standing zero-price set on 2026-10-29 (public GET /v1/models and the + // model cards): three reset on a daily quota and three were listed free with no end date. Dated + // promotions and EVREN bring-your-own-key rows also list a zero price, so the note must carry + // those bounds instead of advertising the whole catalog as free. + const row = FREE_PROVIDER_DIRECTORY.find(entry => entry.id === "llmtr"); + expect(row).toMatchObject({ + label: "LLMTR", + adapter: "openai-chat", + baseUrl: "https://llmtr.com/v1", + authKind: "key", + accessGroups: ["recurring-or-keyless"], + supportLevel: "experimental", + verification: "official", + documentationUrl: "https://llmtr.com/docs/gateway/chat-completions/", + modelsUrl: "https://llmtr.com/v1/models", + dashboardUrl: "https://llmtr.com/dashboard/api-keys", + discovery: "hybrid", + liveModels: true, + lastVerified: "2026-10-29", + }); + expect(row?.keyOptional).toBeUndefined(); + expect(row?.models).toEqual([ + "nvidia/nemotron-3-ultra-550b-a55b", + "openai/gpt-oss-safeguard-20b", + "qwen/qwen3.8-27b-free", + "nvidia/nemotron-3.5-super-vl-free", + "agnes/agnes-3.0-flash", + "agnes/agnes-2.5-flash", + ]); + const note = row?.note ?? ""; + expect(note).toContain("qwen/qwen3.8-27b-free"); + expect(note).toContain("daily quota"); + expect(note).toContain("A key is still required"); + expect(note).toContain("dated promotions"); + expect(note).toContain("EVREN bring-your-own-key"); + expect(note).toContain("Models that are neither listed above nor covered by those promotions or routes are paid"); + expect(note).toContain("llmtr/qwen3-8-27b"); + expect(note).toContain("https://llmtr.com/pricing"); + expect(row?.modelDiscovery).toEqual({ + filter: { + allOf: [ + { path: ["supported_operations"], containsAll: ["CHAT_COMPLETIONS"] }, + { path: ["supported_endpoints"], containsAll: ["/v1/chat/completions"] }, + ], + }, + }); + expect(providerModelDiscoverySpecError(row!.modelDiscovery!)).toBeNull(); + // Directory-only: it must not become a canonical credential destination. + expect(PROVIDER_REGISTRY.some(entry => entry.id === "llmtr")).toBe(false); + }); + + test("live discovery keeps chat rows and drops image, embedding, rerank, video, and responses-only rows", () => { + const provider = { adapter: "openai-chat", baseUrl: "https://llmtr.com/v1" }; + const discovery = resolveProviderModelDiscovery("llmtr", provider); + expect(discovery.spec).toEqual(FREE_PROVIDER_DIRECTORY.find(entry => entry.id === "llmtr")?.modelDiscovery); + // A renamed row still points at the fixed destination, so it keeps the same filter. + expect(resolveProviderModelDiscovery("llmtr-gateway", provider).spec).toEqual(discovery.spec); + expect(resolveProviderModelDiscovery("llmtr", { ...provider, baseUrl: "https://llmtr.com/v1/" }).spec).toEqual(discovery.spec); + expect(resolveProviderModelDiscovery("llmtr", { adapter: "openai-chat", baseUrl: "https://example.test/v1" }).spec).toBeUndefined(); + expect(resolveProviderModelDiscovery("llmtr", { ...provider, adapter: "anthropic" }).spec).toBeUndefined(); + expect(resolveProviderModelDiscovery("llmtr", { ...provider, authMode: "oauth" }).spec).toBeUndefined(); + // A registry name whose transport does not match stays on that registry result. Pointing the + // row at LLMTR's host must not pick up the directory chat filter. + const mismatched = resolveProviderModelDiscovery("nebius", { + adapter: "anthropic", + baseUrl: "https://llmtr.com/v1", + authMode: "key", + }); + expect(mismatched.spec).toBeUndefined(); + expect(mismatched).toEqual(resolveProviderModelDiscovery("nebius", { + adapter: "anthropic", + baseUrl: "https://api.tokenfactory.nebius.com/v1", + authMode: "key", + })); + + const catalog = JSON.parse(readFileSync(fixturePath("llmtr-model-discovery.json"), "utf8")) as unknown; + const result = extractProviderModelItems(catalog, discovery); + expect(result).toMatchObject({ ok: true }); + if (!result.ok) return; + expect(result.items.map(item => item.id)).toEqual([ + "qwen/qwen3.8-27b-free", + "llmtr/qwen3-8-27b", + "google/gemini-3.8-flash", + ]); + }); +});