diff --git a/README.md b/README.md index 83f314e5..e1788305 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ keep the panel mounted without receiving telemetry. | [Android](#android) | ✅ Yes | Android | [ML Kit GenAI](https://developers.google.com/ml-kit/genai) | On-device Gemini Nano, transcription and embeddings | | [Core AI](#core-ai) | ❌ No | iOS | [Core AI](https://github.com/apple/coreai-models) | Text, image, video, speech, embedding and vision models | | [Llama](#llama) | ❌ No | iOS, Android | [llama.rn](https://github.com/mybigday/llama.rn) | Run GGUF models via llama.rn | +| [LiteRT-LM](#litert-lm) | ❌ No | iOS, Android | [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm) | Run `.litertlm` models via react-native-litert-lm | --- @@ -275,6 +276,40 @@ Any GGUF model from HuggingFace can be used. Use the format `owner/repo/filename 📚 **[View full Llama documentation →](https://react-native-ai.dev/docs/llama/getting-started)** +--- + +### LiteRT-LM + +Run `.litertlm` models on-device using [LiteRT-LM](https://ai.google.dev/edge/litert-lm) through [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm). **Requires download** - models are downloaded from HuggingFace. + +#### Installation + +```bash +npm install @react-native-ai/litert react-native-litert-lm react-native-nitro-modules +``` + +#### Usage + +```typescript +import { litert } from '@react-native-ai/litert' +import { generateText } from 'ai' + +const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm') + +// Download once and load into memory +await model.prepare() + +const { text } = await generateText({ + model, + prompt: 'Write a haiku about coding.', +}) + +// Cleanup when done +await model.unload() +``` + +📚 **[View full LiteRT-LM documentation →](https://react-native-ai.dev/docs/litert/getting-started)** + ## JSON UI `@react-native-ai/json-ui` lets a tool-calling model build React Native UI. The model calls tools that add, update, and delete nodes in a JSON spec, and `GenerativeUIView` renders it. It's designed for small on-device models with limited context. diff --git a/bun.lock b/bun.lock index 72d8a326..a08f1dbb 100644 --- a/bun.lock +++ b/bun.lock @@ -178,6 +178,23 @@ "react-native": "*", }, }, + "packages/litert": { + "name": "@react-native-ai/litert", + "version": "0.12.0", + "dependencies": { + "@ai-sdk/provider": "^4.0.33", + }, + "devDependencies": { + "ai": "^7.0.0", + "react-native": "0.88.0", + "react-native-litert-lm": "0.8.2", + "zod": "^4.0.0", + }, + "peerDependencies": { + "react-native": ">=0.76.0", + "react-native-litert-lm": ">=0.8.2", + }, + }, "packages/llama": { "name": "@react-native-ai/llama", "version": "0.12.0", @@ -952,6 +969,8 @@ "@react-native-ai/json-ui": ["@react-native-ai/json-ui@workspace:packages/json-ui"], + "@react-native-ai/litert": ["@react-native-ai/litert@workspace:packages/litert"], + "@react-native-ai/llama": ["@react-native-ai/llama@workspace:packages/llama"], "@react-native-ai/utils": ["@react-native-ai/utils@workspace:packages/utils"], @@ -2362,6 +2381,8 @@ "new-github-release-url": ["new-github-release-url@2.0.0", "", { "dependencies": { "type-fest": "^2.5.1" } }, "sha512-NHDDGYudnvRutt/VhKFlX26IotXe1w0cmkDm6JGquh5bz/bDTw0LufSmH/GxTjEdpHEO+bVKFTwdrcGa/9XlKQ=="], + "nitro-core": ["nitro-core@0.37.2", "", {}, "sha512-dhagikz9VXXzfidRH/vPNJfYWPmTBArDOo9wE/wEqeERU9C8uRWz9YccOyNUOvADccPAZq319yzhDMoJ9excSQ=="], + "nocache": ["nocache@4.0.0", "", {}, "sha512-AntnTbmKZvNYIsTVPPwv7dfZdAfo/6H/2ZlZACK66NAOQtIApxkB/6pf/c+s+ACW8vemGJzUCyVTssrzNUK6yQ=="], "node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="], @@ -2582,6 +2603,10 @@ "react-native-keyboard-controller": ["react-native-keyboard-controller@1.22.7", "", { "dependencies": { "react-native-is-edge-to-edge": "^1.2.1" }, "peerDependencies": { "react": "*", "react-native": "*", "react-native-reanimated": ">=3.0.0" } }, "sha512-IBR9z6Nl/iDLRfiv1zN9QQroOXAwYMugeAjVDb+CFbRBNC8fayXlHTBbshVcWc4j95Itsv+1/FgwfxKnIn1Vmg=="], + "react-native-litert-lm": ["react-native-litert-lm@0.8.2", "", { "peerDependencies": { "expo": ">=55.0.0", "react": "*", "react-native": "*", "react-native-nitro-modules": "^0.37.1" }, "optionalPeers": ["expo", "react", "react-native"] }, "sha512-osPYTXeY9NHo/IfrbPzT7+DdI/uCc5tiPjQ7ccsPQOzjOvpw4h6RclmkMSsJvmKoD/3AkrIBmpN4wCoRiOMF+g=="], + + "react-native-nitro-modules": ["react-native-nitro-modules@0.37.2", "", { "dependencies": { "nitro-core": "0.37.2" }, "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-Odnrq/p6/q0R01qZlkqeXu9VHHMuGEnZZmyxMvehEu6QiACNHhBCmAHgyBDqOSo9Lb3G0TscFAT+HG0uxsR/GA=="], + "react-native-reanimated": ["react-native-reanimated@4.7.1", "", { "dependencies": { "react-native-is-edge-to-edge": "^1.3.1", "semver": "^7.7.3" }, "peerDependencies": { "react": "*", "react-native": "0.86 - 0.88", "react-native-worklets": "0.13.x" } }, "sha512-HviDFSZ6V4KTRsrLS6hRbBJ4j63l/L8a5az0EoWdJwaMLYnBNyYISTDxwXpEYkMbAbLzCMCviIdiOxXHYQWcNA=="], "react-native-safe-area-context": ["react-native-safe-area-context@5.10.1", "", { "peerDependencies": { "react": "*", "react-native": "*" } }, "sha512-ax0T3cAHQIhuAjDQpJ9C1Le96aRiM9PF+Kmgg7Y8+mD7QJMEIx04cG0eG3EGUTdvymgOkCrkakgZFwuOIuNQGw=="], diff --git a/packages/litert/README.md b/packages/litert/README.md new file mode 100644 index 00000000..a9091564 --- /dev/null +++ b/packages/litert/README.md @@ -0,0 +1,38 @@ +# LiteRT-LM Provider for Vercel AI SDK + +A Vercel AI SDK provider for [LiteRT-LM](https://ai.google.dev/edge/litert-lm), Google's runtime for LLMs on the device. It runs `.litertlm` models on iOS and Android through [react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm). + +**Requirements:** +- react-native-litert-lm >= 0.8.2 +- Vercel AI SDK v7 + +```ts +import { litert } from '@react-native-ai/litert' +import { generateText } from 'ai' + +const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm') +await model.prepare() + +const { text } = await generateText({ + model, + prompt: 'Write a haiku about coding.', +}) +``` + +## Features + +- ✅ Text generation with `.litertlm` models from Hugging Face, a URL or a file +- ✅ Streaming +- ✅ Tool calling +- ✅ Structured output (`Output.object`), enforced by the engine +- ✅ Multi-turn conversations that continue in the engine, so the history isn't sent again +- ✅ Model download with progress +- ✅ GPU and CPU backends. NPU with a bundle compiled for the phone's chip, not yet verified on a device + +## Documentation + +For the complete guide, see the [LiteRT-LM documentation](https://react-native-ai.dev/docs/litert/getting-started). + +## License + +MIT diff --git a/packages/litert/package.json b/packages/litert/package.json new file mode 100644 index 00000000..42430723 --- /dev/null +++ b/packages/litert/package.json @@ -0,0 +1,73 @@ +{ + "name": "@react-native-ai/litert", + "version": "0.12.0", + "description": "LiteRT-LM provider for Vercel AI SDK", + "main": "lib/commonjs/index", + "module": "lib/module/index", + "types": "lib/typescript/index.d.ts", + "react-native": "src/index", + "source": "src/index", + "files": [ + "src", + "lib", + "!**/__tests__", + "!**/__fixtures__", + "!**/__mocks__", + "!**/.*" + ], + "license": "MIT", + "homepage": "https://github.com/callstackincubator/ai#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/callstackincubator/ai.git" + }, + "bugs": { + "url": "https://github.com/callstackincubator/ai/issues" + }, + "publishConfig": { + "registry": "https://registry.npmjs.org/" + }, + "scripts": { + "clean": "del-cli lib", + "typecheck": "tsc --noEmit", + "test": "bun test", + "prepare": "bob build" + }, + "dependencies": { + "@ai-sdk/provider": "^4.0.33" + }, + "peerDependencies": { + "react-native": ">=0.76.0", + "react-native-litert-lm": ">=0.8.2" + }, + "devDependencies": { + "ai": "^7.0.0", + "react-native": "0.88.0", + "react-native-litert-lm": "0.8.2", + "zod": "^4.0.0" + }, + "react-native-builder-bob": { + "source": "src", + "output": "lib", + "targets": [ + "commonjs", + "module", + [ + "typescript", + { + "project": "tsconfig.build.json" + } + ] + ] + }, + "keywords": [ + "react-native", + "litert", + "litert-lm", + "llm", + "on-device", + "ai", + "sdk", + "vercel" + ] +} diff --git a/packages/litert/src/__tests__/fakeBinding.ts b/packages/litert/src/__tests__/fakeBinding.ts new file mode 100644 index 00000000..d4ec75f9 --- /dev/null +++ b/packages/litert/src/__tests__/fakeBinding.ts @@ -0,0 +1,136 @@ +import type { LiteRTLMInstance } from 'react-native-litert-lm' + +import type { LiteRTBinding } from '../runtime' + +function cancelledError(): Error { + return Object.assign(new Error('The message was cancelled.'), { + code: 'INFERENCE_CANCELLED', + }) +} + +export interface ScriptedTurn { + /** Tokens the engine streams, in order. */ + tokens: string[] + /** Rejects after the tokens, the way the engine reports a failed stream. */ + failWith?: Error + /** Leaves the message generating until it is cancelled. */ + hangAfterTokens?: boolean +} + +/** A scripted stand-in for the phone's LiteRT-LM binding that records what it was asked. */ +export class FakeBinding { + turns: ScriptedTurn[] = [] + loads: { path: string; config: Record; options: any }[] = [] + resets: { historyJson?: string; systemPrompt?: string }[] = [] + executes: { parts: any[]; options: any }[] = [] + loadError: Error | undefined + closes = 0 + unloads = 0 + instancesCreated = 0 + manifest: any = null + platform: 'android' | 'ios' = 'android' + manifestCalls: any[] = [] + keptFiles: string[] = [] + /** False once the system has taken the engine's memory back. */ + ready = true + /** Holds a load until it is called, like a long download. */ + holdLoad: Promise | undefined + + private cancel: (() => void) | undefined + + /** The system takes the engine's memory back: a message in progress ends as cancelled. */ + freeMemory(): void { + this.ready = false + this.cancel?.() + } + + binding: LiteRTBinding = { + createLLM: () => { + this.instancesCreated += 1 + return this.instance() + }, + getDeviceInfo: (() => ({ + platform: this.platform, + })) as LiteRTBinding['getDeviceInfo'], + listKeptFiles: () => this.keptFiles, + resolveFromManifest: (async (repo: string, options: unknown) => { + this.manifestCalls.push({ repo, options }) + return this.manifest + }) as LiteRTBinding['resolveFromManifest'], + toolResultPart: (name, response) => ({ + type: 'toolResult', + name, + responseJson: JSON.stringify(response) ?? 'null', + }), + } + + private instance(): LiteRTLMInstance { + const self = this + return { + async loadModel(path: string, config: Record, options: any) { + self.loads.push({ path, config, options }) + self.ready = true + options?.onProgress?.(0.5) + if (self.holdLoad !== undefined) await self.holdLoad + if (self.loadError !== undefined) { + const error = self.loadError + self.loadError = undefined + throw error + } + }, + resetConversation(historyJson?: string, systemPrompt?: string) { + self.resets.push({ historyJson, systemPrompt }) + self.cancel?.() + }, + execute( + parts: any[], + onToken: (token: string, done: boolean) => void, + options: any + ) { + self.executes.push({ parts, options }) + const turn = self.turns.shift() + if (turn === undefined) + return Promise.reject(new Error('No scripted turn left.')) + return new Promise((resolve, reject) => { + self.cancel = () => { + self.cancel = undefined + onToken('', true) + reject(cancelledError()) + } + void (async () => { + for (const token of turn.tokens) { + await Promise.resolve() + onToken(token, false) + } + if (turn.hangAfterTokens) return + self.cancel = undefined + if (turn.failWith !== undefined) { + onToken(`Error: ${turn.failWith.message}`, true) + reject(turn.failWith) + return + } + onToken('', true) + resolve(turn.tokens.join('')) + })() + }) + }, + isReady: () => self.ready, + getStats: () => ({ + promptTokens: 10, + completionTokens: 5, + totalTokens: 15, + timeToFirstToken: 120, + totalTime: 500, + tokensPerSecond: 42, + }), + countTokens: () => -1, + getContextTokenCount: () => -1, + async unload() { + self.unloads += 1 + }, + close() { + self.closes += 1 + }, + } as unknown as LiteRTLMInstance + } +} diff --git a/packages/litert/src/__tests__/huggingFace.test.ts b/packages/litert/src/__tests__/huggingFace.test.ts new file mode 100644 index 00000000..9df860c4 --- /dev/null +++ b/packages/litert/src/__tests__/huggingFace.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it } from 'bun:test' + +import { + parseHuggingFaceId, + parseTree, + resolveUrl, + selectKeptFile, + selectRepoFile, +} from '../huggingFace' + +describe('parseHuggingFaceId', () => { + it('reads hf:// ids', () => { + expect(parseHuggingFaceId('hf://org/repo')).toEqual({ + repo: 'org/repo', + revision: 'main', + path: undefined, + }) + expect(parseHuggingFaceId('hf://org/repo@abc/dir/file.litertlm')).toEqual({ + repo: 'org/repo', + revision: 'abc', + path: 'dir/file.litertlm', + }) + }) + + it('reads a bare repo', () => { + expect(parseHuggingFaceId('org/repo')?.repo).toBe('org/repo') + }) + + it('leaves paths and URLs alone', () => { + expect(parseHuggingFaceId('/data/model.litertlm')).toBeUndefined() + expect( + parseHuggingFaceId('https://example.com/a/b.litertlm') + ).toBeUndefined() + expect(parseHuggingFaceId('models/gemma.litertlm')).toBeUndefined() + }) + + it('rejects a repo without a name', () => { + expect(parseHuggingFaceId('hf://org')).toBeUndefined() + }) +}) + +const gemma = [ + 'gemma-4-E2B-it-gpu.litertlm', + 'gemma-4-E2B-it-web.litertlm', + 'gemma-4-E2B-it.litertlm', + 'gemma-4-E2B-it_Google_Tensor_G5.litertlm', + 'gemma-4-E2B-it_NPU_Uncompiled.litertlm', + 'gemma-4-E2B-it_Qualcomm_QCS8275.litertlm', + 'gemma-4-E2B-it_intel_LNL.litertlm', + 'README.md', +].map((path) => ({ path })) + +const REPO = 'org/repo' + +describe('selectKeptFile', () => { + it('goes by the start of the name, however the separators differ', () => { + const kept = [ + 'qwen3_0_6b_mixed_int4.litertlm', + 'gemma-4-E2B-it.litertlm', + 'other.litertlm', + ] + expect( + selectKeptFile( + kept, + 'litert-community/gemma-4-E2B-it-litert-lm', + undefined, + 'android' + )?.path + ).toBe('gemma-4-E2B-it.litertlm') + expect( + selectKeptFile(kept, 'litert-community/Qwen3-0.6B', undefined, 'android') + ?.path + ).toBe('qwen3_0_6b_mixed_int4.litertlm') + expect( + selectKeptFile(kept, 'litert-community/SmolLM3-3B', undefined, 'android') + ).toBeUndefined() + }) + + it('leaves out a file that may be another model', () => { + const kept = [ + 'Qwen3-8B_mixed_int4.litertlm', + 'Phi-4-mini_multi-prefill-seq_q8.litertlm', + ] + expect( + selectKeptFile(kept, 'org/Qwen3', undefined, 'android') + ).toBeUndefined() + expect( + selectKeptFile(kept, 'org/phi-4', undefined, 'android') + ).toBeUndefined() + expect( + selectKeptFile(kept, 'org/Qwen3-8B', undefined, 'android')?.path + ).toBe(kept[0]) + expect( + selectKeptFile(kept, 'org/Phi-4-mini', undefined, 'android')?.path + ).toBe(kept[1]) + }) + + it('takes the platform order among the kept files', () => { + const kept = ['gemma-4-E2B-it-gpu.litertlm', 'gemma-4-E2B-it.litertlm'] + const repo = 'litert-community/gemma-4-E2B-it-litert-lm' + expect(selectKeptFile(kept, repo, undefined, 'ios')?.path).toBe( + 'gemma-4-E2B-it-gpu.litertlm' + ) + expect(selectKeptFile(kept, repo, undefined, 'android')?.path).toBe( + 'gemma-4-E2B-it.litertlm' + ) + expect( + selectKeptFile(['gemma-4-E2B-it-gpu.litertlm'], repo, 'cpu', 'android') + ).toBeUndefined() + }) +}) + +describe('selectRepoFile', () => { + it('takes the full bundle on Android', () => { + expect(selectRepoFile(gemma, REPO, undefined, 'android')?.path).toBe( + 'gemma-4-E2B-it.litertlm' + ) + expect(selectRepoFile(gemma, REPO, 'gpu', 'android')?.path).toBe( + 'gemma-4-E2B-it.litertlm' + ) + }) + + it('takes the GPU file on iOS', () => { + expect(selectRepoFile(gemma, REPO, undefined, 'ios')?.path).toBe( + 'gemma-4-E2B-it-gpu.litertlm' + ) + expect(selectRepoFile(gemma, REPO, 'gpu', 'ios')?.path).toBe( + 'gemma-4-E2B-it-gpu.litertlm' + ) + }) + + it('skips the GPU file when the CPU was asked for', () => { + expect(selectRepoFile(gemma, REPO, 'cpu', 'ios')?.path).toBe( + 'gemma-4-E2B-it.litertlm' + ) + expect(selectRepoFile(gemma, REPO, 'cpu', 'android')?.path).toBe( + 'gemma-4-E2B-it.litertlm' + ) + }) + + it('takes the only file there is on either platform', () => { + const gpuOnly = [{ path: 'model-gpu.litertlm' }] + const fullOnly = [{ path: 'model.litertlm' }] + expect(selectRepoFile(gpuOnly, REPO, undefined, 'android')?.path).toBe( + 'model-gpu.litertlm' + ) + expect(selectRepoFile(fullOnly, REPO, undefined, 'ios')?.path).toBe( + 'model.litertlm' + ) + expect(selectRepoFile(gpuOnly, REPO, 'cpu', 'android')).toBeUndefined() + }) + + it('skips chip-specific bundles', () => { + const gemma3 = [ + 'Gemma3-1B-IT_q4_ekv1280_mt6989.litertlm', + 'Gemma3-1B-IT_q4_ekv1280_sm8850.litertlm', + 'gemma3-1b-it-int4.litertlm', + ].map((path) => ({ path })) + expect(selectRepoFile(gemma3, REPO, undefined, 'android')?.path).toBe( + 'gemma3-1b-it-int4.litertlm' + ) + }) + + it('prefers the file named after the repo, as the browser does', () => { + const qwen = [ + 'Qwen3-0.6B.litertlm', + 'Qwen3-0.6B.mediatek.mt6993.litertlm', + 'Qwen3-0.6B_dynamic_wi4b32_afp32.litertlm', + 'qwen3_0_6b_mixed_int4.litertlm', + ].map((path) => ({ path })) + expect( + selectRepoFile(qwen, 'litert-community/Qwen3-0.6B', undefined, 'android') + ?.path + ).toBe('Qwen3-0.6B.litertlm') + expect( + selectRepoFile( + gemma, + 'litert-community/gemma-4-E2B-it-litert-lm', + undefined, + 'ios' + )?.path + ).toBe('gemma-4-E2B-it-gpu.litertlm') + }) + + it('prefers a file called model next', () => { + const files = ['a-int4.litertlm', 'model.litertlm'].map((path) => ({ + path, + })) + expect(selectRepoFile(files, REPO, undefined, 'android')?.path).toBe( + 'model.litertlm' + ) + }) + + it('prefers int4 over a shorter name when no file is named after the repo', () => { + const files = [ + 'qwen3_4b_channelwise_int8_float32kv.litertlm', + 'a.litertlm', + 'qwen3_4b_mixed_int4.litertlm', + ].map((path) => ({ path })) + expect( + selectRepoFile(files, 'litert-community/Qwen3-4B', undefined, 'android') + ?.path + ).toBe('qwen3_4b_mixed_int4.litertlm') + }) + + it('finds nothing in a repo without a usable file', () => { + expect( + selectRepoFile([{ path: 'README.md' }], REPO, undefined, 'android') + ).toBeUndefined() + expect( + selectRepoFile([{ path: 'x_NPU_y.litertlm' }], REPO, undefined, 'android') + ).toBeUndefined() + }) +}) + +describe('Hub helpers', () => { + it('builds a download URL', () => { + expect( + resolveUrl({ repo: 'org/repo', revision: 'main' }, 'a b.litertlm') + ).toBe('https://huggingface.co/org/repo/resolve/main/a%20b.litertlm') + }) + + it('reads a tree listing', () => { + expect( + parseTree([ + { type: 'file', path: 'a.litertlm', size: 5, lfs: { oid: 'abc' } }, + { type: 'directory', path: 'dir' }, + { type: 'file', path: 'b.txt', size: 1 }, + ]) + ).toEqual([ + { path: 'a.litertlm', size: 5, sha256: 'abc' }, + { path: 'b.txt', size: 1, sha256: undefined }, + ]) + expect(parseTree({ error: 'x' })).toEqual([]) + }) +}) diff --git a/packages/litert/src/__tests__/outputParser.test.ts b/packages/litert/src/__tests__/outputParser.test.ts new file mode 100644 index 00000000..9a5ec0ec --- /dev/null +++ b/packages/litert/src/__tests__/outputParser.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from 'bun:test' + +import type { OutputChunk } from '../backend' +import { + DEFAULT_CHANNELS, + OutputParser, + toolCallFromBody, +} from '../outputParser' + +function parse(tokens: string[], channels = DEFAULT_CHANNELS): OutputChunk[] { + const chunks: OutputChunk[] = [] + const parser = new OutputParser(channels, (chunk) => chunks.push(chunk)) + for (const token of tokens) parser.push(token) + parser.finish() + return chunks +} + +const textOf = (chunks: OutputChunk[]) => + chunks.map((chunk) => (chunk.type === 'text' ? chunk.text : '')).join('') + +describe('OutputParser', () => { + it('passes plain text through', () => { + expect(textOf(parse(['Hello', ' world']))).toBe('Hello world') + }) + + it('finds a tool call split across tokens', () => { + const chunks = parse([ + 'Sure. {"name":"a",', + '"arguments":{"x":1}} Done.', + ]) + expect(textOf(chunks)).toBe('Sure. Done.') + expect(chunks.filter((chunk) => chunk.type === 'tool-call')).toEqual([ + { type: 'tool-call', name: 'a', argumentsJson: '{"x":1}' }, + ]) + }) + + it('keeps two calls apart', () => { + const calls = parse([ + '{"name":"a","arguments":{}}{"name":"b","arguments":{}}', + ]).filter((chunk) => chunk.type === 'tool-call') + expect(calls.map((call) => (call as { name: string }).name)).toEqual([ + 'a', + 'b', + ]) + }) + + it('reads the think tags an older bundle leaves in the text', () => { + expect(parse(['\n\nHmm', '\n\nParis'])).toEqual([ + { type: 'reasoning', text: 'Hmm' }, + { type: 'text', text: 'Paris' }, + ]) + }) + + it('drops an empty think block', () => { + expect(parse(['\n\n\n\n', 'Paris'])).toEqual([ + { type: 'text', text: 'Paris' }, + ]) + }) + + it('keeps the spaces inside an answer', () => { + expect(parse(['aPar', ' is', ' big'])).toEqual([ + { type: 'reasoning', text: 'a' }, + { type: 'text', text: 'Par' }, + { type: 'text', text: ' is' }, + { type: 'text', text: ' big' }, + ]) + }) + + it('separates reasoning from the answer', () => { + const chunks = parse(['Let me', ' seeParis']) + expect(chunks).toEqual([ + { type: 'reasoning', text: 'Let me' }, + { type: 'reasoning', text: ' see' }, + { type: 'text', text: 'Paris' }, + ]) + }) + + it('does not hold back text that only looks like a marker at the end', () => { + expect(textOf(parse(['a < b', ' and <', '']))).toBe('a < b and <') + }) + + it('turns an unterminated call into a call the SDK can report', () => { + const chunks = parse(['{"name":"a","arguments":{"x":']) + expect(chunks).toEqual([ + { + type: 'tool-call', + name: 'a', + argumentsJson: '{"name":"a","arguments":{"x":', + }, + ]) + }) + + it('leaves markers alone when the channel is off', () => { + const channels = DEFAULT_CHANNELS.filter( + (channel) => channel.type !== 'tool-call' + ) + expect(textOf(parse(['x'], channels))).toBe( + 'x' + ) + }) +}) + +describe('a reasoning tag', () => { + const run = (...tokens: string[]) => { + const chunks: OutputChunk[] = [] + const parser = new OutputParser(DEFAULT_CHANNELS, (chunk) => + chunks.push(chunk) + ) + for (const token of tokens) parser.push(token) + parser.finish() + const of = (type: string) => + chunks + .flatMap((c) => (c.type === type && 'text' in c ? [c.text] : [])) + .join('') + return { text: of('text'), reasoning: of('reasoning') } + } + + it('opens reasoning before any text', () => { + expect(run('Hm.Paris')).toEqual({ + text: 'Paris', + reasoning: 'Hm.', + }) + expect(run('\n\nHm.\n\nParis').reasoning).toBe('Hm.') + expect(run('Hm.Paris')).toEqual({ + text: 'Paris', + reasoning: 'Hm.', + }) + }) + + it('is only text after text: an answer can be about the tag', () => { + const answer = 'Models wrap thoughts in a tag and then answer.' + expect(run(answer)).toEqual({ text: answer, reasoning: '' }) + expect(run('Models wrap thoughts in a tag.')).toEqual({ + text: 'Models wrap thoughts in a tag.', + reasoning: '', + }) + }) + + it('still opens after an earlier reasoning part', () => { + expect(run('abParis')).toEqual({ + text: 'Paris', + reasoning: 'ab', + }) + }) +}) + +describe('toolCallFromBody', () => { + it('reads a whole call', () => { + expect(toolCallFromBody('{"name":"a","arguments":{"x":1}}')).toEqual({ + type: 'tool-call', + name: 'a', + argumentsJson: '{"x":1}', + }) + }) + + it('gives a call without arguments empty ones', () => { + expect(toolCallFromBody('{"name":"a"}').argumentsJson).toBe('{}') + }) + + it('keeps the name of a call whose JSON is broken', () => { + const call = toolCallFromBody('{"name":"a","arguments":{"x":') + expect(call.name).toBe('a') + expect(call.argumentsJson).toBe('{"name":"a","arguments":{"x":') + }) + + it('has no name for text that is not a call', () => { + expect(toolCallFromBody('not json').name).toBe('') + }) +}) diff --git a/packages/litert/src/__tests__/prompt.test.ts b/packages/litert/src/__tests__/prompt.test.ts new file mode 100644 index 00000000..2855ea30 --- /dev/null +++ b/packages/litert/src/__tests__/prompt.test.ts @@ -0,0 +1,212 @@ +import { UnsupportedFunctionalityError } from '@ai-sdk/provider' +import { describe, expect, it } from 'bun:test' + +import { + canonicalJson, + selectTools, + toConversation, + transcriptKey, +} from '../prompt' + +describe('toConversation', () => { + it('splits system text, history and the final turn', () => { + const conversation = toConversation([ + { role: 'system', content: 'Be brief.' }, + { role: 'user', content: [{ type: 'text', text: 'Hi' }] }, + { + role: 'assistant', + content: [ + { type: 'reasoning', text: 'They greet me.' }, + { type: 'text', text: 'Hello.' }, + ], + }, + { role: 'user', content: [{ type: 'text', text: 'Bye' }] }, + ]) + expect(conversation).toEqual({ + system: 'Be brief.', + history: [ + { role: 'user', text: 'Hi' }, + { role: 'assistant', text: 'Hello.', toolCalls: [] }, + ], + final: { role: 'user', text: 'Bye' }, + }) + }) + + it('reads tool calls and every kind of tool result', () => { + const conversation = toConversation([ + { role: 'user', content: [{ type: 'text', text: 'Weather?' }] }, + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: '1', + toolName: 'w', + input: { city: 'Tokyo' }, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: '1', + toolName: 'w', + output: { type: 'json', value: { c: 20 } }, + }, + { + type: 'tool-result', + toolCallId: '2', + toolName: 'x', + output: { type: 'text', value: 'ok' }, + }, + { + type: 'tool-result', + toolCallId: '3', + toolName: 'y', + output: { type: 'error-text', value: 'boom' }, + }, + { + type: 'tool-result', + toolCallId: '4', + toolName: 'z', + output: { type: 'execution-denied' }, + }, + { + type: 'tool-result', + toolCallId: '5', + toolName: 'v', + output: { + type: 'content', + value: [ + { type: 'text', text: 'a' }, + { type: 'text', text: 'b' }, + ], + }, + }, + ], + }, + ]) + expect(conversation.history[1]).toEqual({ + role: 'assistant', + text: '', + toolCalls: [{ name: 'w', arguments: { city: 'Tokyo' } }], + }) + expect(conversation.final).toEqual({ + role: 'tool', + results: [ + { name: 'w', value: { c: 20 } }, + { name: 'x', value: 'ok' }, + { name: 'y', value: { error: 'boom' } }, + { name: 'z', value: { error: 'The tool call was denied.' } }, + { name: 'v', value: 'ab' }, + ], + }) + }) + + it('refuses file parts', () => { + expect(() => + toConversation([ + { + role: 'user', + content: [ + { + type: 'file', + mediaType: 'image/png', + data: { type: 'data', data: new Uint8Array(1) }, + }, + ], + }, + ]) + ).toThrow(UnsupportedFunctionalityError) + }) + + it('refuses a prompt that ends with the assistant', () => { + expect(() => + toConversation([ + { role: 'assistant', content: [{ type: 'text', text: 'Sure' }] }, + ]) + ).toThrow(UnsupportedFunctionalityError) + expect(() => toConversation([])).toThrow(UnsupportedFunctionalityError) + }) +}) + +describe('selectTools', () => { + const tools = [ + { + type: 'function' as const, + name: 'a', + inputSchema: { type: 'object' as const }, + }, + { + type: 'function' as const, + name: 'b', + inputSchema: { type: 'object' as const }, + }, + ] + + it('offers no tools for toolChoice none', () => { + expect(selectTools({ tools, toolChoice: { type: 'none' } }, [])).toEqual([]) + }) + + it('warns that it cannot force a call', () => { + const warnings: any[] = [] + expect( + selectTools({ tools, toolChoice: { type: 'required' } }, warnings) + ).toHaveLength(2) + expect(warnings[0].feature).toBe('toolChoice: required') + }) + + it('offers only the chosen tool', () => { + const warnings: any[] = [] + const selected = selectTools( + { tools, toolChoice: { type: 'tool', toolName: 'b' } }, + warnings + ) + expect(selected.map((tool) => tool.name)).toEqual(['b']) + expect(warnings).toHaveLength(1) + }) + + it('warns about provider tools', () => { + const warnings: any[] = [] + const selected = selectTools( + { tools: [{ type: 'provider', id: 'x.y', name: 'y', args: {} }] }, + warnings + ) + expect(selected).toEqual([]) + expect(warnings[0].type).toBe('unsupported') + }) +}) + +describe('transcriptKey', () => { + it('ignores key order and a trimmed reply', () => { + const a = transcriptKey('s', [ + { + role: 'assistant', + text: ' Hello. ', + toolCalls: [{ name: 't', arguments: { a: 1, b: 2 } }], + }, + ]) + const b = transcriptKey('s', [ + { + role: 'assistant', + text: 'Hello.', + toolCalls: [{ name: 't', arguments: { b: 2, a: 1 } }], + }, + ]) + expect(a).toBe(b) + }) + + it('tells conversations apart', () => { + const turn = { role: 'user' as const, text: 'a' } + expect(transcriptKey('s', [turn])).not.toBe(transcriptKey('t', [turn])) + expect(transcriptKey('s', [turn])).not.toBe(transcriptKey('s', [])) + }) + + it('writes canonical JSON', () => { + expect(canonicalJson({ b: [{ d: 1, c: 2 }], a: null })).toBe( + '{"a":null,"b":[{"c":2,"d":1}]}' + ) + }) +}) diff --git a/packages/litert/src/__tests__/runtime.test.ts b/packages/litert/src/__tests__/runtime.test.ts new file mode 100644 index 00000000..b2d50f1f --- /dev/null +++ b/packages/litert/src/__tests__/runtime.test.ts @@ -0,0 +1,1547 @@ +import { + generateText, + jsonSchema, + Output, + stepCountIs, + streamText, + tool, +} from 'ai' +import { afterEach, describe, expect, it, mock, spyOn } from 'bun:test' + +import { LiteRTError } from '../errors' +import { createProviderWithRuntime } from '../provider' +import { createRuntime } from '../runtime' +import { FakeBinding } from './fakeBinding' + +const MODEL = '/data/gemma.litertlm' +const realFetch = globalThis.fetch +const setFetch = (stub: unknown) => { + globalThis.fetch = stub as typeof fetch +} + +function setup() { + const fake = new FakeBinding() + const provider = createProviderWithRuntime((modelId, settings) => + createRuntime(modelId, settings, fake.binding) + ) + return { fake, provider } +} + +const weather = tool({ + description: 'Get the weather for a city.', + inputSchema: jsonSchema<{ city: string }>({ + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }), + execute: async ({ city }) => ({ city, temperature: 21 }), +}) + +afterEach(() => { + globalThis.fetch = realFetch +}) + +describe('generateText and streamText on a phone', () => { + it('generates text and reports usage', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Par', 'is'] }) + + const result = await generateText({ + model: provider(MODEL), + system: 'Be brief.', + prompt: 'What is the capital of France?', + }) + + expect(result.text).toBe('Paris') + expect(result.finishReason).toBe('stop') + expect(result.usage.inputTokens).toBe(10) + // The binding's stats estimate from characters. What it streamed is closer. + expect(result.usage.outputTokens).toBe(2) + expect(fake.resets).toEqual([ + { historyJson: undefined, systemPrompt: 'Be brief.' }, + ]) + expect(fake.executes[0]?.parts).toEqual([ + { type: 'text', text: 'What is the capital of France?' }, + ]) + expect(result.providerMetadata?.litert).toMatchObject({ + continued: false, + reloaded: false, + }) + }) + + it('works out the rate from what Android streamed, past the first token', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['one', ' two', ' three', ' four', ' five'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Count.', + }) + + // Four tokens after the first, in the 380 ms between the first token and the end. + const metadata = result.providerMetadata?.litert as { + tokensPerSecond: number + } + expect(metadata.tokensPerSecond).toBeCloseTo(4 / 0.38) + }) + + it('reports no rate for a one-token reply on Android', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Paris'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Capital of France?', + }) + + expect(result.providerMetadata?.litert).not.toHaveProperty( + 'tokensPerSecond' + ) + }) + + it('streams text as it is generated', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Hel', 'lo', ' world'] }) + + const result = streamText({ model: provider(MODEL), prompt: 'Say hello.' }) + const deltas: string[] = [] + for await (const delta of result.textStream) deltas.push(delta) + + expect(deltas).toEqual(['Hel', 'lo', ' world']) + expect(await result.finishReason).toBe('stop') + }) + + it('passes maxOutputTokens per message and reports a cut-off answer', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a', 'b', 'c', 'd', 'e'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Count.', + maxOutputTokens: 5, + }) + + expect(fake.executes[0]?.options?.maxOutputTokens).toBe(5) + expect(result.finishReason).toBe('length') + }) + + it('does not call a long word that fits a cut-off answer', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Internationalization'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + maxOutputTokens: 5, + }) + + expect(result.finishReason).toBe('stop') + }) + + it('reports an answer cut off at the limit the binding sets when nothing else does', async () => { + const { fake, provider } = setup() + fake.turns.push( + { tokens: Array.from({ length: 1024 }, () => 'x') }, + { tokens: ['x'] } + ) + const model = provider(MODEL) + + expect( + (await generateText({ model, prompt: 'A long story.' })).finishReason + ).toBe('length') + expect( + (await generateText({ model, prompt: 'A short one.' })).finishReason + ).toBe('stop') + }) + + it('reports thoughts that used up the limit as a cut-off answer', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: [] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + reasoning: 'high', + }) + + expect(result.text).toBe('') + expect(result.finishReason).toBe('length') + }) + + it('warns about a reload when an earlier call in the queue changed what is loaded', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }, { tokens: ['c'] }) + const model = provider(MODEL) + await generateText({ model, prompt: 'One' }) + + const prompt = [ + { + role: 'user' as const, + content: [{ type: 'text' as const, text: 'Two' }], + }, + ] + const [withTools, plain] = await Promise.all([ + model.doGenerate({ + prompt, + tools: [ + { + type: 'function', + name: 'weather', + inputSchema: { type: 'object' }, + }, + ], + }), + model.doGenerate({ prompt }), + ]) + + expect(withTools.warnings[0]).toMatchObject({ type: 'other' }) + expect(plain.warnings[0]).toMatchObject({ type: 'other' }) + expect(fake.loads).toHaveLength(3) + }) + + it('loads once for repeated calls, even through new model objects', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + + await generateText({ model: provider(MODEL), prompt: 'One' }) + await generateText({ model: provider(MODEL), prompt: 'Two' }) + + expect(fake.loads).toHaveLength(1) + expect(fake.instancesCreated).toBe(1) + }) +}) + +describe('conversation reuse', () => { + it('continues when the prompt extends what the engine holds', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Paris'] }, { tokens: ['Rome'] }) + const model = provider(MODEL) + + const first = await generateText({ + model, + system: 'Be brief.', + prompt: 'France?', + }) + const second = await generateText({ + model, + system: 'Be brief.', + messages: [ + { role: 'user', content: 'France?' }, + ...first.response.messages, + { role: 'user', content: 'Italy?' }, + ], + }) + + expect(second.providerMetadata?.litert).toMatchObject({ continued: true }) + expect(fake.resets).toHaveLength(1) + expect(fake.executes[1]?.parts).toEqual([{ type: 'text', text: 'Italy?' }]) + }) + + it('replays the history when it does not match', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Paris'] }, { tokens: ['Rome'] }) + const model = provider(MODEL) + + await generateText({ model, prompt: 'France?' }) + const second = await generateText({ + model, + messages: [ + { role: 'user', content: 'France?' }, + { role: 'assistant', content: 'An edited reply.' }, + { role: 'user', content: 'Italy?' }, + ], + }) + + expect(second.providerMetadata?.litert).toMatchObject({ continued: false }) + expect(fake.resets).toHaveLength(2) + expect(JSON.parse(fake.resets[1]?.historyJson as string)).toEqual([ + { role: 'user', content: 'France?' }, + { role: 'model', content: 'An edited reply.' }, + ]) + }) + + it('starts over when the system text changes', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + + const first = await generateText({ model, system: 'One', prompt: 'Hi' }) + await generateText({ + model, + system: 'Two', + messages: [ + { role: 'user', content: 'Hi' }, + ...first.response.messages, + { role: 'user', content: 'Again' }, + ], + }) + + expect(fake.resets[1]?.systemPrompt).toBe('Two') + }) + + it('continues after an app trims the reply', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: [' Paris \n'] }, { tokens: ['Rome'] }) + const model = provider(MODEL) + + await generateText({ model, prompt: 'France?' }) + const second = await generateText({ + model, + messages: [ + { role: 'user', content: 'France?' }, + { role: 'assistant', content: 'Paris' }, + { role: 'user', content: 'Italy?' }, + ], + }) + + expect(second.providerMetadata?.litert).toMatchObject({ continued: true }) + }) +}) + +describe('tools', () => { + it('runs a tool loop without replaying the conversation', async () => { + const { fake, provider } = setup() + fake.turns.push( + { + tokens: [ + '{"name":"weather","arguments":{"city":"Tokyo"}}', + ], + }, + { tokens: ['It is 21 degrees.'] } + ) + + const result = await generateText({ + model: provider(MODEL), + tools: { weather }, + stopWhen: stepCountIs(3), + prompt: 'Weather in Tokyo?', + }) + + expect(result.text).toBe('It is 21 degrees.') + expect(result.steps).toHaveLength(2) + expect(result.steps[0]?.finishReason).toBe('tool-calls') + expect(result.steps[0]?.toolCalls[0]?.input).toEqual({ city: 'Tokyo' }) + expect(fake.loads).toHaveLength(1) + expect(fake.resets).toHaveLength(1) + expect(fake.loads[0]?.config.tools[0]).toMatchObject({ + name: 'weather', + description: 'Get the weather for a city.', + }) + expect(fake.executes[1]?.parts).toEqual([ + { + type: 'toolResult', + name: 'weather', + responseJson: JSON.stringify({ city: 'Tokyo', temperature: 21 }), + }, + ]) + }) + + it('replays a tool exchange as the runtime expects', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['Done.'] }) + + await generateText({ + model: provider(MODEL), + messages: [ + { role: 'user', content: 'Weather?' }, + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: '1', + toolName: 'weather', + input: { city: 'Rome' }, + }, + ], + }, + { + role: 'tool', + content: [ + { + type: 'tool-result', + toolCallId: '1', + toolName: 'weather', + output: { type: 'json', value: { temperature: 21 } }, + }, + ], + }, + { role: 'user', content: 'And now?' }, + ], + }) + + expect(JSON.parse(fake.resets[0]?.historyJson as string)).toEqual([ + { role: 'user', content: 'Weather?' }, + { + role: 'model', + content: '', + toolCalls: [{ name: 'weather', argumentsJson: '{"city":"Rome"}' }], + }, + { role: 'tool', content: '{"temperature":21}', toolName: 'weather' }, + ]) + }) + + it('treats markers as text when the call has no tools', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['{"name":"x"}'] }) + + const result = await generateText({ model: provider(MODEL), prompt: 'Hi' }) + + expect(result.text).toBe('{"name":"x"}') + expect(result.finishReason).toBe('stop') + }) +}) + +describe('settings that change per call', () => { + it('reloads with a warning when tools change, and only then', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }, { tokens: ['c'] }) + const model = provider(MODEL) + + const plain = await generateText({ model, prompt: 'Hi' }) + const withTools = await generateText({ + model, + prompt: 'Hi', + tools: { weather }, + }) + const again = await generateText({ + model, + prompt: 'Hi', + tools: { weather }, + }) + + expect(fake.loads).toHaveLength(2) + expect(plain.warnings ?? []).toHaveLength(0) + expect(withTools.warnings?.[0]).toMatchObject({ type: 'other' }) + expect((withTools.warnings?.[0] as { message: string }).message).toContain( + 'tools' + ) + expect(withTools.providerMetadata?.litert).toMatchObject({ reloaded: true }) + expect(again.warnings ?? []).toHaveLength(0) + }) + + it('frees the old engine first, and starts the new one with nothing left over', async () => { + const { fake, provider } = setup() + const order: string[] = [] + const createLLM = fake.binding.createLLM + fake.binding.createLLM = () => { + order.push('create') + const llm = createLLM() + const { unload, loadModel } = llm + llm.unload = () => { + order.push('unload') + return unload() + } + llm.loadModel = ((...args: Parameters) => { + order.push('load') + return loadModel(...args) + }) as typeof loadModel + return llm + } + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + + await generateText({ model, prompt: 'Hi', temperature: 0 }) + await generateText({ model, prompt: 'Hi' }) + + expect(order).toEqual(['create', 'load', 'unload', 'create', 'load']) + expect(fake.closes).toBe(1) + expect(fake.loads[1]?.config.temperature).toBeUndefined() + }) + + it('reloads with a warning when sampling changes', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + + await generateText({ model, prompt: 'Hi', temperature: 0.2 }) + const second = await generateText({ + model, + prompt: 'Hi', + temperature: 0.9, + topK: 5, + }) + + expect(fake.loads).toHaveLength(2) + expect(fake.loads[1]?.config).toMatchObject({ temperature: 0.9, topK: 5 }) + expect((second.warnings?.[0] as { message: string }).message).toContain( + 'sampling' + ) + }) + + it('takes sampling defaults from the model settings without a reload warning', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }) + + const result = await generateText({ + model: provider(MODEL, { + temperature: 0.1, + maxContextTokens: 2048, + backend: 'gpu', + speculativeDecoding: true, + }), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.config).toEqual({ + backend: 'gpu', + maxContextTokens: 2048, + enableSpeculativeDecoding: true, + temperature: 0.1, + }) + expect(result.warnings ?? []).toHaveLength(0) + }) + + it('warns about what the engine cannot do', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + seed: 1, + stopSequences: ['x'], + }) + + expect( + (result.warnings ?? []).map( + (warning) => (warning as { feature?: string }).feature + ) + ).toEqual(['stopSequences', 'seed']) + }) + + it('passes penalties and reasoning per message', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }) + + await generateText({ + model: provider(MODEL), + prompt: 'Hi', + frequencyPenalty: 0.5, + presencePenalty: 0.25, + reasoning: 'none', + }) + + expect(fake.executes[0]?.options).toEqual({ + frequencyPenalty: 0.5, + presencePenalty: 0.25, + thinking: { enabled: false }, + }) + }) +}) + +describe('reasoning', () => { + it('stays off unless the call asks for it', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + + await generateText({ model: provider(MODEL), prompt: 'Hi' }) + await generateText({ + model: provider(MODEL), + prompt: 'Hi', + reasoning: 'high', + }) + + expect(fake.executes[0]?.options).toEqual({ thinking: { enabled: false } }) + expect(fake.executes[1]?.options).toEqual({ thinking: { enabled: true } }) + }) +}) + +describe('structured output', () => { + it('asks the engine for JSON and reads it', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['{"city":', '"Paris"}'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Capital of France?', + output: Output.object({ + schema: jsonSchema<{ city: string }>({ + type: 'object', + properties: { city: { type: 'string' } }, + required: ['city'], + additionalProperties: false, + }), + }), + }) + + expect(result.output).toEqual({ city: 'Paris' }) + expect(fake.loads[0]?.config.enableStructuredOutput).toBe(true) + expect(JSON.parse(fake.executes[0]?.options.responseSchema)).toMatchObject({ + type: 'object', + }) + }) + + const answerSchema = jsonSchema<{ a: number }>({ + type: 'object', + properties: { a: { type: 'number' } }, + required: ['a'], + }) + + it('reloads when structured output is turned on and off', async () => { + const { fake, provider } = setup() + fake.turns.push( + { tokens: ['a'] }, + { tokens: ['{"a":1}'] }, + { tokens: ['b'] } + ) + const model = provider(MODEL) + + await generateText({ model, prompt: 'One' }) + const structured = await generateText({ + model, + prompt: 'Two', + output: Output.object({ schema: answerSchema }), + }) + const plainAfter = await generateText({ model, prompt: 'Three' }) + + expect(fake.loads).toHaveLength(3) + expect(fake.loads[1]?.config.enableStructuredOutput).toBe(true) + expect(fake.loads[2]?.config.enableStructuredOutput).toBeUndefined() + expect((structured.warnings?.[0] as { message: string }).message).toContain( + 'structured output' + ) + expect(plainAfter.warnings).toHaveLength(1) + }) + + it('asks for JSON with no schema in the system text, with a warning and no reload', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['{"a":1}'] }) + const model = provider(MODEL) + await generateText({ model, prompt: 'One' }) + + const result = await generateText({ + model, + prompt: 'Two', + output: Output.json(), + }) + + expect(result.output).toEqual({ a: 1 }) + expect(fake.loads).toHaveLength(1) + expect(fake.resets.at(-1)?.systemPrompt).toContain('one JSON value') + expect(fake.executes[1]?.options?.responseSchema).toBeUndefined() + expect(result.warnings?.[0]).toMatchObject({ + type: 'compatibility', + feature: 'structured output', + }) + }) + + it('asks for the schema in the system text when the call has tools, which a constraint would rule out', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['{"a":1}'] }) + + const result = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + tools: { weather }, + output: Output.object({ schema: answerSchema }), + }) + + expect(result.output).toEqual({ a: 1 }) + expect(fake.loads[0]?.config.enableStructuredOutput).toBeUndefined() + expect(fake.executes[0]?.options?.responseSchema).toBeUndefined() + expect(fake.resets[0]?.systemPrompt).toContain('JSON Schema') + expect(result.warnings?.[0]).toMatchObject({ + type: 'compatibility', + feature: 'structured output', + }) + }) + + it('removes a code fence, in a stream too', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['```json\n', '{"a":1}', '\n```'] }) + + const result = streamText({ + model: provider(MODEL), + prompt: 'JSON please.', + output: Output.json(), + }) + let text = '' + for await (const delta of result.textStream) text += delta + + expect(text).toBe('{"a":1}') + }) +}) + +describe('the system taking the memory back', () => { + it('loads the model again on the next call', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + await generateText({ model, prompt: 'One' }) + + fake.ready = false + const result = await generateText({ model, prompt: 'Two' }) + + expect(result.text).toBe('b') + expect(fake.loads).toHaveLength(2) + }) + + it('ends a stream with an error, and the next call loads again', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'], hangAfterTokens: true }, { tokens: ['b'] }) + const model = provider(MODEL) + const errors: unknown[] = [] + + const result = streamText({ + model, + prompt: 'One', + onError: ({ error }) => errors.push(error), + }) + let streamed = '' + for await (const delta of result.textStream) { + streamed += delta + fake.freeMemory() + } + + expect(streamed).toBe('a') + expect((errors[0] as LiteRTError).code).toBe('OUT_OF_MEMORY') + expect((await generateText({ model, prompt: 'Two' })).text).toBe('b') + expect(fake.loads).toHaveLength(2) + }) +}) + +describe('an abort while the model loads', () => { + it('rejects at once, closes the instance, and the next call starts on a new one', async () => { + const { fake, provider } = setup() + let finishLoad = () => {} + fake.holdLoad = new Promise((resolve) => { + finishLoad = resolve + }) + fake.turns.push({ tokens: ['ok'] }) + const model = provider(MODEL) + const controller = new AbortController() + + const prepared = model + .prepare({ abortSignal: controller.signal }) + .catch((e: unknown) => e) + await new Promise((resolve) => setTimeout(resolve, 5)) + controller.abort() + expect(((await prepared) as Error).name).toBe('AbortError') + expect(fake.closes).toBe(1) + + const next = generateText({ model, prompt: 'Hi' }) + await new Promise((resolve) => setTimeout(resolve, 5)) + expect(fake.loads).toHaveLength(2) + finishLoad() + + expect((await next).text).toBe('ok') + expect(fake.instancesCreated).toBe(2) + }) + + it('does not use an abandoned instance when its load finishes after all', async () => { + const { fake, provider } = setup() + let finishLoad = () => {} + fake.holdLoad = new Promise((resolve) => { + finishLoad = resolve + }) + const model = provider(MODEL) + const controller = new AbortController() + + const prepared = model + .prepare({ abortSignal: controller.signal }) + .catch((e: unknown) => e) + await new Promise((resolve) => setTimeout(resolve, 5)) + controller.abort() + await prepared + finishLoad() + await new Promise((resolve) => setTimeout(resolve, 5)) + fake.holdLoad = undefined + fake.turns.push({ tokens: ['ok'] }) + + expect((await generateText({ model, prompt: 'Hi' })).text).toBe('ok') + expect(fake.loads).toHaveLength(2) + }) + + it('stops reporting progress', async () => { + const { fake, provider } = setup() + let finishLoad = () => {} + fake.holdLoad = new Promise((resolve) => { + finishLoad = resolve + }) + const progress: unknown[] = [] + const controller = new AbortController() + const model = provider(MODEL) + + const prepared = model + .prepare({ + abortSignal: controller.signal, + onProgress: (p) => progress.push(p), + }) + .catch((e: unknown) => e) + await new Promise((resolve) => setTimeout(resolve, 5)) + controller.abort() + await prepared + fake.loads[0]?.options.onProgress(0.9) + finishLoad() + await model.unload() + + expect(progress).toEqual([{ fraction: 0.5 }]) + }) + + it('does not start with a signal that has already aborted', async () => { + const { fake, provider } = setup() + const controller = new AbortController() + controller.abort() + + const error = await provider(MODEL) + .prepare({ abortSignal: controller.signal }) + .catch((e: unknown) => e) + + expect((error as Error).name).toBe('AbortError') + expect(fake.loads).toHaveLength(0) + }) + + it('says nothing about an abandoned load that fails', async () => { + const { fake, provider } = setup() + let finishLoad = () => {} + fake.holdLoad = new Promise((resolve) => { + finishLoad = resolve + }) + fake.loadError = new Error( + 'The instance was closed before it finished loading.' + ) + const controller = new AbortController() + + const prepared = provider(MODEL) + .prepare({ abortSignal: controller.signal }) + .catch((e: unknown) => e) + await new Promise((resolve) => setTimeout(resolve, 5)) + controller.abort() + await prepared + finishLoad() + // An unhandled rejection here fails the test run. + await new Promise((resolve) => setTimeout(resolve, 5)) + + expect(fake.closes).toBe(1) + }) +}) + +describe('failure paths', () => { + it.each([ + ['UNSUPPORTED', 'LOAD_FAILED'], + ['INVALID_INPUT', 'LOAD_FAILED'], + ['INFERENCE_FAILED', 'LOAD_FAILED'], + ])('reports a %s error from a load as %s', async (nativeCode, code) => { + const { fake, provider } = setup() + fake.loadError = Object.assign(new Error('Nope'), { code: nativeCode }) + + const error = await provider(MODEL) + .prepare() + .catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe(code) + }) + + it.each([['HTTP 404'], ['HTTP 401'], ['HTTP 403'], ['HTTP 410']])( + 'reports a download that got %s as no such model', + async (status) => { + const { fake, provider } = setup() + fake.loadError = Object.assign( + new Error(`Failed to download x: Failed to download model: ${status}`), + { + code: 'DOWNLOAD_FAILED', + } + ) + + const error = await provider(MODEL) + .prepare() + .catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('MODEL_NOT_FOUND') + } + ) + + it('reports a download that got HTTP 500 as a failed download', async () => { + const { fake, provider } = setup() + fake.loadError = Object.assign( + new Error('Failed to download x: HTTP 500'), + { + code: 'DOWNLOAD_FAILED', + } + ) + + const error = await provider(MODEL) + .prepare() + .catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('DOWNLOAD_FAILED') + }) + + const nativeError = (code: string, message: string) => + Object.assign(new Error(message), { code }) + + it('reports a missing model and can try again', async () => { + const { fake, provider } = setup() + fake.loadError = nativeError( + 'MODEL_NOT_FOUND', + 'No file at /data/gemma.litertlm' + ) + const model = provider(MODEL) + + const error = await generateText({ model, prompt: 'Hi' }).catch( + (e: unknown) => e + ) + expect(LiteRTError.isInstance(error)).toBe(true) + expect((error as LiteRTError).code).toBe('MODEL_NOT_FOUND') + expect((error as LiteRTError).message).toContain('/data/gemma.litertlm') + + fake.turns.push({ tokens: ['ok'] }) + const retry = await generateText({ model, prompt: 'Hi' }) + expect(retry.text).toBe('ok') + expect(fake.instancesCreated).toBe(2) + }) + + it('reports a broken file', async () => { + const { fake, provider } = setup() + fake.loadError = nativeError('INVALID_MODEL', 'Not a .litertlm bundle') + + const error = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('INVALID_MODEL') + expect(fake.closes).toBe(1) + }) + + it('reports a truncated file as a broken one, not a missing one', async () => { + const { fake, provider } = setup() + fake.loadError = nativeError( + 'MODEL_NOT_FOUND', + 'Failed to create the engine — tried gpu, then the CPU fallbacks: Failed to create engine: NOT_FOUND: ERROR' + ) + + const error = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('INVALID_MODEL') + }) + + it.each([ + 'Failed to create the engine — tried cpu, then the CPU fallbacks: Failed to create engine: INVALID_ARGUMENT: Invalid magic number or failed to read the header', + 'Failed to create the engine (cpu): Unsupported or unknown file format. — tried cpu, then the CPU fallbacks', + ])( + 'reports a file that is not a bundle as a broken one: %s', + async (message) => { + const { fake, provider } = setup() + fake.loadError = nativeError('ENGINE_INIT_FAILED', message) + + const error = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('INVALID_MODEL') + } + ) + + it('reports a prompt longer than the context', async () => { + const { fake, provider } = setup() + fake.turns.push({ + tokens: [], + failWith: nativeError( + 'INFERENCE_FAILED', + 'Inference failed: Input token ids are too long. 4903 >= 256' + ), + }) + + const error = await generateText({ + model: provider(MODEL), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('CONTEXT_TOO_LONG') + }) + + it('reports a missing model through a stream as an error part', async () => { + const { fake, provider } = setup() + fake.loadError = nativeError('MODEL_NOT_FOUND', 'No file') + + const errors: unknown[] = [] + const result = streamText({ + model: provider(MODEL), + prompt: 'Hi', + onError: ({ error }) => errors.push(error), + }) + await result.consumeStream() + + expect((errors[0] as LiteRTError).code).toBe('MODEL_NOT_FOUND') + }) + + it('reports a failed generation without showing the error token as text', async () => { + const { fake, provider } = setup() + fake.turns.push({ + tokens: ['par'], + failWith: nativeError('INFERENCE_FAILED', 'GPU lost'), + }) + + const errors: unknown[] = [] + const result = streamText({ + model: provider(MODEL), + prompt: 'Hi', + onError: ({ error }) => errors.push(error), + }) + let text = '' + for await (const delta of result.textStream) text += delta + + expect(text).toBe('par') + expect((errors[0] as LiteRTError).code).toBe('GENERATION_FAILED') + }) + + it('does not continue a conversation after a failure', async () => { + const { fake, provider } = setup() + fake.turns.push( + { tokens: ['Paris'] }, + { tokens: [], failWith: new Error('boom') }, + { tokens: ['Rome'] } + ) + const model = provider(MODEL) + + const first = await generateText({ model, prompt: 'France?' }) + const messages = [ + { role: 'user' as const, content: 'France?' }, + ...first.response.messages, + { role: 'user' as const, content: 'Italy?' }, + ] + await generateText({ model, messages }).catch(() => {}) + const third = await generateText({ model, messages }) + + expect(third.providerMetadata?.litert).toMatchObject({ continued: false }) + }) + + it('stops on abort and works again after it', async () => { + const { fake, provider } = setup() + fake.turns.push( + { tokens: ['one ', 'two '], hangAfterTokens: true }, + { tokens: ['fresh'] } + ) + const model = provider(MODEL) + const controller = new AbortController() + + const result = streamText({ + model, + prompt: 'Count.', + abortSignal: controller.signal, + }) + const parts: string[] = [] + let aborted = false + for await (const part of result.fullStream) { + parts.push(part.type) + if (part.type === 'text-delta' && !controller.signal.aborted) + controller.abort() + if (part.type === 'abort') aborted = true + } + + expect(aborted).toBe(true) + expect(fake.resets.length).toBeGreaterThanOrEqual(2) + + const next = await generateText({ model, prompt: 'Again.' }) + expect(next.text).toBe('fresh') + expect(next.providerMetadata?.litert).toMatchObject({ continued: false }) + }) + + it('rejects generateText with an AbortError', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['one '], hangAfterTokens: true }) + const controller = new AbortController() + + const pending = generateText({ + model: provider(MODEL), + prompt: 'Count.', + abortSignal: controller.signal, + }) + setTimeout(() => controller.abort(), 10) + + await expect(pending).rejects.toMatchObject({ name: 'AbortError' }) + }) + + it('does nothing for a signal already aborted', async () => { + const { fake, provider } = setup() + const controller = new AbortController() + controller.abort() + + await expect( + generateText({ + model: provider(MODEL), + prompt: 'Hi', + abortSignal: controller.signal, + }) + ).rejects.toMatchObject({ name: 'AbortError' }) + expect(fake.loads).toHaveLength(0) + }) + + it('reports a tool call whose JSON is broken as a tool error', async () => { + const { fake, provider } = setup() + fake.turns.push({ + tokens: ['{"name":"weather","arguments":{"city":'], + }) + + const result = streamText({ + model: provider(MODEL), + tools: { weather }, + prompt: 'Weather?', + }) + const parts: any[] = [] + for await (const part of result.fullStream) parts.push(part) + + const toolError = parts.find((part) => part.type === 'tool-error') + expect(toolError?.toolName).toBe('weather') + expect(await result.text).toBe('') + expect(await result.finishReason).toBe('tool-calls') + }) + + it('reports a tool call with no readable name as a tool error', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['not json at all'] }) + + const result = await generateText({ + model: provider(MODEL), + tools: { weather }, + prompt: 'Weather?', + }) + + expect(result.text).toBe('') + expect(result.content.some((part) => part.type === 'tool-error')).toBe(true) + }) + + it('lets the model recover from a broken call on the next step', async () => { + const { fake, provider } = setup() + fake.turns.push( + { + tokens: [ + '{"name":"weather","arguments":{"city":', + ], + }, + { + tokens: [ + '{"name":"weather","arguments":{"city":"Oslo"}}', + ], + }, + { tokens: ['Mild.'] } + ) + + const result = await generateText({ + model: provider(MODEL), + tools: { weather }, + stopWhen: stepCountIs(4), + prompt: 'Weather in Oslo?', + }) + + expect(result.text).toBe('Mild.') + expect(result.steps).toHaveLength(3) + }) + + it('refuses images with a clear error before loading', async () => { + const { fake, provider } = setup() + + await expect( + generateText({ + model: provider(MODEL), + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + image: new Uint8Array(4), + mediaType: 'image/png', + }, + ], + }, + ], + }) + ).rejects.toThrow(/text prompts only/) + expect(fake.loads).toHaveLength(0) + }) +}) + +describe('choosing a file from a repo', () => { + const tree = [ + { + type: 'file', + path: 'gemma-gpu.litertlm', + size: 2, + lfs: { oid: 'a'.repeat(64) }, + }, + { + type: 'file', + path: 'gemma.litertlm', + size: 3, + lfs: { oid: 'b'.repeat(64) }, + }, + { type: 'file', path: 'gemma_NPU_Uncompiled.litertlm', size: 4 }, + ] + + const stubFetch = (status: number, body: unknown) => { + const fetchStub = mock( + async () => new Response(JSON.stringify(body), { status }) + ) + setFetch(fetchStub) + return fetchStub + } + + it('downloads the full bundle on Android and checks its SHA-256', async () => { + const { fake, provider } = setup() + stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ model: provider('hf://org/gemma'), prompt: 'Hi' }) + + expect(fake.loads[0]?.path).toBe( + 'https://huggingface.co/org/gemma/resolve/main/gemma.litertlm' + ) + expect(fake.loads[0]?.options.sha256).toBe('b'.repeat(64)) + }) + + it('downloads the GPU file on iOS', async () => { + const { fake, provider } = setup() + fake.platform = 'ios' + stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ model: provider('hf://org/gemma'), prompt: 'Hi' }) + + expect(fake.loads[0]?.path).toBe( + 'https://huggingface.co/org/gemma/resolve/main/gemma-gpu.litertlm' + ) + expect(fake.loads[0]?.options.sha256).toBe('a'.repeat(64)) + }) + + it('loads the file already on the phone when the Hub does not answer', async () => { + const { fake, provider } = setup() + fake.keptFiles = ['other.litertlm', 'gemma-gpu.litertlm', 'gemma.litertlm'] + setFetch( + mock(async () => { + throw new TypeError('Network request failed') + }) + ) + const warn = spyOn(console, 'warn').mockImplementation(() => {}) + fake.turns.push({ tokens: ['ok'] }) + + const result = await generateText({ + model: provider('hf://org/gemma'), + prompt: 'Hi', + }) + + expect(result.text).toBe('ok') + expect(fake.loads[0]?.path).toBe( + 'https://huggingface.co/org/gemma/resolve/main/gemma.litertlm' + ) + expect(fake.loads[0]?.options.sha256).toBeUndefined() + expect(warn).toHaveBeenCalledOnce() + warn.mockRestore() + }) + + it('loads the file already on the phone when the Hub limits the rate', async () => { + const { fake, provider } = setup() + fake.keptFiles = ['gemma.litertlm'] + stubFetch(429, {}) + const warn = spyOn(console, 'warn').mockImplementation(() => {}) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ model: provider('hf://org/gemma'), prompt: 'Hi' }) + + expect(fake.loads[0]?.path).toContain('gemma.litertlm') + warn.mockRestore() + }) + + it('says the download failed when the Hub does not answer and nothing from the repo is here', async () => { + const { fake, provider } = setup() + fake.keptFiles = ['other.litertlm'] + setFetch( + mock(async () => { + throw new TypeError('Network request failed') + }) + ) + + const error = await generateText({ + model: provider('hf://org/gemma'), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('DOWNLOAD_FAILED') + expect(fake.loads).toHaveLength(0) + }) + + it('loads the file already on the phone when a sign-in page comes back for the list', async () => { + const { fake, provider } = setup() + fake.keptFiles = ['gemma.litertlm'] + setFetch(mock(async () => new Response('Sign in'))) + const warn = spyOn(console, 'warn').mockImplementation(() => {}) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ model: provider('hf://org/gemma'), prompt: 'Hi' }) + + expect(fake.loads[0]?.path).toContain('gemma.litertlm') + warn.mockRestore() + }) + + it('says the download failed when a sign-in page comes back and nothing is kept', async () => { + const { provider } = setup() + setFetch(mock(async () => new Response('Sign in'))) + + const error = await generateText({ + model: provider('hf://org/gemma'), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('DOWNLOAD_FAILED') + }) + + it('does not fall back on a kept file when the repo is gone', async () => { + const { provider, fake } = setup() + fake.keptFiles = ['gemma.litertlm'] + stubFetch(404, {}) + + const error = await generateText({ + model: provider('hf://org/gemma'), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('MODEL_NOT_FOUND') + }) + + it('reads the same id with no scheme', async () => { + const { fake, provider } = setup() + stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('org/gemma', { backend: 'cpu' }), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.path).toBe( + 'https://huggingface.co/org/gemma/resolve/main/gemma.litertlm' + ) + }) + + it('goes straight to a named file, with its hash so another revision of it gets downloaded', async () => { + const { fake, provider } = setup() + stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('hf://org/gemma@abc/gemma.litertlm'), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.path).toBe( + 'https://huggingface.co/org/gemma/resolve/abc/gemma.litertlm' + ) + expect(fake.loads[0]?.options.sha256).toBe('b'.repeat(64)) + expect(fake.manifestCalls).toHaveLength(0) + }) + + it('finds a named file in a folder', async () => { + const { fake, provider } = setup() + const fetchStub = stubFetch(200, [ + { + type: 'file', + path: 'gpu/gemma.litertlm', + lfs: { oid: 'c'.repeat(64) }, + }, + ]) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('hf://org/gemma/gpu/gemma.litertlm'), + prompt: 'Hi', + }) + + expect(fetchStub.mock.calls[0]?.[0 as never]).toBe( + 'https://huggingface.co/api/models/org/gemma/tree/main/gpu' + ) + expect(fake.loads[0]?.options.sha256).toBe('c'.repeat(64)) + }) + + it('loads a named file with no hash when the Hub does not answer', async () => { + const { fake, provider } = setup() + setFetch( + mock(async () => { + throw new TypeError('Network request failed') + }) + ) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('hf://org/gemma/gemma.litertlm'), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.path).toContain('gemma.litertlm') + expect(fake.loads[0]?.options.sha256).toBeUndefined() + }) + + it('takes the sha256 it is given over the Hub, and does not ask', async () => { + const { fake, provider } = setup() + const fetchStub = stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('hf://org/gemma/gemma.litertlm', { + sha256: 'd'.repeat(64), + }), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.options.sha256).toBe('d'.repeat(64)) + expect(fetchStub).not.toHaveBeenCalled() + }) + + it('uses a manifest when the repo has one', async () => { + const { fake, provider } = setup() + fake.manifest = { + url: 'https://huggingface.co/org/m/resolve/main/best.litertlm', + config: { backend: 'gpu', temperature: 0.2 }, + sha256: 'c'.repeat(64), + streamChannels: [ + { type: 'toolCall', start: '', end: '' }, + { type: 'thinking', start: '', end: '' }, + ], + } + fake.turns.push({ tokens: ['HmmHello'] }) + + const result = await generateText({ + model: provider('hf://org/m'), + prompt: 'Hi', + }) + + expect(fake.loads[0]?.path).toContain('best.litertlm') + expect(fake.loads[0]?.config).toEqual({ backend: 'gpu', temperature: 0.2 }) + expect(fake.loads[0]?.options.sha256).toBe('c'.repeat(64)) + expect(result.text).toBe('Hello') + expect(result.reasoningText).toBe('Hmm') + }) + + it('reports a repo that does not exist', async () => { + const { fake, provider } = setup() + stubFetch(404, { error: 'Repository not found' }) + + const error = await generateText({ + model: provider('hf://org/nope'), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('MODEL_NOT_FOUND') + expect(fake.loads).toHaveLength(0) + }) + + it('reports a repo without a usable file and names what it has', async () => { + const { provider } = setup() + stubFetch(200, [{ type: 'file', path: 'x_NPU_y.litertlm', size: 1 }]) + + const error = await generateText({ + model: provider('hf://org/npu-only'), + prompt: 'Hi', + }).catch((e: unknown) => e) + + expect((error as LiteRTError).code).toBe('MODEL_NOT_FOUND') + expect((error as LiteRTError).message).toContain( + 'hf://org/npu-only/x_NPU_y.litertlm' + ) + }) + + it('sends a token for a gated repo', async () => { + const { fake, provider } = setup() + const fetchStub = stubFetch(200, tree) + fake.turns.push({ tokens: ['ok'] }) + + await generateText({ + model: provider('hf://org/gemma', { accessToken: 't' }), + prompt: 'Hi', + }) + + const init = fetchStub.mock.calls[0] as unknown as [string, RequestInit] + expect(init[1].headers).toEqual({ Authorization: 'Bearer t' }) + expect(fake.loads[0]?.options.headers).toEqual({ + Authorization: 'Bearer t', + }) + }) +}) + +describe('the provider', () => { + it('prepares ahead of the first call and reports progress', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['ok'] }) + const model = provider(MODEL) + const progress: unknown[] = [] + + await model.prepare({ onProgress: (p) => progress.push(p) }) + const result = await generateText({ model, prompt: 'Hi' }) + + expect(progress).toEqual([{ fraction: 0.5 }]) + expect(fake.loads).toHaveLength(1) + expect(result.warnings ?? []).toHaveLength(0) + }) + + it('runs calls one at a time', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + + const [first, second] = await Promise.all([ + generateText({ model, prompt: 'One' }), + generateText({ model, prompt: 'Two' }), + ]) + + expect([first.text, second.text]).toEqual(['a', 'b']) + expect(fake.executes.map((e) => e.parts[0].text)).toEqual(['One', 'Two']) + }) + + it('unloads the loaded model before loading another', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + + await generateText({ model: provider('/data/a.litertlm'), prompt: 'Hi' }) + await generateText({ model: provider('/data/b.litertlm'), prompt: 'Hi' }) + + expect(fake.unloads).toBe(1) + expect(fake.closes).toBe(1) + expect(fake.loads.map((load) => load.path)).toEqual([ + '/data/a.litertlm', + '/data/b.litertlm', + ]) + }) + + it('loads again after unload', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }, { tokens: ['b'] }) + const model = provider(MODEL) + + await generateText({ model, prompt: 'Hi' }) + await model.unload() + await generateText({ model, prompt: 'Hi' }) + + expect(fake.loads).toHaveLength(2) + }) + + it('frees everything with unloadAll', async () => { + const { fake, provider } = setup() + fake.turns.push({ tokens: ['a'] }) + + await generateText({ model: provider(MODEL), prompt: 'Hi' }) + await provider.unloadAll() + + expect(fake.unloads).toBe(1) + }) + + it('has no embedding or image models', () => { + const { provider } = setup() + expect(() => provider.embeddingModel('x')).toThrow(/No such embeddingModel/) + expect(() => provider.imageModel('x')).toThrow(/No such imageModel/) + }) + + it('names itself', () => { + const { provider } = setup() + const model = provider(MODEL) + expect(model.specificationVersion).toBe('v4') + expect(model.provider).toBe('litert') + expect(model.modelId).toBe(MODEL) + expect(provider.languageModel(MODEL).modelId).toBe(MODEL) + }) +}) diff --git a/packages/litert/src/backend.ts b/packages/litert/src/backend.ts new file mode 100644 index 00000000..afcf32ba --- /dev/null +++ b/packages/litert/src/backend.ts @@ -0,0 +1,75 @@ +import type { SharedV4Warning } from '@ai-sdk/provider' + +import type { Conversation, ToolSpec } from './prompt' +import type { LiteRTPrepareOptions } from './types' + +/** What one call asks the engine to generate. */ +export interface GenerationRequest { + conversation: Conversation + tools: ToolSpec[] + temperature?: number + topK?: number + topP?: number + seed?: number + maxOutputTokens?: number + frequencyPenalty?: number + presencePenalty?: number + /** JSON Schema the output must follow, as text. An empty string asks for any JSON. */ + jsonSchema?: string + /** `true` or `false` turns the model's reasoning on or off. Undefined keeps the model's default. */ + thinking?: boolean +} + +/** A piece of generated output. */ +export type OutputChunk = + | { type: 'text'; text: string } + | { type: 'reasoning'; text: string } + | { + type: 'tool-call' + /** May be empty when the engine's output had no readable name. */ + name: string + /** The arguments as JSON text. Passed on as is, so `ai` reports it when it doesn't parse. */ + argumentsJson: string + } + +export interface GenerationResult { + continued: boolean + reloaded: boolean + inputTokens?: number + outputTokens?: number + tokensPerSecond?: number + timeToFirstTokenMs?: number + /** The engine stopped at its output limit, before the model finished. */ + cutOff?: boolean +} + +/** + * The runtime under a language model. + * Calls are made one at a time. + */ +export interface LiteRTRuntime { + /** What a call would change in the loaded engine, and what it can't honor. */ + warningsFor(request: GenerationRequest): SharedV4Warning[] + prepare(options?: LiteRTPrepareOptions): Promise + /** + * Generate, sending output through `onChunk`. Resolves when the model stops. + * Aborting the signal stops generation and rejects with an AbortError. + */ + generate( + request: GenerationRequest, + onChunk: (chunk: OutputChunk) => void, + abortSignal?: AbortSignal + ): Promise + unload(): Promise +} + +/** Runs tasks one after another. */ +export class SerialQueue { + private tail: Promise = Promise.resolve() + + run(task: () => Promise): Promise { + const result = this.tail.then(task, task) + this.tail = result.catch(() => {}) + return result + } +} diff --git a/packages/litert/src/errors.ts b/packages/litert/src/errors.ts new file mode 100644 index 00000000..39a8cdd4 --- /dev/null +++ b/packages/litert/src/errors.ts @@ -0,0 +1,53 @@ +import { AISDKError } from '@ai-sdk/provider' + +const name = 'AI_LiteRTError' +const marker = `vercel.ai.error.${name}` +const symbol = Symbol.for(marker) + +export type LiteRTErrorCode = + /** No file at the path, no such Hugging Face repo, or a repo without a usable `.litertlm` file. */ + | 'MODEL_NOT_FOUND' + /** The file exists but is not a readable `.litertlm` bundle. */ + | 'INVALID_MODEL' + /** The download failed. */ + | 'DOWNLOAD_FAILED' + /** The engine ran out of memory, or refused to load because it would. */ + | 'OUT_OF_MEMORY' + /** The bundle needs a newer LiteRT-LM than the installed runtime. */ + | 'RUNTIME_TOO_OLD' + /** The engine could not start for another reason. */ + | 'LOAD_FAILED' + /** The prompt is longer than the conversation can hold. */ + | 'CONTEXT_TOO_LONG' + /** Generation failed for another reason. */ + | 'GENERATION_FAILED' + +/** A failure from LiteRT-LM. Branch on `code`; `cause` holds the engine's own error. */ +export class LiteRTError extends AISDKError { + private readonly [symbol] = true + readonly code: LiteRTErrorCode + + constructor(code: LiteRTErrorCode, message: string, cause?: unknown) { + super({ name, message, cause }) + this.code = code + } + + static override isInstance(error: unknown): error is LiteRTError { + return AISDKError.hasMarker(error, marker) + } +} + +/** The error a cancelled call fails with: what `ai` recognizes as an abort. */ +export function createAbortError(): Error { + const error = new Error('The call was aborted.') + error.name = 'AbortError' + return error +} + +export function isAbortError(error: unknown): boolean { + return error instanceof Error && error.name === 'AbortError' +} + +export function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/packages/litert/src/huggingFace.ts b/packages/litert/src/huggingFace.ts new file mode 100644 index 00000000..72920f9a --- /dev/null +++ b/packages/litert/src/huggingFace.ts @@ -0,0 +1,170 @@ +/** A model on the Hugging Face Hub. */ +export interface HuggingFaceRef { + repo: string + revision: string + /** The file in the repo. When missing, the file is chosen from the repo's top level. */ + path?: string +} + +const BARE_REPO = /^[\w.-]+\/[\w.-]+$/ + +/** + * Read `hf:///[@][/]`, or a bare `/`. + * Returns undefined for anything else: a URL or a local path. + */ +export function parseHuggingFaceId( + modelId: string +): HuggingFaceRef | undefined { + let rest: string + if (modelId.startsWith('hf://')) rest = modelId.slice('hf://'.length) + else if (BARE_REPO.test(modelId) && !modelId.endsWith('.litertlm')) + rest = modelId + else return undefined + + const [org, repoAndRevision, ...path] = rest.split('/') + if (!org || !repoAndRevision) return undefined + const [repo, revision] = repoAndRevision.split('@') + if (!repo) return undefined + return { + repo: `${org}/${repo}`, + revision: revision !== undefined ? decodeURIComponent(revision) : 'main', + path: path.length > 0 ? path.join('/') : undefined, + } +} + +export interface RepoFile { + path: string + size?: number + /** SHA-256 of a file stored with Git LFS. */ + sha256?: string +} + +/** Bundles compiled for one chip, or the browser's copy: never the default on a phone. */ +const NOT_FOR_ANY_PHONE = + /(^|[-_.])(google_tensor|intel|qualcomm|mediatek|npu|web|mt\d{4}|sm\d{4}|qcs\d{4})([-_.]|$)/i +/** A GPU-only bundle: smaller, but it has no CPU path. */ +const GPU_FILE = /[-_]gpu\.litertlm$/i +/** Suffixes of repo names that aren't part of their models' names. */ +const REPO_NAME_SUFFIX = /[-_](litert[-_]?lm|litert)$/i +const INT4 = /int4/i + +function repoBaseName(repo: string): string { + return repo.slice(repo.indexOf('/') + 1).replace(REPO_NAME_SUFFIX, '') +} + +/** + * Choose the `.litertlm` file a phone should load from a repo. A repo can hold a GPU-only + * bundle next to a full one, and which of them comes first differs by platform: + * + * - iOS: the GPU file. On an iPhone 18 Pro the full bundle with its drafter declined a long + * prompt on the GPU. + * - Android: the full bundle. With LiteRT-LM 0.18.1 the GPU file gave wrong text, and no + * error, on a Pixel 11 Pro (PowerVR). The full bundle answered on every phone tried, and + * it can fall back to the CPU. + * + * Among those, the order follows `@litert-lm/core` in the browser, without its first step (a + * file with a GPU-only twin): the file named after the repo, then `model`, then the smallest + * quantization, then the shortest name. + */ +export function selectRepoFile( + files: RepoFile[], + repo: string, + backend: 'gpu' | 'cpu' | 'npu' | undefined, + platform: 'android' | 'ios' +): RepoFile | undefined { + const candidates = files.filter( + (file) => + file.path.toLowerCase().endsWith('.litertlm') && + !NOT_FOR_ANY_PHONE.test(file.path) + ) + const gpuFiles = candidates.filter((file) => GPU_FILE.test(file.path)) + const otherFiles = candidates.filter((file) => !GPU_FILE.test(file.path)) + let matching: RepoFile[] + if (backend === 'cpu') matching = otherFiles + else if (platform === 'ios') + matching = gpuFiles.length > 0 ? gpuFiles : otherFiles + else matching = otherFiles.length > 0 ? otherFiles : gpuFiles + + const repoName = repoBaseName(repo).toLowerCase() + const score = (file: RepoFile): number[] => { + const path = file.path.toLowerCase() + const name = path.replace(GPU_FILE, '').replace(/\.litertlm$/, '') + return [ + name === repoName ? 0 : 1, + name === 'model' ? 0 : 1, + INT4.test(name) ? 0 : 1, + path.length, + ] + } + return [...matching].sort((a, b) => { + const [sa, sb] = [score(a), score(b)] + const at = sa.findIndex((value, i) => value !== sb[i]) + if (at !== -1) return (sa[at] as number) - (sb[at] as number) + return a.path < b.path ? -1 : 1 + })[0] +} + +/** Letters and digits only: `Qwen3-0.6B`, `qwen3_0_6b` and `Qwen3_0.6B` read the same. */ +const plainName = (name: string) => name.toLowerCase().replace(/[^a-z0-9]/g, '') + +/** What follows a model's name in a file name when it says how the file was built. */ +const BUILD_SUFFIX = + /^(gpu|cpu|mixed|dynamic|int\d|q\d|fp\d|f\d|multi|fixed|seq|ekv)/ + +/** + * The file already on the phone for a repo, when the Hub can't say which it is. The phone's + * cache holds file names, not repos, so this goes by name: the repo's name, then nothing or + * how the file was built. Anything else may be another model (`Qwen3` and `Qwen3-8B`). + */ +export function selectKeptFile( + fileNames: string[], + repo: string, + backend: 'gpu' | 'cpu' | 'npu' | undefined, + platform: 'android' | 'ios' +): RepoFile | undefined { + const repoName = plainName(repoBaseName(repo)) + const fromRepo = fileNames + .filter((name) => { + const plain = plainName(name.replace(/\.litertlm$/i, '')) + if (!plain.startsWith(repoName)) return false + const rest = plain.slice(repoName.length) + return rest === '' || BUILD_SUFFIX.test(rest) + }) + .map((path) => ({ path })) + return selectRepoFile(fromRepo, repo, backend, platform) +} + +export function resolveUrl(ref: HuggingFaceRef, path: string): string { + const encodedPath = path.split('/').map(encodeURIComponent).join('/') + return `https://huggingface.co/${ref.repo}/resolve/${encodeURIComponent(ref.revision)}/${encodedPath}` +} + +/** The Hub's list of a folder in a repo: its top level, when no folder is given. */ +export function treeUrl(ref: HuggingFaceRef, folder?: string): string { + const url = `https://huggingface.co/api/models/${ref.repo}/tree/${encodeURIComponent(ref.revision)}` + return folder !== undefined + ? `${url}/${folder.split('/').map(encodeURIComponent).join('/')}` + : url +} + +/** Read the Hub's tree listing into the fields needed to choose and check a file. */ +export function parseTree(body: unknown): RepoFile[] { + if (!Array.isArray(body)) return [] + return body.flatMap((item): RepoFile[] => { + if (typeof item !== 'object' || item === null) return [] + const entry = item as { + type?: unknown + path?: unknown + size?: unknown + lfs?: { oid?: unknown } + } + if (entry.type !== 'file' || typeof entry.path !== 'string') return [] + return [ + { + path: entry.path, + size: typeof entry.size === 'number' ? entry.size : undefined, + sha256: typeof entry.lfs?.oid === 'string' ? entry.lfs.oid : undefined, + }, + ] + }) +} diff --git a/packages/litert/src/index.ts b/packages/litert/src/index.ts new file mode 100644 index 00000000..4909c92a --- /dev/null +++ b/packages/litert/src/index.ts @@ -0,0 +1,41 @@ +import { + createLLM, + getDeviceInfo, + ModelRegistry, + resolveFromManifest, + toolResultPart, +} from 'react-native-litert-lm' + +import { createProviderWithRuntime } from './provider' +import { createRuntime, type LiteRTBinding } from './runtime' +import type { LiteRTProvider } from './types' + +export { LiteRTError, type LiteRTErrorCode } from './errors' +export type { + LiteRTBackendName, + LiteRTCallMetadata, + LiteRTLanguageModel, + LiteRTModelSettings, + LiteRTPrepareOptions, + LiteRTProgress, + LiteRTProvider, +} from './types' + +const binding: LiteRTBinding = { + createLLM: createLLM as LiteRTBinding['createLLM'], + getDeviceInfo, + listKeptFiles: () => + ModelRegistry.listCachedFiles().map((file) => file.fileName), + resolveFromManifest, + toolResultPart, +} + +/** Create a LiteRT-LM provider. Most apps use the default {@link litert}. */ +export function createLiteRTProvider(): LiteRTProvider { + return createProviderWithRuntime((modelId, settings) => + createRuntime(modelId, settings, binding) + ) +} + +/** The default LiteRT-LM provider: `litert('hf://org/repo')`. */ +export const litert = createLiteRTProvider() diff --git a/packages/litert/src/languageModel.ts b/packages/litert/src/languageModel.ts new file mode 100644 index 00000000..5d119641 --- /dev/null +++ b/packages/litert/src/languageModel.ts @@ -0,0 +1,324 @@ +import type { + LanguageModelV4CallOptions, + LanguageModelV4Content, + LanguageModelV4FinishReason, + LanguageModelV4GenerateResult, + LanguageModelV4StreamPart, + LanguageModelV4StreamResult, + LanguageModelV4Usage, + SharedV4ProviderMetadata, + SharedV4Warning, +} from '@ai-sdk/provider' + +import type { + GenerationRequest, + GenerationResult, + LiteRTRuntime, + OutputChunk, + SerialQueue, +} from './backend' +import { createAbortError } from './errors' +import { canonicalJson, selectTools, toConversation } from './prompt' +import { + type LiteRTCallMetadata, + type LiteRTLanguageModel, + type LiteRTPrepareOptions, + PROVIDER_NAME, +} from './types' + +let idCounter = 0 +function nextId(prefix: string): string { + idCounter += 1 + return `${prefix}-${Date.now().toString(36)}-${idCounter.toString(36)}` +} + +/** A model asked for JSON may still wrap it in a code fence, which `ai` can't parse. */ +function stripCodeFence(text: string): string { + const match = /^\s*```(?:json)?\s*\n?([\s\S]*?)\n?\s*```\s*$/i.exec(text) + return match?.[1] ?? text.trim() +} + +/** A tool name the model's output did not give. `ai` reports the call as a tool error. */ +const UNREADABLE_TOOL_NAME = 'unreadable_tool_call' + +function toToolCall(chunk: Extract) { + return { + type: 'tool-call' as const, + toolCallId: nextId('call'), + toolName: chunk.name.length > 0 ? chunk.name : UNREADABLE_TOOL_NAME, + input: chunk.argumentsJson, + } +} + +function buildRequest(options: LanguageModelV4CallOptions): { + request: GenerationRequest + warnings: SharedV4Warning[] +} { + const warnings: SharedV4Warning[] = [] + const conversation = toConversation(options.prompt) + const tools = selectTools(options, warnings) + + if (options.stopSequences !== undefined && options.stopSequences.length > 0) { + warnings.push({ type: 'unsupported', feature: 'stopSequences' }) + } + + let jsonSchema: string | undefined + if (options.responseFormat?.type === 'json') { + jsonSchema = + options.responseFormat.schema !== undefined + ? canonicalJson(options.responseFormat.schema) + : '' + } + + let thinking: boolean | undefined + if ( + options.reasoning !== undefined && + options.reasoning !== 'provider-default' + ) { + thinking = options.reasoning !== 'none' + } + + return { + warnings, + request: { + conversation, + tools, + temperature: options.temperature, + topK: options.topK, + topP: options.topP, + seed: options.seed, + maxOutputTokens: options.maxOutputTokens, + frequencyPenalty: options.frequencyPenalty, + presencePenalty: options.presencePenalty, + jsonSchema, + thinking, + }, + } +} + +function toUsage(result: GenerationResult): LanguageModelV4Usage { + return { + inputTokens: { + total: result.inputTokens, + noCache: undefined, + cacheRead: undefined, + cacheWrite: undefined, + }, + outputTokens: { + total: result.outputTokens, + text: undefined, + reasoning: undefined, + }, + } +} + +function toFinishReason( + result: GenerationResult, + sawToolCall: boolean +): LanguageModelV4FinishReason { + if (sawToolCall) return { unified: 'tool-calls', raw: 'tool-calls' } + if (result.cutOff === true) return { unified: 'length', raw: 'length' } + return { unified: 'stop', raw: 'stop' } +} + +function toProviderMetadata( + result: GenerationResult +): SharedV4ProviderMetadata { + const metadata: LiteRTCallMetadata = { + continued: result.continued, + reloaded: result.reloaded, + tokensPerSecond: result.tokensPerSecond, + timeToFirstTokenMs: result.timeToFirstTokenMs, + } + // Keep only defined values: provider metadata must be JSON. + return { + [PROVIDER_NAME]: JSON.parse(JSON.stringify(metadata)) as Record< + string, + never + >, + } +} + +export class LanguageModel implements LiteRTLanguageModel { + readonly specificationVersion = 'v4' as const + readonly provider = PROVIDER_NAME + readonly supportedUrls: Record = {} + + constructor( + readonly modelId: string, + private readonly runtime: LiteRTRuntime, + /** Shared by a provider's models: phones and browsers run one engine at a time. */ + private readonly queue: SerialQueue + ) {} + + prepare(options?: LiteRTPrepareOptions): Promise { + return this.queue.run(() => this.runtime.prepare(options)) + } + + unload(): Promise { + return this.queue.run(() => this.runtime.unload()) + } + + async doGenerate( + options: LanguageModelV4CallOptions + ): Promise { + const { request, warnings } = buildRequest(options) + + let text = '' + let reasoning = '' + const toolCalls: LanguageModelV4Content[] = [] + const result = await this.queue.run(() => { + if (options.abortSignal?.aborted) throw createAbortError() + // Read once it's this call's turn: what it changes depends on the calls before it. + warnings.push(...this.runtime.warningsFor(request)) + return this.runtime.generate( + request, + (chunk) => { + if (chunk.type === 'text') text += chunk.text + else if (chunk.type === 'reasoning') reasoning += chunk.text + else toolCalls.push(toToolCall(chunk)) + }, + options.abortSignal + ) + }) + + if (request.jsonSchema !== undefined) text = stripCodeFence(text) + const content: LanguageModelV4Content[] = [] + if (reasoning.length > 0) + content.push({ type: 'reasoning', text: reasoning }) + if (text.length > 0) content.push({ type: 'text', text }) + content.push(...toolCalls) + + return { + content, + finishReason: toFinishReason(result, toolCalls.length > 0), + usage: toUsage(result), + providerMetadata: toProviderMetadata(result), + warnings, + } + } + + async doStream( + options: LanguageModelV4CallOptions + ): Promise { + if (typeof ReadableStream === 'undefined') { + throw new TypeError( + 'ReadableStream is not available here. Bare React Native apps need a polyfill such as web-streams-polyfill.' + ) + } + const { request, warnings } = buildRequest(options) + + // A consumer that cancels the stream stops generation the same way the caller's signal does. + const abort = new AbortController() + const onCallerAbort = () => abort.abort() + if (options.abortSignal?.aborted) abort.abort() + else options.abortSignal?.addEventListener('abort', onCallerAbort) + + const stream = new ReadableStream({ + start: (controller) => { + // Tokens already queued natively can arrive after a cancel, and enqueueing + // on a cancelled stream throws inside the native callback. + const enqueue = (part: LanguageModelV4StreamPart) => { + if (!abort.signal.aborted) controller.enqueue(part) + } + + // A consumer that cancelled the stream is gone. A caller's abort ends it like a fetch's. + const endAborted = () => { + if (options.abortSignal?.aborted !== true) return + try { + controller.error(createAbortError()) + } catch { + // The consumer already cancelled the stream. + } + } + + let open: { type: 'text' | 'reasoning'; id: string } | undefined + const close = () => { + if (open === undefined) return + enqueue({ type: `${open.type}-end`, id: open.id }) + open = undefined + } + const openPart = (type: 'text' | 'reasoning'): string => { + if (open?.type === type) return open.id + close() + open = { type, id: nextId(type) } + enqueue({ type: `${type}-start`, id: open.id }) + return open.id + } + + // JSON goes out whole, once its code fence (if any) can be removed. + let bufferedJson = request.jsonSchema !== undefined ? '' : undefined + let sawToolCall = false + const onChunk = (chunk: OutputChunk) => { + if (chunk.type === 'tool-call') { + close() + sawToolCall = true + enqueue(toToolCall(chunk)) + return + } + if (chunk.type === 'text' && bufferedJson !== undefined) { + bufferedJson += chunk.text + return + } + const id = openPart(chunk.type) + enqueue({ type: `${chunk.type}-delta`, id, delta: chunk.text }) + } + + void this.queue + .run(() => { + if (abort.signal.aborted) throw createAbortError() + // Read once it's this call's turn: what it changes depends on the calls before it. + warnings.push(...this.runtime.warningsFor(request)) + enqueue({ type: 'stream-start', warnings }) + return this.runtime.generate(request, onChunk, abort.signal) + }) + .then( + (result) => { + if (abort.signal.aborted) { + endAborted() + return + } + if (bufferedJson !== undefined && bufferedJson.length > 0) { + const id = openPart('text') + enqueue({ + type: 'text-delta', + id, + delta: stripCodeFence(bufferedJson), + }) + } + close() + enqueue({ + type: 'finish', + finishReason: toFinishReason(result, sawToolCall), + usage: toUsage(result), + providerMetadata: toProviderMetadata(result), + }) + controller.close() + }, + (error: unknown) => { + if (abort.signal.aborted) { + endAborted() + return + } + close() + enqueue({ type: 'error', error }) + enqueue({ + type: 'finish', + finishReason: { unified: 'error', raw: undefined }, + usage: toUsage({ continued: false, reloaded: false }), + }) + controller.close() + } + ) + .finally(() => + options.abortSignal?.removeEventListener('abort', onCallerAbort) + ) + }, + cancel: () => { + abort.abort() + }, + }) + + return { stream } + } +} diff --git a/packages/litert/src/outputParser.ts b/packages/litert/src/outputParser.ts new file mode 100644 index 00000000..e2b3d8c8 --- /dev/null +++ b/packages/litert/src/outputParser.ts @@ -0,0 +1,166 @@ +import type { OutputChunk } from './backend' + +/** + * Turn the JSON text inside a `` marker into a tool call. Text that + * doesn't parse still becomes a call, so `ai` reports it as a tool error the + * model can recover from, instead of showing the marker to the user. + */ +export function toolCallFromBody( + body: string +): Extract { + try { + const parsed = JSON.parse(body) as { + name?: unknown + arguments?: unknown + } | null + if ( + parsed !== null && + typeof parsed === 'object' && + typeof parsed.name === 'string' + ) { + return { + type: 'tool-call', + name: parsed.name, + argumentsJson: JSON.stringify(parsed.arguments ?? {}), + } + } + } catch { + // Handled below. + } + const name = /"name"\s*:\s*"([^"]*)"/.exec(body)?.[1] ?? '' + return { type: 'tool-call', name, argumentsJson: body } +} + +export interface Channel { + type: 'reasoning' | 'tool-call' + start: string + end: string + /** Only opens before any text. Later, the marker is text: an answer can be about the tag. */ + leading?: boolean +} + +export const DEFAULT_CHANNELS: Channel[] = [ + { type: 'tool-call', start: '', end: '' }, + { type: 'reasoning', start: '', end: '', leading: true }, + // An older bundle's chat template leaves the model's own tags in the text. + { type: 'reasoning', start: '', end: '', leading: true }, +] + +/** + * Split a token stream at its channel markers. Works on partial tokens: text that + * could be the start of a marker waits for the next one. + */ +export class OutputParser { + private buffer = '' + private active: Channel | undefined + private toolBody = '' + /** Drop the blank lines a model writes around its reasoning. */ + private trimNext = false + private sawText = false + + constructor( + private readonly channels: Channel[], + private readonly emit: (chunk: OutputChunk) => void + ) {} + + push(token: string): void { + this.buffer += token + this.drain(false) + } + + /** The model stopped: flush what's left. An unterminated tool call still becomes a call. */ + finish(): void { + this.drain(true) + if (this.active?.type === 'tool-call') { + this.emit(toolCallFromBody(this.toolBody)) + this.toolBody = '' + } + this.active = undefined + } + + /** Whether a channel's marker at this offset in the buffer opens it. */ + private canOpen(channel: Channel, at: number): boolean { + if (channel.leading !== true) return true + return !this.sawText && this.buffer.slice(0, at).trim().length === 0 + } + + private drain(flush: boolean): void { + for (;;) { + if (this.active === undefined) { + let next: Channel | undefined + let at = -1 + for (const channel of this.channels) { + const index = this.buffer.indexOf(channel.start) + if ( + index !== -1 && + this.canOpen(channel, index) && + (at === -1 || index < at) + ) { + at = index + next = channel + } + } + if (next === undefined) break + this.emitContent(this.buffer.slice(0, at)) + this.buffer = this.buffer.slice(at + next.start.length) + this.active = next + if (next.type === 'reasoning') this.trimNext = true + } else { + const index = this.buffer.indexOf(this.active.end) + if (index === -1) break + this.emitContent(this.buffer.slice(0, index)) + this.buffer = this.buffer.slice(index + this.active.end.length) + if (this.active.type === 'tool-call') { + this.emit(toolCallFromBody(this.toolBody)) + this.toolBody = '' + } else { + this.trimNext = true + } + this.active = undefined + } + } + + const hold = flush ? 0 : this.partialMarkerLength() + const ready = this.buffer.slice(0, this.buffer.length - hold) + this.buffer = this.buffer.slice(this.buffer.length - hold) + this.emitContent(ready) + } + + private emitContent(raw: string): void { + if (this.active?.type === 'tool-call') { + this.toolBody += raw + return + } + const text = this.trimNext ? raw.trimStart() : raw + if (text.length === 0) return + this.trimNext = false + if (this.active?.type === 'reasoning') { + this.emit({ type: 'reasoning', text }) + return + } + if (text.trim().length > 0) this.sawText = true + this.emit({ type: 'text', text }) + } + + /** Length of the buffer's tail that could still grow into a marker. */ + private partialMarkerLength(): number { + const markers: { text: string; channel?: Channel }[] = + this.active !== undefined + ? [{ text: this.active.end }] + : this.channels.map((channel) => ({ text: channel.start, channel })) + let longest = 0 + for (const { text, channel } of markers) { + const max = Math.min(this.buffer.length, text.length - 1) + for (let length = max; length > longest; length--) { + const opens = + channel === undefined || + this.canOpen(channel, this.buffer.length - length) + if (opens && this.buffer.endsWith(text.slice(0, length))) { + longest = length + break + } + } + } + return longest + } +} diff --git a/packages/litert/src/prompt.ts b/packages/litert/src/prompt.ts new file mode 100644 index 00000000..e98083bb --- /dev/null +++ b/packages/litert/src/prompt.ts @@ -0,0 +1,229 @@ +import { + type LanguageModelV4CallOptions, + type LanguageModelV4Prompt, + type LanguageModelV4ToolResultOutput, + type SharedV4Warning, + UnsupportedFunctionalityError, +} from '@ai-sdk/provider' + +/** A tool the model may call. */ +export interface ToolSpec { + name: string + description?: string + /** JSON Schema of the tool's arguments. */ + parameters: unknown +} + +export interface ToolCallSpec { + name: string + arguments: unknown +} + +export interface ToolResultSpec { + name: string + /** The tool's result, as a JSON value. */ + value: unknown +} + +/** One turn of a conversation, in a form both runtimes can be given. */ +export type Turn = + | { role: 'user'; text: string } + | { role: 'assistant'; text: string; toolCalls: ToolCallSpec[] } + | { role: 'tool'; results: ToolResultSpec[] } + +/** The last turn: what the model answers now. */ +export type FinalTurn = Extract + +/** A prompt split the way a conversation holds it. */ +export interface Conversation { + system: string | undefined + /** Every turn before the final one. */ + history: Turn[] + final: FinalTurn +} + +function toolOutputValue(output: LanguageModelV4ToolResultOutput): unknown { + switch (output.type) { + case 'text': + case 'json': + return output.value + case 'error-text': + case 'error-json': + return { error: output.value } + case 'execution-denied': + return { error: output.reason ?? 'The tool call was denied.' } + case 'content': + return output.value + .map((part) => (part.type === 'text' ? part.text : '')) + .join('') + } +} + +function fileParts(): UnsupportedFunctionalityError { + return new UnsupportedFunctionalityError({ + functionality: 'file parts (images and audio) in the prompt', + message: + 'ai-sdk-litert-lm supports text prompts only. Remove the file part from the prompt.', + }) +} + +/** Split a prompt into system text, the turns before the last one, and the last turn. */ +export function toConversation(prompt: LanguageModelV4Prompt): Conversation { + const system: string[] = [] + const turns: Turn[] = [] + + for (const message of prompt) { + switch (message.role) { + case 'system': + system.push(message.content) + break + case 'user': { + let text = '' + for (const part of message.content) { + if (part.type === 'file') throw fileParts() + text += part.text + } + turns.push({ role: 'user', text }) + break + } + case 'assistant': { + let text = '' + const toolCalls: ToolCallSpec[] = [] + for (const part of message.content) { + // Reasoning is not replayed: the chat templates drop it from history too. + if (part.type === 'text') text += part.text + else if (part.type === 'tool-call') { + toolCalls.push({ name: part.toolName, arguments: part.input }) + } else if (part.type === 'file') throw fileParts() + } + turns.push({ role: 'assistant', text, toolCalls }) + break + } + case 'tool': { + const results: ToolResultSpec[] = [] + for (const part of message.content) { + if (part.type !== 'tool-result') continue + results.push({ + name: part.toolName, + value: toolOutputValue(part.output), + }) + } + if (results.length > 0) turns.push({ role: 'tool', results }) + break + } + } + } + + const final = turns.pop() + if (final === undefined || final.role === 'assistant') { + throw new UnsupportedFunctionalityError({ + functionality: 'a prompt that does not end with a user or tool message', + message: + 'The last message of the prompt must be a user message or a tool result.', + }) + } + return { + system: system.length > 0 ? system.join('\n\n') : undefined, + history: turns, + final, + } +} + +/** The tools for this call, after `toolChoice`. Adds a warning for what the engine can't do. */ +export function selectTools( + options: Pick, + warnings: SharedV4Warning[] +): ToolSpec[] { + const choice = options.toolChoice + if (choice?.type === 'none') return [] + + const tools: ToolSpec[] = [] + for (const tool of options.tools ?? []) { + if (tool.type !== 'function') { + warnings.push({ + type: 'unsupported', + feature: `provider tool ${tool.name}`, + details: 'Only function tools run on the device.', + }) + continue + } + tools.push({ + name: tool.name, + description: tool.description, + parameters: tool.inputSchema, + }) + } + + if (choice?.type === 'required') { + warnings.push({ + type: 'unsupported', + feature: 'toolChoice: required', + details: 'The engine cannot force a tool call. The model decides.', + }) + } + if (choice?.type === 'tool') { + warnings.push({ + type: 'unsupported', + feature: 'toolChoice: tool', + details: `The engine cannot force a tool call. Only ${choice.toolName} is offered, and the model decides.`, + }) + return tools.filter((tool) => tool.name === choice.toolName) + } + return tools +} + +function sorted(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sorted) + if (value !== null && typeof value === 'object') { + const source = value as Record + const result: Record = {} + for (const key of Object.keys(source).sort()) + result[key] = sorted(source[key]) + return result + } + return value +} + +/** JSON with sorted object keys: the same value always gives the same text. */ +export function canonicalJson(value: unknown): string { + return JSON.stringify(sorted(value)) ?? 'null' +} + +function normalizeTurn(turn: Turn): unknown { + switch (turn.role) { + case 'user': + return { role: 'user', text: turn.text } + case 'assistant': + // Apps may trim a reply before sending it back. + return { + role: 'assistant', + text: turn.text.trim(), + toolCalls: turn.toolCalls, + } + case 'tool': + return turn + } +} + +/** Identifies a list of turns, to tell whether the engine already holds them. */ +export function transcriptKey( + system: string | undefined, + turns: Turn[] +): string { + return canonicalJson({ + system: system ?? null, + turns: turns.map(normalizeTurn), + }) +} + +/** Ask for JSON in the system text, where the engine can't constrain the output to it. */ +export function withJsonInstruction( + system: string | undefined, + schema: string +): string { + const instruction = + schema === '' + ? 'Answer with one JSON value only, with no other text and no code fences.' + : `Answer with one JSON value only, with no other text and no code fences. It must follow this JSON Schema: ${schema}` + return system !== undefined ? `${system}\n\n${instruction}` : instruction +} diff --git a/packages/litert/src/provider.ts b/packages/litert/src/provider.ts new file mode 100644 index 00000000..d71bdfae --- /dev/null +++ b/packages/litert/src/provider.ts @@ -0,0 +1,86 @@ +import { NoSuchModelError } from '@ai-sdk/provider' + +import { type LiteRTRuntime, SerialQueue } from './backend' +import { LanguageModel } from './languageModel' +import type { + LiteRTLanguageModel, + LiteRTModelSettings, + LiteRTProvider, +} from './types' + +export type RuntimeFactory = ( + modelId: string, + settings: LiteRTModelSettings +) => LiteRTRuntime + +function settingsKey(modelId: string, settings: LiteRTModelSettings): string { + // Functions don't identify a model: a new callback must not load it again. + return JSON.stringify({ modelId, settings }) +} + +/** + * A provider whose models share one queue and one loaded engine: a second model + * unloads the first, because two models rarely fit in a phone's memory. + */ +export function createProviderWithRuntime( + createRuntime: RuntimeFactory +): LiteRTProvider { + const queue = new SerialQueue() + const runtimes = new Map() + let active: LiteRTRuntime | undefined + + const exclusive = (runtime: LiteRTRuntime): LiteRTRuntime => { + const makeActive = async () => { + if (active !== undefined && active !== runtime) await active.unload() + active = runtime + } + return { + warningsFor: (request) => runtime.warningsFor(request), + prepare: async (options) => { + await makeActive() + await runtime.prepare(options) + }, + generate: async (request, onChunk, abortSignal) => { + await makeActive() + return runtime.generate(request, onChunk, abortSignal) + }, + unload: async () => { + if (active === runtime) active = undefined + await runtime.unload() + }, + } + } + + const languageModel = ( + modelId: string, + settings: LiteRTModelSettings = {} + ): LiteRTLanguageModel => { + const key = settingsKey(modelId, settings) + let runtime = runtimes.get(key) + if (runtime === undefined) { + runtime = exclusive(createRuntime(modelId, settings)) + runtimes.set(key, runtime) + } + return new LanguageModel(modelId, runtime, queue) + } + + const provider = ((modelId: string, settings?: LiteRTModelSettings) => + languageModel(modelId, settings)) as LiteRTProvider + + return Object.assign(provider, { + specificationVersion: 'v4' as const, + languageModel, + embeddingModel: (modelId: string): never => { + throw new NoSuchModelError({ modelId, modelType: 'embeddingModel' }) + }, + imageModel: (modelId: string): never => { + throw new NoSuchModelError({ modelId, modelType: 'imageModel' }) + }, + unloadAll: () => + queue.run(async () => { + const current = active + active = undefined + await current?.unload() + }), + }) +} diff --git a/packages/litert/src/runtime.ts b/packages/litert/src/runtime.ts new file mode 100644 index 00000000..479a7f4f --- /dev/null +++ b/packages/litert/src/runtime.ts @@ -0,0 +1,681 @@ +import type { SharedV4Warning } from '@ai-sdk/provider' +import type { + getDeviceInfo, + LiteRTLMInstance, + LLMConfig, + Message, + ModelDownloadOptions, + MultimodalPart, + resolveFromManifest, + toolResultPart, +} from 'react-native-litert-lm' + +import type { + GenerationRequest, + GenerationResult, + LiteRTRuntime, + OutputChunk, +} from './backend' +import { + createAbortError, + errorMessage, + LiteRTError, + type LiteRTErrorCode, +} from './errors' +import { + type HuggingFaceRef, + parseHuggingFaceId, + parseTree, + type RepoFile, + resolveUrl, + selectKeptFile, + selectRepoFile, + treeUrl, +} from './huggingFace' +import { type Channel, DEFAULT_CHANNELS, OutputParser } from './outputParser' +import { + canonicalJson, + type ToolSpec, + transcriptKey, + type Turn, + withJsonInstruction, +} from './prompt' +import type { LiteRTModelSettings, LiteRTPrepareOptions } from './types' + +type ExecuteOptions = NonNullable[2]> + +/** The parts of `react-native-litert-lm` the runtime uses. Replaced in tests. */ +export interface LiteRTBinding { + createLLM(): LiteRTLMInstance + getDeviceInfo: typeof getDeviceInfo + /** The names of the files already downloaded. */ + listKeptFiles(): string[] + resolveFromManifest: typeof resolveFromManifest + toolResultPart: typeof toolResultPart +} + +interface LoadedModel { + llm: LiteRTLMInstance + /** The configuration the engine holds. Part of it can only change with a reload. */ + configKey: string +} + +/** What the model holds in its conversation, to skip replaying it on the next call. */ +interface ConversationState { + system: string | undefined + /** Identifies every turn the engine holds, the model's last answer included. */ + key: string +} + +function historyMessage(turn: Turn): Message[] { + switch (turn.role) { + case 'user': + return [{ role: 'user', content: turn.text }] + case 'assistant': + return [ + { + role: 'model', + content: turn.text, + toolCalls: + turn.toolCalls.length > 0 + ? turn.toolCalls.map((call) => ({ + name: call.name, + argumentsJson: + typeof call.arguments === 'string' + ? call.arguments + : JSON.stringify(call.arguments ?? {}), + })) + : undefined, + }, + ] + case 'tool': + return turn.results.map((result) => ({ + role: 'tool', + content: JSON.stringify(result.value) ?? 'null', + toolName: result.name, + })) + } +} + +function toNativeTools(tools: ToolSpec[]): LLMConfig['tools'] { + return tools.map((tool) => ({ + name: tool.name, + description: tool.description ?? '', + parametersJson: JSON.stringify(tool.parameters ?? {}), + })) +} + +function toLiteRTError( + error: unknown, + phase: 'load' | 'generate' +): LiteRTError { + if (error instanceof LiteRTError) return error + const message = errorMessage(error) + const code = (error as { code?: unknown } | null)?.code + const known: Record = { + MODEL_NOT_FOUND: 'MODEL_NOT_FOUND', + INVALID_MODEL: 'INVALID_MODEL', + MODEL_TYPE_MISMATCH: 'INVALID_MODEL', + DOWNLOAD_FAILED: 'DOWNLOAD_FAILED', + OUT_OF_MEMORY: 'OUT_OF_MEMORY', + MEMORY_ESTIMATE_EXCEEDED: 'OUT_OF_MEMORY', + RUNTIME_TOO_OLD: 'RUNTIME_TOO_OLD', + ENGINE_INIT_FAILED: 'LOAD_FAILED', + INVALID_CONFIG: 'LOAD_FAILED', + MODEL_NOT_LOADED: 'LOAD_FAILED', + } + // The binding reads the engine's NOT_FOUND as a missing file, but it also means a file + // that exists and lacks a section the engine needs: a truncated download, for example. + if ( + (code === 'MODEL_NOT_FOUND' && message.includes('Failed to create')) || + message.includes('Invalid magic number') || + message.includes('Unsupported or unknown file format') + ) { + return new LiteRTError('INVALID_MODEL', message, error) + } + if (message.includes('Input token ids are too long')) { + return new LiteRTError('CONTEXT_TOO_LONG', message, error) + } + if ( + code === 'DOWNLOAD_FAILED' && + /\bHTTP (401|403|404|410)\b/.test(message) + ) { + return new LiteRTError('MODEL_NOT_FOUND', message, error) + } + const mapped = typeof code === 'string' ? known[code] : undefined + return new LiteRTError( + mapped ?? (phase === 'load' ? 'LOAD_FAILED' : 'GENERATION_FAILED'), + message, + error + ) +} + +function isCancellation(error: unknown): boolean { + return (error as { code?: unknown } | null)?.code === 'INFERENCE_CANCELLED' +} + +/** The engine reports a failed stream as a last token, after the promise has the error. */ +const ERROR_TOKEN = /^Error: / + +/** What `react-native-litert-lm` stops a reply at when nothing sets `maxOutputTokens`. */ +const DEFAULT_MAX_OUTPUT_TOKENS = 1024 + +/** The engine constrains output to a schema, leaving no way to call a tool. It has no "any JSON". */ +function usesConstraint(request: GenerationRequest): boolean { + return ( + request.jsonSchema !== undefined && + request.jsonSchema !== '' && + request.tools.length === 0 + ) +} + +/** Wait for the work, or reject as soon as the signal aborts, leaving the work to finish. */ +function untilAborted( + work: Promise, + signal: AbortSignal | undefined +): Promise { + if (signal === undefined) return work + return new Promise((resolve, reject) => { + const onAbort = () => reject(createAbortError()) + if (signal.aborted) onAbort() + else signal.addEventListener('abort', onAbort, { once: true }) + work + .then(resolve, reject) + .finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + +export function createRuntime( + modelId: string, + settings: LiteRTModelSettings, + binding: LiteRTBinding +): LiteRTRuntime { + let loaded: LoadedModel | undefined + let conversation: ConversationState | undefined + let modelPath: string | undefined + let manifestConfig: Partial = {} + let channels: Channel[] = DEFAULT_CHANNELS + let downloadOptions: ModelDownloadOptions | undefined + + const headers: Record | undefined = + settings.accessToken !== undefined + ? { Authorization: `Bearer ${settings.accessToken}` } + : undefined + + /** The Hub didn't answer, as with no network: a file from this repo that's here still loads. */ + const pickKeptFile = (ref: HuggingFaceRef): boolean => { + const file = selectKeptFile( + binding.listKeptFiles(), + ref.repo, + settings.backend, + binding.getDeviceInfo().platform + ) + if (file === undefined) return false + console.warn( + `Could not ask the Hub about ${ref.repo}, so this loads ${file.path}, already on the phone.` + ) + modelPath = resolveUrl(ref, file.path) + downloadOptions = { headers, sha256: settings.sha256 } + return true + } + + /** + * A named file's hash, when the Hub answers. The phone keeps one file per name, and the hash + * is what tells it from another revision's or another repo's file of the same name. + */ + const hashFromHub = async ( + ref: HuggingFaceRef, + path: string, + signal?: AbortSignal + ): Promise => { + const slash = path.lastIndexOf('/') + try { + const response = await fetch( + treeUrl(ref, slash === -1 ? undefined : path.slice(0, slash)), + { headers, signal } + ) + if (!response.ok) return undefined + return parseTree(await response.json()).find((file) => file.path === path) + ?.sha256 + } catch { + if (signal?.aborted) throw createAbortError() + return undefined + } + } + + /** The file to load: a local path, a URL, or the file chosen from a Hugging Face repo. */ + const resolveSource = async (signal?: AbortSignal): Promise => { + if (modelPath !== undefined) return + const ref: HuggingFaceRef | undefined = parseHuggingFaceId(modelId) + if (ref === undefined) { + modelPath = modelId + downloadOptions = { headers, sha256: settings.sha256 } + return + } + if (ref.path !== undefined) { + const sha256 = + settings.sha256 ?? (await hashFromHub(ref, ref.path, signal)) + modelPath = resolveUrl(ref, ref.path) + downloadOptions = { headers, sha256 } + return + } + + // A repo can say which file and settings fit each phone. Never throws: null means no manifest. + const fromManifest = await binding.resolveFromManifest(ref.repo, { + revision: ref.revision, + backend: settings.backend, + signal, + }) + if (signal?.aborted) throw createAbortError() + if (fromManifest !== null) { + modelPath = fromManifest.url + manifestConfig = fromManifest.config + channels = fromManifest.streamChannels.map((channel) => ({ + type: channel.type === 'toolCall' ? 'tool-call' : 'reasoning', + start: channel.start, + end: channel.end, + })) + downloadOptions = { + headers, + sha256: settings.sha256 ?? fromManifest.sha256, + } + return + } + + let response: Response + try { + response = await fetch(treeUrl(ref), { headers, signal }) + } catch (error) { + if (signal?.aborted) throw createAbortError() + if (pickKeptFile(ref)) return + throw new LiteRTError( + 'DOWNLOAD_FAILED', + `Could not list the files of ${ref.repo}: ${errorMessage(error)}`, + error + ) + } + if ( + (response.status === 429 || response.status >= 500) && + pickKeptFile(ref) + ) + return + if (!response.ok) { + throw new LiteRTError( + response.status === 404 || response.status === 401 + ? 'MODEL_NOT_FOUND' + : 'DOWNLOAD_FAILED', + `Could not list the files of ${ref.repo}: HTTP ${response.status}. Check the repo name; a gated or private repo needs accessToken.` + ) + } + let files: RepoFile[] + try { + files = parseTree(await response.json()) + } catch (error) { + if (signal?.aborted) throw createAbortError() + // A page in place of the list, from a network that wants a sign-in, is no answer either. + if (pickKeptFile(ref)) return + throw new LiteRTError( + 'DOWNLOAD_FAILED', + `Could not read the list of files of ${ref.repo}: ${errorMessage(error)}`, + error + ) + } + const file = selectRepoFile( + files, + ref.repo, + settings.backend, + binding.getDeviceInfo().platform + ) + if (file === undefined) { + const names = files + .map((f) => f.path) + .filter((path) => path.endsWith('.litertlm')) + throw new LiteRTError( + 'MODEL_NOT_FOUND', + names.length === 0 + ? `${ref.repo} has no .litertlm file.` + : `${ref.repo} has no .litertlm file for this backend. Name one in the model id: hf://${ref.repo}/${names[0]}` + ) + } + modelPath = resolveUrl(ref, file.path) + downloadOptions = { headers, sha256: settings.sha256 ?? file.sha256 } + } + + const buildConfig = (request: GenerationRequest): LLMConfig => { + const config: LLMConfig = { + ...manifestConfig, + ...(settings.backend !== undefined ? { backend: settings.backend } : {}), + ...(settings.maxContextTokens !== undefined + ? { maxContextTokens: settings.maxContextTokens } + : {}), + ...(settings.speculativeDecoding !== undefined + ? { enableSpeculativeDecoding: settings.speculativeDecoding } + : {}), + } + const temperature = request.temperature ?? settings.temperature + const topK = request.topK ?? settings.topK + const topP = request.topP ?? settings.topP + if (temperature !== undefined) config.temperature = temperature + if (topK !== undefined) config.topK = topK + if (topP !== undefined) config.topP = topP + if (request.tools.length > 0) config.tools = toNativeTools(request.tools) + if (usesConstraint(request)) config.enableStructuredOutput = true + return config + } + + /** The configuration a request needs, once any setting that can't change per call is known. */ + const configKey = (config: LLMConfig) => canonicalJson(config) + + const release = async (): Promise => { + const llm = loaded?.llm + loaded = undefined + conversation = undefined + if (llm !== undefined) { + await llm.unload().catch(() => {}) + llm.close() + } + } + + /** The system can take a loaded engine's memory back, and then the model has to load again. */ + const forgetIfFreed = async (): Promise => { + if (loaded !== undefined && !loaded.llm.isReady()) await release() + } + + const load = async ( + config: LLMConfig, + key: string, + options?: LiteRTPrepareOptions + ): Promise => { + const onProgress = options?.onProgress ?? settings.onProgress + const signal = options?.abortSignal + // A new instance, with the old engine gone first: an instance keeps sampling values that + // a new config leaves unset, and the binding's memory check counts a loaded engine as used. + await release() + const llm = binding.createLLM() + let abandoned = false + const finished = llm + .loadModel(modelPath as string, config, { + ...downloadOptions, + onProgress: onProgress + ? (fraction) => { + if (!abandoned) onProgress({ fraction }) + } + : undefined, + }) + .then( + () => { + if (abandoned) return + loaded = { llm, configKey: key } + conversation = undefined + }, + (error: unknown) => { + if (abandoned) return + llm.close() + throw toLiteRTError(error, 'load') + } + ) + try { + await untilAborted(finished, signal) + } catch (error) { + if (signal?.aborted) { + // The binding can't stop a download. With the instance closed no engine loads into it, + // and the download goes on for the next call to join. + abandoned = true + llm.close() + } + throw error + } + } + + const describeReload = (request: GenerationRequest): string => { + const reasons: string[] = [] + if (request.tools.length > 0) reasons.push('tools') + if ( + request.temperature !== undefined || + request.topK !== undefined || + request.topP !== undefined + ) { + reasons.push('sampling') + } + if (usesConstraint(request)) reasons.push('structured output') + const what = reasons.length > 0 ? reasons.join(' and ') : 'its settings' + return `The model reloads for this call because ${what} changed: these apply when the model loads. Use the same values on every call to avoid the reload.` + } + + return { + warningsFor(request) { + const warnings: SharedV4Warning[] = [] + if (request.seed !== undefined) { + warnings.push({ type: 'unsupported', feature: 'seed' }) + } + if ( + loaded !== undefined && + configKey(buildConfig(request)) !== loaded.configKey + ) { + warnings.push({ type: 'other', message: describeReload(request) }) + } + if (request.jsonSchema !== undefined && !usesConstraint(request)) { + warnings.push({ + type: 'compatibility', + feature: 'structured output', + details: + request.jsonSchema === '' + ? 'The engine constrains output only to a schema, so the request for JSON goes to the model as an instruction and the output is not guaranteed to be JSON.' + : 'A model constrained to a schema can not call a tool, so with tools the schema goes to the model as an instruction and the output is not guaranteed to match.', + }) + } + return warnings + }, + + async prepare(options) { + const signal = options?.abortSignal + if (signal?.aborted) throw createAbortError() + await resolveSource(signal) + await forgetIfFreed() + if (loaded !== undefined) return + const config = buildConfig({ + conversation: { + system: undefined, + history: [], + final: { role: 'user', text: '' }, + }, + tools: [], + }) + await load(config, configKey(config), options) + }, + + async generate(request, onChunk, abortSignal) { + if (abortSignal?.aborted) throw createAbortError() + await resolveSource(abortSignal) + await forgetIfFreed() + + const config = buildConfig(request) + const key = configKey(config) + const wasLoaded = loaded !== undefined + let reloaded = false + if (loaded?.configKey !== key) { + reloaded = wasLoaded + await load(config, key, { abortSignal }) + } + if (abortSignal?.aborted) throw createAbortError() + const llm = (loaded as LoadedModel).llm + + const { history, final } = request.conversation + const system = + request.jsonSchema !== undefined && !usesConstraint(request) + ? withJsonInstruction(request.conversation.system, request.jsonSchema) + : request.conversation.system + const historyKey = transcriptKey(system, history) + const continued = conversation?.key === historyKey + // A failure or a cancel below leaves the engine's conversation unknown. + conversation = undefined + if (!continued) { + const messages = history.flatMap(historyMessage) + try { + llm.resetConversation( + messages.length > 0 ? JSON.stringify(messages) : undefined, + system + ) + } catch (error) { + throw toLiteRTError(error, 'generate') + } + } + + const parts: MultimodalPart[] = + final.role === 'user' + ? [{ type: 'text', text: final.text }] + : final.results.map((result) => + binding.toolResultPart(result.name, result.value) + ) + + const options: ExecuteOptions = {} + if (request.maxOutputTokens !== undefined) + options.maxOutputTokens = request.maxOutputTokens + if (request.frequencyPenalty !== undefined) + options.frequencyPenalty = request.frequencyPenalty + if (request.presencePenalty !== undefined) + options.presencePenalty = request.presencePenalty + if (usesConstraint(request)) options.responseSchema = request.jsonSchema + // Off unless the call asks: the binding never hands thoughts to JS, so a model that + // thinks by default would spend the output budget on text nobody can read. + options.thinking = { enabled: request.thinking ?? false } + + let text = '' + let rawOutput = '' + let tokens = 0 + const toolCalls: Extract[] = [] + const parser = new OutputParser( + // Without tools, a marker the model writes is only text. + request.tools.length > 0 + ? channels + : channels.filter((c) => c.type !== 'tool-call'), + (chunk) => { + if (chunk.type === 'text') text += chunk.text + if (chunk.type === 'tool-call') toolCalls.push(chunk) + onChunk(chunk) + } + ) + + let aborted = false + const onAbort = () => { + aborted = true + // Resetting is how the binding cancels a message in progress. + try { + llm.resetConversation() + } catch { + // Nothing to cancel. + } + } + abortSignal?.addEventListener('abort', onAbort) + + try { + await llm.execute( + parts, + (token, done) => { + if (aborted) return + if (done && ERROR_TOKEN.test(token)) return + if (token.length > 0) { + tokens += 1 + rawOutput += token + parser.push(token) + } + if (done) parser.finish() + }, + options + ) + } catch (error) { + if (aborted) throw createAbortError() + if (isCancellation(error)) { + // Nothing here cancelled it, so the system took the engine's memory back. + await release() + throw new LiteRTError( + 'OUT_OF_MEMORY', + 'The system freed the engine while it was generating. The next call loads the model again.', + error + ) + } + throw toLiteRTError(error, 'generate') + } finally { + abortSignal?.removeEventListener('abort', onAbort) + } + if (aborted) throw createAbortError() + + // The engine now holds the history, the final turn and this answer. + conversation = { + system, + key: transcriptKey(system, [ + ...history, + final, + { + role: 'assistant', + text, + toolCalls: toolCalls.map((call) => ({ + name: call.name, + arguments: safeParse(call.argumentsJson), + })), + }, + ]), + } + + const usage = readUsage(llm, rawOutput, tokens) + const limit = + request.maxOutputTokens ?? + config.maxOutputTokens ?? + DEFAULT_MAX_OUTPUT_TOKENS + return { + continued, + reloaded, + ...usage, + // Thoughts never reach JavaScript, so no text at all means they used up the limit. + cutOff: + Math.max(tokens, usage.outputTokens ?? 0) >= limit || + (request.thinking === true && tokens === 0), + } + }, + + unload: release, + } +} + +function safeParse(json: string): unknown { + try { + return JSON.parse(json) as unknown + } catch { + return json + } +} + +function readUsage( + llm: LiteRTLMInstance, + rawOutput: string, + tokens: number +): Pick< + GenerationResult, + 'inputTokens' | 'outputTokens' | 'tokensPerSecond' | 'timeToFirstTokenMs' +> { + // What the engine streamed, one piece a token. Android's stats estimate from characters. + const usage: ReturnType = { outputTokens: tokens } + try { + const stats = llm.getStats() + usage.tokensPerSecond = stats.tokensPerSecond + usage.timeToFirstTokenMs = stats.timeToFirstToken + usage.inputTokens = Math.round(stats.promptTokens) + // The bundle's tokenizer is better still, where the binding can use it. + const counted = llm.countTokens(rawOutput) + const held = llm.getContextTokenCount() + if (counted >= 0) { + usage.outputTokens = counted + if (held >= counted) usage.inputTokens = held - counted + } else { + // Android's rate is characters over the whole call, the wait for the first token included. + const generatingMs = stats.totalTime - stats.timeToFirstToken + usage.tokensPerSecond = + tokens > 1 && generatingMs > 0 + ? ((tokens - 1) * 1000) / generatingMs + : undefined + } + } catch { + // The rest stays unknown. + } + return usage +} diff --git a/packages/litert/src/types.ts b/packages/litert/src/types.ts new file mode 100644 index 00000000..55d1fee8 --- /dev/null +++ b/packages/litert/src/types.ts @@ -0,0 +1,69 @@ +import type { LanguageModelV4, ProviderV4 } from '@ai-sdk/provider' + +/** Where a model runs. `npu` needs a bundle compiled for the phone's chip. */ +export type LiteRTBackendName = 'gpu' | 'cpu' | 'npu' + +export interface LiteRTProgress { + /** Download progress from 0 to 1. Undefined when the total size is unknown. */ + fraction?: number +} + +export interface LiteRTPrepareOptions { + onProgress?: (progress: LiteRTProgress) => void + abortSignal?: AbortSignal +} + +/** Settings for one model. They apply when the model loads. */ +export interface LiteRTModelSettings { + /** + * Compute backend. Defaults to the bundle's preference, then the CPU. + */ + backend?: LiteRTBackendName + /** Tokens the conversation can hold: prompt, history and output together. */ + maxContextTokens?: number + /** Sampling defaults, used when a call does not set its own. */ + temperature?: number + topK?: number + topP?: number + /** Use the bundle's drafter for speculative decoding, if it has one. */ + speculativeDecoding?: boolean + /** Hugging Face token of the user loading a gated or private model. */ + accessToken?: string + /** The file's SHA-256 as lowercase hex, checked after the download. */ + sha256?: string + /** Called while the model downloads, when it loads on the first call instead of through `prepare()`. */ + onProgress?: (progress: LiteRTProgress) => void +} + +/** A language model that can also be downloaded, loaded and released ahead of a call. */ +export interface LiteRTLanguageModel extends LanguageModelV4 { + /** Download and load the model now, so the first call does not wait for it. */ + prepare(options?: LiteRTPrepareOptions): Promise + /** Free the engine's memory. The next call or `prepare()` loads the model again. */ + unload(): Promise +} + +export interface LiteRTProvider extends ProviderV4 { + (modelId: string, settings?: LiteRTModelSettings): LiteRTLanguageModel + languageModel( + modelId: string, + settings?: LiteRTModelSettings + ): LiteRTLanguageModel + /** Free every model this provider has loaded. */ + unloadAll(): Promise +} + +/** Key of the provider's entry in `providerMetadata`. */ +export const PROVIDER_NAME = 'litert' + +/** What each call reports in `providerMetadata.litert`. */ +export interface LiteRTCallMetadata { + /** The call continued the conversation already in the engine, so nothing was replayed. */ + continued: boolean + /** The call reloaded the model because its sampling or tools changed. */ + reloaded: boolean + /** Generated tokens per second, when the engine reports it. */ + tokensPerSecond?: number + /** Milliseconds to the first generated token, when the engine reports it. */ + timeToFirstTokenMs?: number +} diff --git a/packages/litert/tsconfig.build.json b/packages/litert/tsconfig.build.json new file mode 100644 index 00000000..acbe349f --- /dev/null +++ b/packages/litert/tsconfig.build.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig", + "include": ["src/**/*"], + "exclude": ["**/__tests__/**"] +} diff --git a/packages/litert/tsconfig.json b/packages/litert/tsconfig.json new file mode 100644 index 00000000..54f5777b --- /dev/null +++ b/packages/litert/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.base.json", + "include": ["src/**/*"], + "exclude": ["src/**/__tests__/**"] +} diff --git a/website/astro.config.mjs b/website/astro.config.mjs index 7f661155..c571e5a3 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -122,6 +122,10 @@ export default defineConfig({ ['speech', 'Speech'], ['model-management', 'Model management'], ]), + group('LiteRT-LM', 'litert', [ + ['getting-started', 'Getting started'], + ['generating', 'Generating'], + ]), group('JSON UI', 'json-ui', [ ['getting-started', 'Getting started'], ['tools', 'Tools'], diff --git a/website/src/content/docs/docs/index.md b/website/src/content/docs/docs/index.md index e9c600c3..0f1fb2f1 100644 --- a/website/src/content/docs/docs/index.md +++ b/website/src/content/docs/docs/index.md @@ -30,10 +30,11 @@ const result = streamText({ | [Core AI](/docs/core-ai/getting-started) | iOS 27 | Yours: language, image, speech and vision models | Native, provided by the platform | | [Android](/docs/android/getting-started) | Android | Built in: Gemini Nano, speech recognition, embeddings | Native, provided by the platform | | [Llama](/docs/llama/getting-started) | iOS and Android | Yours: GGUF models from Hugging Face | Third-party binding, llama.rn | +| [LiteRT-LM](/docs/litert/getting-started) | iOS and Android | Yours: `.litertlm` models from Hugging Face | Third-party binding, react-native-litert-lm | Built-in models need no download and work on supported devices right away. -With Core AI and Llama, your app downloads or bundles the models, and we point -you to ready-to-use ones. +With Core AI, Llama and LiteRT-LM, your app downloads or bundles the models, and +we point you to ready-to-use ones. Every provider needs the AI SDK's [polyfills](/docs/polyfills). diff --git a/website/src/content/docs/docs/litert/generating.md b/website/src/content/docs/docs/litert/generating.md new file mode 100644 index 00000000..71e807fa --- /dev/null +++ b/website/src/content/docs/docs/litert/generating.md @@ -0,0 +1,153 @@ +--- +title: "Generating" +--- + +Generate text with a `.litertlm` model: pass a LiteRT language model to the AI +SDK's `generateText` or `streamText`. The examples start from a prepared model, +as in [Getting Started](/docs/litert/getting-started#usage): + +```typescript +import { litert } from '@react-native-ai/litert' + +const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm') +await model.prepare() +``` + +## Text Generation + +```typescript +import { generateText } from 'ai' + +const result = await generateText({ + model, + prompt: 'Explain quantum computing in simple terms', +}) + +console.log(result.text) +``` + +## Streaming + +```typescript +import { streamText } from 'ai' + +const { textStream } = streamText({ + model, + prompt: 'Write a short story about a robot learning to paint', +}) + +for await (const delta of textStream) { + console.log(delta) +} +``` + +## Options + +Sampling and tools apply when a LiteRT-LM model loads. The provider honors them +by reloading the model when they change, and the call carries a warning, so keep +them the same from call to call. + +| Option | Applies | +| ------------------------------------- | ------------------------------------ | +| `maxOutputTokens` | Per call, 1024 if not set | +| `temperature`, `topK`, `topP` | Reload when changed | +| `tools` | Reload when changed | +| `output: Output.object(...)` | Reload when turned on or off | +| `reasoning` | Per call, off by default | +| `frequencyPenalty`, `presencePenalty` | Per call | +| `seed`, `stopSequences` | Not supported, a warning | + +To set sampling once, pass it with the model, as in +[Getting Started](/docs/litert/getting-started#options). That doesn't reload. + +A reply that stops at `maxOutputTokens` reports `finishReason: 'length'`. + +Images and audio in the prompt aren't supported yet: a file part fails with a +clear error. + +## Tool Calling + +Pass tools as usual. The AI SDK runs them and sends the results back to the +model: + +```typescript +import { generateText, stepCountIs, tool } from 'ai' +import { z } from 'zod' + +const result = await generateText({ + model, + prompt: 'What is the weather in Tokyo?', + tools: { + getWeather: tool({ + description: 'Get current weather information', + inputSchema: z.object({ city: z.string() }), + execute: async ({ city }) => `Weather in ${city}: Sunny, 25°C`, + }), + }, + stopWhen: stepCountIs(3), +}) +``` + +A tool call that doesn't parse, or that fails its schema, arrives as a +`tool-error` part, so the model can try again on the next step. The engine can't +force a call: `toolChoice: 'required'` gives a warning and the model decides. + +## Structured Output + +Generate JSON that matches a schema with `Output.object`. The engine constrains +the output to the schema: + +```typescript +import { generateText, Output } from 'ai' +import { z } from 'zod' + +const { output } = await generateText({ + model, + output: Output.object({ + schema: z.object({ + capital: z.string(), + country: z.string(), + }), + }), + prompt: 'Name the capital of Japan and its country.', +}) +``` + +A model held to a schema can't call a tool. So a call that also has tools sends the +schema to the model as an instruction instead. `Output.json()` has no schema, so +it's an instruction too. An instruction comes with a warning on the call, and a reply that +doesn't match fails the way it does with any provider. + +## Reasoning + +react-native-litert-lm doesn't hand a model's thoughts to JavaScript, so the +provider turns thinking off unless a call sets `reasoning`. Otherwise a model that +thinks by default would spend `maxOutputTokens` on text nobody can read. + +With `reasoning: 'high'`, the model thinks, `reasoningText` stays empty, and the +thoughts count against `maxOutputTokens`, so leave room: on a Pixel 11 Pro, +Qwen3-0.6B returned empty text for 17 × 23 with 512 tokens. A call that ends with +no text this way reports `finishReason: 'length'`. + +`` tags that a bundle leaves at the start of its text become reasoning parts. +After the answer has begun, a tag is text. + +## Conversations + +Replaying a whole conversation for every message gets slow as the chat grows. The +provider keeps the conversation in the engine, and when a call's messages extend +what the engine already holds, it sends only the new message. Send the messages +back as `response.messages` gave them: + +```typescript +import { generateText, type ModelMessage } from 'ai' + +const messages: ModelMessage[] = [{ role: 'user', content: 'Hello' }] +const first = await generateText({ model, messages }) + +messages.push(...first.response.messages, { role: 'user', content: 'And then?' }) +const second = await generateText({ model, messages }) +``` + +`providerMetadata.litert.continued` tells you whether a call did this. A different +history replays, which is always correct and only slower. diff --git a/website/src/content/docs/docs/litert/getting-started.mdx b/website/src/content/docs/docs/litert/getting-started.mdx new file mode 100644 index 00000000..4c1f6a65 --- /dev/null +++ b/website/src/content/docs/docs/litert/getting-started.mdx @@ -0,0 +1,171 @@ +--- +title: "Getting Started" +--- + +import PackageManagerTabs from '../../../../components/starlight/PackageManagerTabs.astro' + +Run `.litertlm` models from Hugging Face on iOS and Android, on the device. The +provider is built on +[react-native-litert-lm](https://github.com/hung-yueh/react-native-litert-lm), a +third-party React Native binding for Google's +[LiteRT-LM](https://ai.google.dev/edge/litert-lm). + +## Installation + + + +The provider works with AI SDK v7 and needs its [polyfills](/docs/polyfills). It +requires react-native-litert-lm 0.8.2 or newer. + +### Expo + +Add the config plugin to your app config: + +```json title="app.json" +{ + "expo": { + "plugins": [ + [ + "react-native-litert-lm", + { "ios": { "extendedVirtualAddressing": true, "increasedMemoryLimit": true } } + ] + ] + } +} +``` + +The two iOS settings let an app load models of about 2 GB and more, and they need +a paid Apple Developer account. For a smaller model, +`"plugins": ["react-native-litert-lm"]` is enough. + +Expo Go can't load native modules, so build a native app. + +## Usage + +Create a model, download and load it, and generate: + +```typescript +import { litert } from '@react-native-ai/litert' +import { streamText } from 'ai' + +const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm') + +await model.prepare({ + onProgress: ({ fraction }) => console.log(fraction), +}) + +const { textStream } = streamText({ + model, + prompt: 'Explain quantum computing in simple terms', +}) + +for await (const delta of textStream) { + console.log(delta) +} + +await model.unload() +``` + +- **`prepare()`** downloads the model once and loads it into memory. Without it, + the first request does both. +- **`unload()`** frees the memory when you are done. The next request or + `prepare()` loads the model again. + +One provider keeps one model in memory, so loading a second model unloads the +first. Calls run one at a time, in order. + +A dropped connection doesn't lose the download if the server takes range requests, +as Hugging Face does: the call fails with `DOWNLOAD_FAILED`, and +react-native-litert-lm goes on from there on the next call. + +A downloaded model loads with no network. A phone keeps files by name, not by +repo, so when Hugging Face doesn't answer, the provider looks for a file whose name +is the repo's name, alone or with a build suffix such as `-gpu` or `_mixed_int4`. +Any other name may be another model, and the call fails with `DOWNLOAD_FAILED`. +For those, name the file in the model ID. + +A phone keeps one file per name. With a Hugging Face ID the provider passes the +file's hash, so another revision or another repo's file of that name replaces it. +With a plain URL, pass `sha256` to get the same. + +## Models + +| Model ID | What loads | +| -------------------------------------- | --------------------------------- | +| `hf://org/repo` or `org/repo` | A file chosen from the repo | +| `hf://org/repo@revision/file.litertlm` | That file at that revision | +| `https://…/model.litertlm` | That URL | +| `/path/to/model.litertlm` | A file on the phone | + +Some repos hold a GPU-only `-gpu.litertlm` file next to a full bundle. Given a +repo, the provider takes the GPU-only file on iOS and the full bundle on Android. +On a Pixel 11 Pro, the GPU-only file of Gemma 4 E2B gave wrong text, with no error, +with react-native-litert-lm 0.8.2 (LiteRT-LM 0.18.1). + +To load another file, name it in the model ID. A repo's +`litertlm_manifest.json` takes precedence, files built for one chip are never +chosen, and `backend: 'cpu'` skips the GPU-only file. + +If a model answers as though it never saw the question, look at the file first. +The chat template in some older files loses the user's turn on LiteRT-LM 0.18 and +newer, with no error. `qwen3_0_6b_mixed_int4.litertlm` did this on a Pixel 11 Pro, +and the [Qwen3-8B](https://huggingface.co/litert-community/Qwen3-8B) card reports +it for all of its files. + +## Options + +You can pass settings with the model. They apply when it loads: + +```typescript +const model = litert('hf://litert-community/gemma-4-E2B-it-litert-lm', { + temperature: 0.2, + maxContextTokens: 4096, + backend: 'gpu', +}) +``` + +| Option | Description | +| ---------------------- | ----------------------------------------------------------------------------- | +| `backend` | `gpu`, `cpu`, or `npu` with a bundle compiled for the phone's chip | +| `maxContextTokens` | Tokens the conversation can hold: prompt, history and output together | +| `temperature`, `topK`, `topP` | Sampling defaults, used when a call doesn't set its own | +| `speculativeDecoding` | Use the bundle's drafter for speculative decoding, if it has one | +| `accessToken` | Hugging Face token of the user loading a gated or private model | +| `sha256` | The file's SHA-256 as lowercase hex, checked after the download | +| `onProgress` | Called while the model downloads on the first call, instead of through `prepare()` | + +## Errors + +Failures from the engine are `LiteRTError`s with a `code`: + +```typescript +import { LiteRTError } from '@react-native-ai/litert' +import { generateText } from 'ai' + +try { + await generateText({ model, prompt: 'Hello' }) +} catch (error) { + if (LiteRTError.isInstance(error) && error.code === 'MODEL_NOT_FOUND') { + // ... + } +} +``` + +| Code | Meaning | +| ------------------- | ----------------------------------------------------------------- | +| `MODEL_NOT_FOUND` | No such file or repo, or a repo without a usable `.litertlm` file | +| `INVALID_MODEL` | The file is not a readable bundle | +| `DOWNLOAD_FAILED` | The download failed | +| `OUT_OF_MEMORY` | The engine ran out of memory, or refused to load because it would | +| `RUNTIME_TOO_OLD` | The bundle needs a newer LiteRT-LM | +| `LOAD_FAILED` | The engine could not start for another reason | +| `CONTEXT_TOO_LONG` | The prompt is longer than the conversation can hold | +| `GENERATION_FAILED` | Generation failed for another reason | + +`streamText` reports these through `onError` and the stream's `error` part. An +aborted call stops generation, and the model stays loaded for the next one. +react-native-litert-lm can't cancel a download: the call rejects at once, the +download goes on, and the next call picks it up. + +If Android asks for memory back, the engine is freed. A call in progress ends with +`OUT_OF_MEMORY`, and the next call loads the model again.