From a73cf80de0e356422b317733c9fdca14e5b8cf14 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 22:43:00 +0000 Subject: [PATCH 01/13] feat(litert): run Tensor-compiled LiteRT models on the Pixel TPU On a Pixel 10 every LiteRT load ended on the CPU: JS rewrote any NPU request to GPU, and native skips the GPU on Pixel 10 (open LiteRT crash). The Tensor G5 TPU, reachable through LiteRT-LM's NPU backend, was never used. - Manifest: declare the libedgetpu_* vendor libraries LiteRT's Tensor dispatch library loads; undeclared, the linker namespace refuses them. - LiteRTModule.getTpuSupport: Tensor generation (SOC_MODEL, then SoC/device codenames), Android 16+, and the dispatch library present in the APK. Pure detection lives in TensorTpu.kt with JVM unit tests. - NPU tier: vision runs on the NPU with the decoder, and the engine retries text-only on the NPU before falling to a slower tier (both as Google's Pixel 10 TPU reference app does). loadModel reports the media it came up with; JS adopts it and refuses images a text-only engine would drop. - litert.ts: a file compiled for this phone's Tensor generation (..._Google_Tensor_G5.litertlm) loads on the TPU. Any other NPU request still loads on GPU, as before. - Catalog: Gemma 4 E2B Tensor G5 build, listed only on a Tensor G5 phone (Models tab and onboarding). Not yet pinned to a Hugging Face commit or exact size; tracks main. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- __tests__/harness/nativeBoundary.ts | 2 + ...ensorTpuLiteRTOnboarding.rendered.test.tsx | 43 +++++++++++++++++ .../liteRTReleaseBackends.rendered.test.tsx | 38 +++++++++++++++ .../curatedLiteRTContextLimit.test.ts | 13 ++++- .../services/curatedLiteRTTensorTpu.test.ts | 36 ++++++++++++++ .../generationServiceHelpers.branches.test.ts | 29 ++++++++++++ android/app/src/main/AndroidManifest.xml | 7 +++ .../ai/offgridmobile/litert/LiteRTModule.kt | 47 ++++++++++++++++--- .../java/ai/offgridmobile/litert/TensorTpu.kt | 38 +++++++++++++++ .../ai/offgridmobile/litert/TensorTpuTest.kt | 41 ++++++++++++++++ src/screens/ModelsScreen/TextModelsTab.tsx | 27 +++++++++-- src/services/curatedLiteRTRegistry.ts | 27 +++++++++++ src/services/generationServiceHelpers.ts | 7 +++ src/services/hardware.ts | 24 ++++++++++ src/services/litert.ts | 36 ++++++++++++-- src/utils/modelHelpers.ts | 10 ++++ 16 files changed, 407 insertions(+), 18 deletions(-) create mode 100644 __tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx create mode 100644 __tests__/unit/services/curatedLiteRTTensorTpu.test.ts create mode 100644 android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt create mode 100644 android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt diff --git a/__tests__/harness/nativeBoundary.ts b/__tests__/harness/nativeBoundary.ts index fe03c10f0..50ff2bbe1 100644 --- a/__tests__/harness/nativeBoundary.ts +++ b/__tests__/harness/nativeBoundary.ts @@ -273,6 +273,8 @@ function makeLiteRTFake(handle: FakeEmitterHandle): LiteRTFake { }), stopGeneration: jest.fn().mockResolvedValue(undefined), unloadModel: jest.fn().mockResolvedValue(undefined), + // Not a Google Tensor phone unless a test says otherwise (e.g. a Pixel 10: supported, generation 5). + getTpuSupport: jest.fn().mockResolvedValue({ supported: false, tensorGeneration: null, reason: 'not_tensor' }), getMemoryInfo: jest.fn().mockResolvedValue({ totalRamMb: 12000, usedRamMb: 4000, diff --git a/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx b/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx new file mode 100644 index 000000000..a104f7af3 --- /dev/null +++ b/__tests__/integration/models/tensorTpuLiteRTOnboarding.rendered.test.tsx @@ -0,0 +1,43 @@ +/** + * Pixel 10 (Tensor G5): its TPU runs only LiteRT builds compiled for it, and its GPU has no working + * LiteRT path, so onboarding offers the Tensor G5 build there — and nowhere else, since no other phone + * can run it. + * + * Real AdvancedSetupScreen + real hardwareService + curated registry; fakes ONLY the native boundary + * (the LiteRT module's TPU probe answers like a Pixel 10, or like any other phone). + */ +import { installNativeBoundary, requireRTL, GB } from '../../harness/nativeBoundary'; + +jest.mock('@react-navigation/native', () => ({ + useNavigation: () => ({ navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {}, replace: () => {} }), + useRoute: () => ({ params: {} }), + useFocusEffect: () => {}, useIsFocused: () => true, +})); + +async function renderOnboarding(tpu: { supported: boolean; tensorGeneration: number | null; reason: string | null }) { + const boundary = installNativeBoundary({ ram: { platform: 'android', totalBytes: 12 * GB, availBytes: 8 * GB } }); + boundary.litert.module.getTpuSupport.mockResolvedValue(tpu); + const React = require('react'); + const rtl = requireRTL(); + const { hardwareService } = require('../../../src/services/hardware'); + const { AdvancedSetupScreen } = require('../../../src/screens/ModelDownloadScreen'); + await hardwareService.getDeviceInfo(); + const nav: any = { navigate: () => {}, goBack: () => {}, setOptions: () => {}, addListener: () => () => {}, replace: () => {} }; + const view = rtl.render(React.createElement(AdvancedSetupScreen, { navigation: nav })); + await rtl.waitFor(() => { expect(view.getByTestId('onboarding-litert-model-1')).toBeTruthy(); }, { timeout: 10000 }); + return { rtl, view }; +} + +describe('Tensor TPU LiteRT build in onboarding', () => { + it('offers the Tensor G5 build on a Pixel 10 whose TPU LiteRT can reach', async () => { + const { rtl, view } = await renderOnboarding({ supported: true, tensorGeneration: 5, reason: null }); + const card = await rtl.waitFor(() => view.getByTestId('onboarding-litert-model-2'), { timeout: 10000 }); + expect(rtl.within(card).getByText('Gemma 4 E2B (Tensor TPU)')).toBeTruthy(); + }, 60000); + + it('does not offer it on a phone without a Tensor G5 TPU', async () => { + const { view } = await renderOnboarding({ supported: false, tensorGeneration: null, reason: 'not_tensor' }); + expect(view.queryByText('Gemma 4 E2B (Tensor TPU)')).toBeNull(); + expect(view.queryByTestId('onboarding-litert-model-2')).toBeNull(); + }, 60000); +}); diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index 84a19af5b..7be91d621 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -10,6 +10,44 @@ it('uses GPU at the native boundary when an older build saved NPU', async () => ]); }); +const TENSOR_G5_FILE = '/models/gemma-4-E2B-it_Google_Tensor_G5.litertlm'; +const PIXEL_10_TPU = { supported: true, tensorGeneration: 5, reason: null }; + +it('loads a Tensor G5 build on the TPU of a phone with a Tensor G5 TPU, whatever the saved backend', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu'); + expect(boundary.litert.calls.loadModel).toEqual([[TENSOR_G5_FILE, 'npu', false, false, 4096]]); +}); + +it('keeps a portable LiteRT file off the NPU even on a phone with a Tensor TPU', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel('/models/gemma-4-E2B-it.litertlm', 'npu'); + expect(boundary.litert.calls.loadModel).toEqual([['/models/gemma-4-E2B-it.litertlm', 'gpu', false, false, 4096]]); +}); + +it('does not send a Tensor G5 build to the NPU when the phone has no usable Tensor G5 TPU', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue({ supported: false, tensorGeneration: 5, reason: 'dispatch_lib_missing' }); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'cpu'); + expect(boundary.litert.calls.loadModel).toEqual([[TENSOR_G5_FILE, 'cpu', false, false, 4096]]); +}); + +it('adopts the media the native engine came up with when a TPU load drops to text-only', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false }); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true, supportsAudio: true }); + expect(liteRTService.getActiveBackend()).toBe('npu'); + expect(liteRTService.supportsVision()).toBe(false); + expect(liteRTService.supportsAudio()).toBe(false); +}); + it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { installNativeBoundary(); const React = require('react'); diff --git a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts index 80f09a415..e031b8bbf 100644 --- a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts +++ b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts @@ -1,7 +1,9 @@ import { CURATED_LITERT_ENTRIES, getCuratedLiteRTContextLimit, LITERT_PARENT_ID } from '../../../src/services/curatedLiteRTRegistry'; import { createDownloadedModel } from '../../utils/factories'; -it.each(CURATED_LITERT_ENTRIES)('matches the pinned artifact for $displayName without changing saved model data', entry => { +const ENTRIES_WITH_LIMIT = CURATED_LITERT_ENTRIES.filter(e => e.maxContextTokens !== undefined); + +it.each(ENTRIES_WITH_LIMIT)('matches the pinned artifact for $displayName without changing saved model data', entry => { const model = createDownloadedModel({ engine: 'litert', id: `${LITERT_PARENT_ID}/${entry.fileName}`, fileName: entry.fileName, fileSize: entry.sizeBytes, @@ -14,3 +16,12 @@ it.each(CURATED_LITERT_ENTRIES)('matches the pinned artifact for $displayName wi expect(getCuratedLiteRTContextLimit({ ...model, origin: { ...origin, revision: 'main' } })).toBeNull(); expect(getCuratedLiteRTContextLimit({ ...model, engine: 'llama' })).toBeNull(); }); + +it('imposes no catalog context limit on an entry that declares none (the Tensor TPU build)', () => { + const entry = CURATED_LITERT_ENTRIES.find(e => e.maxContextTokens === undefined)!; + const model = createDownloadedModel({ + engine: 'litert', id: `${LITERT_PARENT_ID}/${entry.fileName}`, + fileName: entry.fileName, fileSize: entry.sizeBytes, + }); + expect(getCuratedLiteRTContextLimit(model)).toBeNull(); +}); diff --git a/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts b/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts new file mode 100644 index 000000000..e0cbadc68 --- /dev/null +++ b/__tests__/unit/services/curatedLiteRTTensorTpu.test.ts @@ -0,0 +1,36 @@ +/** + * A LiteRT file compiled for a Google Tensor TPU runs only on that generation's TPU, so the + * curated list shows it only there (Models tab + onboarding share liteRTFileRunsOnDevice). + * The target generation comes from Google's file naming, read by liteRTTensorTarget. + */ +import { CURATED_LITERT_ENTRIES, liteRTFileRunsOnDevice } from '../../../src/services/curatedLiteRTRegistry'; +import { liteRTTensorTarget } from '../../../src/utils/modelHelpers'; + +const tensorEntry = CURATED_LITERT_ENTRIES.find(e => liteRTTensorTarget(e.fileName) !== null); + +describe('Tensor TPU LiteRT builds', () => { + it('reads the Tensor generation from Google’s file naming', () => { + expect(liteRTTensorTarget('gemma-4-E2B-it_Google_Tensor_G5.litertlm')).toBe(5); + expect(liteRTTensorTarget('Gemma3-1B-IT_google_tensor_g4.litertlm')).toBe(4); + expect(liteRTTensorTarget('gemma-4-E2B-it.litertlm')).toBeNull(); + expect(liteRTTensorTarget('gemma-4-E2B-it_Google_Tensor_G5.gguf')).toBeNull(); + }); + + it('curates a Tensor G5 build (fixture ground-truth)', () => { + expect(tensorEntry?.fileName).toBe('gemma-4-E2B-it_Google_Tensor_G5.litertlm'); + }); + + it('lists the Tensor G5 build only on a phone whose TPU is Tensor G5', () => { + const name = tensorEntry!.fileName; + expect(liteRTFileRunsOnDevice(name, 5)).toBe(true); + expect(liteRTFileRunsOnDevice(name, 4)).toBe(false); + expect(liteRTFileRunsOnDevice(name, null)).toBe(false); + }); + + it('lists portable builds on every phone', () => { + for (const entry of CURATED_LITERT_ENTRIES.filter(e => e !== tensorEntry)) { + expect(liteRTFileRunsOnDevice(entry.fileName, null)).toBe(true); + expect(liteRTFileRunsOnDevice(entry.fileName, 5)).toBe(true); + } + }); +}); diff --git a/__tests__/unit/services/generationServiceHelpers.branches.test.ts b/__tests__/unit/services/generationServiceHelpers.branches.test.ts index bf9b50f0b..d6b6cc53d 100644 --- a/__tests__/unit/services/generationServiceHelpers.branches.test.ts +++ b/__tests__/unit/services/generationServiceHelpers.branches.test.ts @@ -42,6 +42,7 @@ jest.mock('../../../src/services/llm', () => ({ jest.mock('../../../src/services/litert', () => ({ liteRTService: { isModelLoaded: jest.fn(() => false), + supportsVision: jest.fn(() => true), getActiveBackend: jest.fn(() => 'cpu'), prepareConversation: jest.fn(() => Promise.resolve()), sendMessage: jest.fn(() => Promise.resolve()), @@ -192,6 +193,34 @@ describe('runLiteRTResponseImpl — image support guard', () => { expect(mockedLiteRT.sendMessage).not.toHaveBeenCalled(); }); + it('rejects an image when the loaded engine came up text-only (a TPU load without its vision executor)', async () => { + mockedGetState.mockReturnValue(liteRTAppState({ liteRTVision: true })); + mockedLiteRT.supportsVision.mockReturnValueOnce(false); + const store = chatStoreMock(); + const svc = makeServiceSvc(); + + await expect( + generateResponseImpl(svc, { + conversationId: 'conv-1', + messages: [ + { + id: '1', + timestamp: 0, + role: 'user' as const, + content: 'look', + attachments: [ + { id: 'i', type: 'image' as const, uri: 'file:///pic.png' }, + ], + }, + ], + }), + ).rejects.toThrow(/Images are not available for this model on this device/); + + expect(store.clearStreamingMessage).toHaveBeenCalled(); + expect(svc.resetState).toHaveBeenCalled(); + expect(mockedLiteRT.sendMessage).not.toHaveBeenCalled(); + }); + it('accepts an image attachment when the model has vision enabled', async () => { mockedGetState.mockReturnValue(liteRTAppState({ liteRTVision: true })); chatStoreMock(); diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 04fef1ba4..af76c58eb 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -76,6 +76,13 @@ + + + + + + "cpu" } - /** Vision delegate matched to the main backend tier (CPU main -> CPU vision). - * Also honors the Pixel 10 GPU gate: never hand the GPU vision delegate to a - * device where GPU init crashes, even on the NPU/GPU main path. */ - private fun visionBackendFor(mainBackend: Backend): Backend = - if (mainBackend is Backend.CPU || shouldSkipGpu()) Backend.CPU() else Backend.GPU() + /** Vision delegate matched to the main backend tier (CPU main -> CPU vision, NPU main -> + * NPU vision, as in Google's Tensor TPU reference app). Also honors the Pixel 10 GPU + * gate: never hand the GPU vision delegate to a device where GPU init crashes. */ + private fun visionBackendFor(mainBackend: Backend): Backend = when { + mainBackend is Backend.NPU -> Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir) + mainBackend is Backend.CPU || shouldSkipGpu() -> Backend.CPU() + else -> Backend.GPU() + } + + // ------------------------------------------------------------------------- + // getTpuSupport — can this phone run a LiteRT model on its Google Tensor TPU? + // ------------------------------------------------------------------------- + + @ReactMethod + fun getTpuSupport(promise: Promise) { + val safe = SafePromise(promise, TAG) + val socModel = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MODEL else "" + val generation = TensorTpu.generation(socModel, Build.HARDWARE ?: "", Build.BOARD ?: "", Build.DEVICE ?: "") + val dispatchLibPresent = File(reactContext.applicationInfo.nativeLibraryDir, TensorTpu.DISPATCH_LIB).exists() + val reason = TensorTpu.unsupportedReason(generation, Build.VERSION.SDK_INT, dispatchLibPresent) + Log.i(TAG, "getTpuSupport — soc=$socModel hardware=${Build.HARDWARE} tensorGeneration=$generation sdk=${Build.VERSION.SDK_INT} dispatchLib=$dispatchLibPresent reason=$reason") + safe.resolve(Arguments.createMap().apply { + putBoolean("supported", reason == null) + if (generation != null) putInt("tensorGeneration", generation) else putNull("tensorGeneration") + if (reason != null) putString("reason", reason) else putNull("reason") + }) + } /** Current free system RAM in MB. */ private fun availableRamMb(): Long { diff --git a/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt new file mode 100644 index 000000000..ea5ff37e1 --- /dev/null +++ b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt @@ -0,0 +1,38 @@ +package ai.offgridmobile.litert + +/** + * Can this phone run a LiteRT model on its Google Tensor TPU? Pure (no Android deps) so it is + * unit-testable; [LiteRTModule.getTpuSupport] feeds it the Build properties. + */ +internal object TensorTpu { + /** LiteRT's Google Tensor dispatch library, shipped in the app's native lib dir. */ + const val DISPATCH_LIB = "libLiteRtDispatch_GoogleTensor.so" + /** LiteRT supports the Tensor TPU from Android 16 (API 36). */ + const val MIN_SDK = 36 + + /** Pixel SoC and device codenames → Tensor generation, for builds whose SOC_MODEL doesn't name the chip. */ + private val CODENAMES = mapOf( + "gs101" to 1, "gs201" to 2, "zuma" to 3, "zumapro" to 4, "laguna" to 5, "malibu" to 6, + // Pixel 10, 10 Pro, 10 Pro XL, 10 Pro Fold + "frankel" to 5, "blazer" to 5, "mustang" to 5, "rango" to 5, + ) + private val SOC_MODEL = Regex("""tensor\s*g?(\d+)""") + + /** + * Tensor generation (5 for a Pixel 10's "Tensor G5"), or null when this is not a Tensor SoC. + * [socModel] is Build.SOC_MODEL; [codenames] are Build.HARDWARE / BOARD / DEVICE. + */ + fun generation(socModel: String, vararg codenames: String): Int? { + SOC_MODEL.find(socModel.lowercase())?.let { return it.groupValues[1].toIntOrNull() } + // Older Pixels report the SoC codename itself as SOC_MODEL ("GS101"). + return (listOf(socModel) + codenames).firstNotNullOfOrNull { CODENAMES[it.lowercase()] } + } + + /** Why the TPU can't be used here, or null when it can. */ + fun unsupportedReason(generation: Int?, sdkInt: Int, dispatchLibPresent: Boolean): String? = when { + generation == null -> "not_tensor" + sdkInt < MIN_SDK -> "android_too_old" + !dispatchLibPresent -> "dispatch_lib_missing" + else -> null + } +} diff --git a/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt new file mode 100644 index 000000000..9b35d86eb --- /dev/null +++ b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt @@ -0,0 +1,41 @@ +package ai.offgridmobile.litert + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +/** Tests for TensorTpu — whether a phone can run a LiteRT model on its Google Tensor TPU. */ +@Suppress("kotlin:S100") // Backtick test names are idiomatic Kotlin +class TensorTpuTest { + + @Test + fun `reads the generation from a Pixel SOC_MODEL`() { + assertEquals(5, TensorTpu.generation("Tensor G5", "blazer", "blazer", "blazer")) + assertEquals(4, TensorTpu.generation("Tensor G4", "caiman", "caiman", "caiman")) + } + + @Test + fun `falls back to the SoC or device codename when SOC_MODEL does not name the chip`() { + assertEquals(5, TensorTpu.generation("", "laguna")) + assertEquals(5, TensorTpu.generation("", "unknown", "unknown", "mustang")) + assertEquals(1, TensorTpu.generation("GS101", "oriole", "oriole", "oriole")) + } + + @Test + fun `is not a Tensor phone on Snapdragon or MediaTek`() { + assertNull(TensorTpu.generation("SM8750", "qcom", "sun", "pa3q")) + assertNull(TensorTpu.generation("MT6989", "mt6989", "k6989v1_64", "duchamp")) + } + + @Test + fun `allows the TPU on a Pixel 10 on Android 16 with the dispatch library`() { + assertNull(TensorTpu.unsupportedReason(5, 36, dispatchLibPresent = true)) + } + + @Test + fun `explains why the TPU cannot be used`() { + assertEquals("not_tensor", TensorTpu.unsupportedReason(null, 36, dispatchLibPresent = true)) + assertEquals("android_too_old", TensorTpu.unsupportedReason(5, 35, dispatchLibPresent = true)) + assertEquals("dispatch_lib_missing", TensorTpu.unsupportedReason(5, 36, dispatchLibPresent = false)) + } +} diff --git a/src/screens/ModelsScreen/TextModelsTab.tsx b/src/screens/ModelsScreen/TextModelsTab.tsx index 57bf41018..52c796f87 100644 --- a/src/screens/ModelsScreen/TextModelsTab.tsx +++ b/src/screens/ModelsScreen/TextModelsTab.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import { View, Text, FlatList, TextInput, RefreshControl, TouchableOpacity, Platform } from 'react-native'; import { LoadingDots } from '../../components/LoadingDots'; import DeviceInfo from 'react-native-device-info'; @@ -23,9 +23,10 @@ import { TextFiltersSection } from './TextFiltersSection'; import { FilterState, SortOption } from './types'; import { SORT_OPTIONS } from './constants'; import { formatNumber, getTextModelCompatibility } from './utils'; -import { buildCuratedLiteRTFiles, curatedLiteRTDownloadWarning, getCuratedLiteRTEntry, LITERT_PARENT_ID } from '../../services/curatedLiteRTRegistry'; +import { buildCuratedLiteRTFiles, curatedLiteRTDownloadWarning, getCuratedLiteRTEntry, liteRTFileRunsOnDevice, LITERT_PARENT_ID } from '../../services/curatedLiteRTRegistry'; import { LITERT_FILE_META, LITERT_RECOMMENDED_MODEL, LITERT_PARENT_RECOMMENDED } from './litertRecommended'; import { modelManager } from '../../services'; +import { hardwareService } from '../../services/hardware'; import { modelDownloadService } from '../../services/modelDownloadService'; import { uniformDownloadId } from '../../services/modelDownloadService/uniformId'; import { fetchModelFiles } from '../../services/modelCatalogFiles'; @@ -35,6 +36,18 @@ function hasNonSortFilters(fs: FilterState): boolean { return fs.orgs.length > 0 || fs.type !== 'all' || fs.source !== 'all' || fs.size !== 'all' || fs.quant !== 'all'; } +/** This phone's Google Tensor TPU generation, or null (not a Pixel, or before the probe answers). + * Gates the TPU-only LiteRT build in the curated list. */ +function useTensorTpuGeneration(): number | null { + const [generation, setGeneration] = useState(null); + useEffect(() => { + let live = true; + hardwareService.getTensorTpuGeneration().then(g => { if (live) setGeneration(g); }).catch(() => undefined); + return () => { live = false; }; + }, []); + return generation; +} + function getEmptyText(hasSearched: boolean, hasActiveFilters: boolean): string { if (!hasSearched) return 'No recommended models available.'; if (hasActiveFilters) return 'No models match your filters. Try adjusting or clearing them.'; @@ -104,6 +117,7 @@ const ModelDetailView: React.FC = ({ }) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); + const tpuGeneration = useTensorTpuGeneration(); // Pre-set the next pending (Download Manager icon) so it fires regardless of // how the user dismisses step 9 (button or backdrop tap). @@ -257,7 +271,9 @@ const ModelDetailView: React.FC = ({ {selectedModel.id === LITERT_PARENT_ID && Platform.OS === 'android' && DeviceInfo.getModel().toLowerCase().includes('pixel 10') && ( - {'GPU acceleration is not yet supported on Pixel 10. Models will run on CPU. Support coming soon.'} + {tpuGeneration === null + ? 'GPU acceleration is not yet supported on Pixel 10. Models will run on CPU. Support coming soon.' + : "Pixel 10 can't run LiteRT models on its GPU. The Tensor TPU build runs on the TPU; the other files run on the CPU."} )} Available Files @@ -272,7 +288,7 @@ const ModelDetailView: React.FC = ({ ) : ( f.size > 0 && !fileExceedsBudget(f.size, ramGB) && (filterState.quant === 'all' || f.name.includes(filterState.quant))) + .filter(f => f.size > 0 && !fileExceedsBudget(f.size, ramGB) && liteRTFileRunsOnDevice(f.name, tpuGeneration) && (filterState.quant === 'all' || f.name.includes(filterState.quant))) .sort((a, b) => { if (selectedModel.id === LITERT_PARENT_ID) return a.size - b.size; // curated: small-first // Tier: Q4_K_M (CPU default, lowest size) → GPU/NPU Q4_0/Q8_0 → rest (CPU @@ -368,6 +384,7 @@ export const TextModelsTab: React.FC = (props) => { const { colors } = useTheme(); const styles = useThemedStyles(createStyles); + const tpuGeneration = useTensorTpuGeneration(); const downloadRecommendedFile = async (item: ModelInfo) => { const files = await fetchModelFiles([item]); @@ -395,7 +412,7 @@ export const TextModelsTab: React.FC = (props) => { }; const onboardingLiteRTCards = onboarding && Platform.OS === 'android' - ? buildCuratedLiteRTFiles().map((file, index) => { + ? buildCuratedLiteRTFiles().filter(file => liteRTFileRunsOnDevice(file.name, tpuGeneration)).map((file, index) => { const entry = getCuratedLiteRTEntry(file.name); const model = { ...LITERT_RECOMMENDED_MODEL, name: entry?.displayName ?? file.name }; const proceedDownload = () => { handleDownload(model, file); }; diff --git a/src/services/curatedLiteRTRegistry.ts b/src/services/curatedLiteRTRegistry.ts index 835c96e2a..59401a12d 100644 --- a/src/services/curatedLiteRTRegistry.ts +++ b/src/services/curatedLiteRTRegistry.ts @@ -1,4 +1,5 @@ import { DownloadedModel, ModelFile } from '../types'; +import { liteRTTensorTarget } from '../utils/modelHelpers'; import { fileExceedsBudget } from './memoryBudget'; // Synthetic parent id for the curated LiteRT models. Used both as the model id @@ -57,8 +58,34 @@ export const CURATED_LITERT_ENTRIES: readonly CuratedLiteRTEntry[] = [ "The model you have selected may exceed your device's memory and might not run reliably. For the best experience, try a smaller model.", }, }, + { + // Gemma 4 E2B compiled for the Tensor G5 TPU (Pixel 10) — the model Google's Pixel 10 TPU + // sample runs: https://github.com/google-ai-edge/litert-samples/tree/main/samples/litert/google/sample_app_tpu + // Listed only on a Tensor G5 phone (liteRTFileRunsOnDevice) and loaded on the TPU + // (litert.ts resolveLiteRTBackend). Not yet pinned: the commit holding this file and its + // exact byte size could not be resolved when it was added, so it tracks `main` and the size + // is the Hugging Face listing's 3.11 GB. The downloader takes Content-Length as authoritative. + fileName: 'gemma-4-E2B-it_Google_Tensor_G5.litertlm', + hfRepoId: 'litert-community/gemma-4-E2B-it-litert-lm', + commitHash: 'main', + sizeBytes: 3110000000, + displayName: 'Gemma 4 E2B (Tensor TPU)', + highlight: 'Runs on the Pixel 10 Tensor TPU', + liteRTVision: true, + liteRTAudio: false, + }, ]; +/** + * Whether a LiteRT file can run on this phone: a build compiled for a Google Tensor TPU only on + * that generation's TPU (`tpuGeneration` from hardwareService.getTensorTpuGeneration), any + * portable build everywhere. Both the Models tab and onboarding filter the curated list with it. + */ +export function liteRTFileRunsOnDevice(fileName: string, tpuGeneration: number | null): boolean { + const target = liteRTTensorTarget(fileName); + return target === null || target === tpuGeneration; +} + const CURATED_LITERT_INDEX: Map = new Map( CURATED_LITERT_ENTRIES.map(e => [e.fileName, e]), ); diff --git a/src/services/generationServiceHelpers.ts b/src/services/generationServiceHelpers.ts index e205fe4db..3dabf84ea 100644 --- a/src/services/generationServiceHelpers.ts +++ b/src/services/generationServiceHelpers.ts @@ -323,6 +323,13 @@ function assertLiteRTImageSupport( 'This model does not support images. Import it with vision enabled, or remove the image.', ); } + // The model takes images, but the engine came up text-only (a TPU load whose vision executor + // could not start). Say so instead of letting native drop the image silently. + if (liteRTService.isModelLoaded() && !liteRTService.supportsVision()) { + chatStore.clearStreamingMessage(); + svc.resetState(); + throw new Error('Images are not available for this model on this device. Remove the image to continue.'); + } } // assertLiteRTAudioSupport removed: audio is transcript-only (modelInputAudioUris always []), so it diff --git a/src/services/hardware.ts b/src/services/hardware.ts index efa6d5779..794a51842 100644 --- a/src/services/hardware.ts +++ b/src/services/hardware.ts @@ -6,6 +6,7 @@ import { readPerformanceCoreCount } from './cpuTopologyReader'; // Access NativeModules.LocalDreamModule dynamically (not destructured) // so it can be mocked in tests after module import. const getLocalDreamModule = () => NativeModules.LocalDreamModule; +const getLiteRTModule = () => NativeModules.LiteRTModule; import { DeviceInfo as DeviceInfoType, ModelRecommendation, @@ -30,6 +31,7 @@ class HardwareService { private cachedSoCInfo: SoCInfo | null = null; private cachedImageRecommendation: ImageModelRecommendation | null = null; private cachedOpenCLCapability: { supported: boolean; reason?: string } | null = null; + private cachedTensorTpuGeneration: Promise | null = null; async getDeviceInfo(): Promise { if (this.cachedDeviceInfo) { return this.cachedDeviceInfo; @@ -479,6 +481,28 @@ class HardwareService { return { hasNpu: HTP_ENABLED && soc.hasNPU, hasGpu: opencl.supported }; } + /** + * The Google Tensor generation whose TPU can run LiteRT models on this phone (5 on a Pixel 10), + * or null. The native probe checks the SoC, Android 16+, and that LiteRT's Tensor dispatch + * library shipped in the APK. Separate from getAccelerationCapability: llama.rn has no TPU path. + */ + getTensorTpuGeneration(): Promise { + this.cachedTensorTpuGeneration ??= this.probeTensorTpu(); + return this.cachedTensorTpuGeneration; + } + + private async probeTensorTpu(): Promise { + const mod = getLiteRTModule(); + if (Platform.OS !== 'android' || !mod?.getTpuSupport) return null; + try { + const res: { supported: boolean; tensorGeneration: number | null; reason: string | null } = await mod.getTpuSupport(); + logger.log(`[WIRE-TPU] ${JSON.stringify(res)}`); // [WIRE] real Tensor TPU probe (gates TPU routing + the Tensor model card) + return res.supported ? res.tensorGeneration : null; + } catch { + return null; + } + } + async getOpenCLCapability(): Promise<{ supported: boolean; reason?: string }> { if (this.cachedOpenCLCapability) return this.cachedOpenCLCapability; if (Platform.OS !== 'android') return { supported: false, reason: 'not_android' }; diff --git a/src/services/litert.ts b/src/services/litert.ts index 4bb6a21ec..44cbaba87 100644 --- a/src/services/litert.ts +++ b/src/services/litert.ts @@ -12,7 +12,9 @@ import { NativeModules, NativeEventEmitter, EmitterSubscription } from 'react-native'; import logger from '../utils/logger'; +import { liteRTTensorTarget } from '../utils/modelHelpers'; import { summarizeSession, runCompaction } from './liteRTCompaction'; +import { hardwareService } from './hardware'; const TAG = '[LiteRTService]'; @@ -27,6 +29,22 @@ const EVENT_TOOL_CALL = 'litert_tool_call'; type LiteRTBackend = 'cpu' | 'gpu' | 'npu'; +/** + * The backend a load asks native for. A file compiled for this phone's Google Tensor TPU + * (…_Google_Tensor_G5.litertlm on a Pixel 10) runs on the TPU: that build exists only for it, + * and Pixel 10 has no working LiteRT GPU path. Any other NPU request stays out of this release — + * a saved NPU selection loads on GPU rather than attempting an unsupported native load. + */ +async function resolveLiteRTBackend(modelPath: string, requested: LiteRTBackend): Promise { + const target = liteRTTensorTarget(modelPath.split('/').pop() ?? modelPath); + if (target !== null) { + const tpu = await hardwareService.getTensorTpuGeneration(); + if (tpu === target) return 'npu'; + logger.log(TAG, `loadModel — file targets Tensor G${target} TPU, device TPU=${tpu ?? 'none'}; using ${requested}`); + } + return requested === 'npu' ? 'gpu' : requested; +} + interface GenerateRawHandlers { onToken?: (token: string) => void; onToolCall?: (name: string, args: Record) => Promise; @@ -61,6 +79,7 @@ interface LiteRTGenerationCallbacks { class LiteRTService { private loaded = false; private modelSupportsAudio = false; + private modelSupportsVision = false; private activeBackend: LiteRTBackend | null = null; private readonly emitter: NativeEventEmitter | null = null; private subscriptions: EmitterSubscription[] = []; @@ -100,9 +119,7 @@ class LiteRTService { async loadModel(modelPath: string, preferredBackend: LiteRTBackend, opts: { supportsVision?: boolean; supportsAudio?: boolean; maxNumTokens?: number } = {}): Promise { if (!this.isAvailable()) throw new Error('LiteRT is not available on this platform'); - // NPU was exposed in a test build, but is not part of this release. - // Keep saved selections usable without attempting an unsupported native load. - if (preferredBackend === 'npu') preferredBackend = 'gpu'; + preferredBackend = await resolveLiteRTBackend(modelPath, preferredBackend); const { supportsVision = false, supportsAudio = false, maxNumTokens = 4096 } = opts; this.configuredMaxTokens = maxNumTokens; logger.log(TAG, `loadModel — path=${modelPath} backend=${preferredBackend} supportsVision=${supportsVision} supportsAudio=${supportsAudio} maxNumTokens=${maxNumTokens}`); @@ -112,7 +129,7 @@ class LiteRTService { // budget after the native RAM clamp (may be < requested). Adopt it so compaction // thresholds + the context-usage bar aren't stale. Tolerate a bare string from an // older native build (backward-compatible). - const res: string | { backend: string; maxNumTokens?: number } = + const res: string | { backend: string; maxNumTokens?: number; vision?: boolean; audio?: boolean } = await LiteRTModule.loadModel(modelPath, preferredBackend, supportsVision, supportsAudio, maxNumTokens); logger.log(`[WIRE-LITERT-LOAD] ${JSON.stringify({ requested: preferredBackend, supportsVision, supportsAudio, maxNumTokens, res })}`); // [WIRE] const actualBackend = typeof res === 'string' ? res : res.backend; @@ -124,12 +141,15 @@ class LiteRTService { } this.activeBackend = actualBackend as LiteRTBackend; this.loaded = true; - this.modelSupportsAudio = supportsAudio; + // Native reports what the engine came up with: a TPU load can drop to text-only. + this.modelSupportsVision = typeof res === 'object' && typeof res.vision === 'boolean' ? res.vision : supportsVision; + this.modelSupportsAudio = typeof res === 'object' && typeof res.audio === 'boolean' ? res.audio : supportsAudio; logger.log(TAG, `loadModel — loaded on ${this.activeBackend}`); } catch (e) { this.loaded = false; this.activeBackend = null; this.modelSupportsAudio = false; + this.modelSupportsVision = false; logger.log(TAG, `loadModel — failed: ${String(e)}`); throw e; } @@ -140,6 +160,11 @@ class LiteRTService { return this.loaded && this.modelSupportsAudio; } + /** Whether the loaded engine took images — false when a TPU load had to drop to text-only. */ + supportsVision(): boolean { + return this.loaded && this.modelSupportsVision; + } + // --------------------------------------------------------------------------- // resetConversation — cheap: closes + recreates Conversation, Engine stays // --------------------------------------------------------------------------- @@ -515,6 +540,7 @@ class LiteRTService { } finally { this.loaded = false; this.modelSupportsAudio = false; + this.modelSupportsVision = false; this.activeBackend = null; } } diff --git a/src/utils/modelHelpers.ts b/src/utils/modelHelpers.ts index 62ac9d5c6..c5ce293aa 100644 --- a/src/utils/modelHelpers.ts +++ b/src/utils/modelHelpers.ts @@ -13,3 +13,13 @@ export const getMmProjFileSize = (m?: DownloadedModel): number => */ export const isLiteRTFileName = (fileName: string): boolean => fileName.toLowerCase().endsWith('.litertlm'); + +/** + * The Google Tensor generation a LiteRT file was compiled for, read from Google's naming + * (`gemma-4-E2B-it_Google_Tensor_G5.litertlm` → 5), or null for a portable CPU/GPU build. Such a + * file runs only on that generation's TPU. + */ +export const liteRTTensorTarget = (fileName: string): number | null => { + const match = /_google_tensor_g(\d+)\.litertlm$/i.exec(fileName); + return match ? Number(match[1]) : null; +}; From a8b156a1bbc0ae81b3b1976bb93980c4d5cd2c34 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 22:48:22 +0000 Subject: [PATCH 02/13] test: give hand-written hardwareService mocks the Tensor TPU probe The Models tab now asks hardwareService.getTensorTpuGeneration(); mocks that replace the whole service answer like a non-Tensor phone (null). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../models/curatedLiteRTMemoryWarning.rendered.test.tsx | 1 + __tests__/rntl/navigation/AppNavigator.test.tsx | 1 + 2 files changed, 2 insertions(+) diff --git a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx index 1510dc4de..1da2c2a01 100644 --- a/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx +++ b/__tests__/integration/models/curatedLiteRTMemoryWarning.rendered.test.tsx @@ -117,6 +117,7 @@ jest.mock('../../../src/services/hardware', () => ({ getTotalMemoryGB: () => mockGetTotalMemoryGB(), getModelRecommendation: jest.fn(() => ({ maxParameters: 14, recommendedQuantization: 'Q4_K_M', recommendedModels: [], warning: undefined })), getImageModelRecommendation: jest.fn(() => Promise.resolve({ recommendedBackend: 'mnn', maxModelSizeMB: 2048, canRunSD: true, canRunQNN: false })), + getTensorTpuGeneration: jest.fn(() => Promise.resolve(null)), // not a Pixel Tensor TPU }, })); diff --git a/__tests__/rntl/navigation/AppNavigator.test.tsx b/__tests__/rntl/navigation/AppNavigator.test.tsx index 1d87fd43a..b3942edef 100644 --- a/__tests__/rntl/navigation/AppNavigator.test.tsx +++ b/__tests__/rntl/navigation/AppNavigator.test.tsx @@ -95,6 +95,7 @@ jest.mock('../../../src/services/hardware', () => ({ })), formatBytes: jest.fn((bytes: number) => `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GB`), formatModelSize: jest.fn(() => '4.0 GB'), + getTensorTpuGeneration: jest.fn(() => Promise.resolve(null)), // not a Pixel Tensor TPU }, })); From 84ee4cad04b6ad3dc5e684d2d7fe2382fa2b575d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:34:16 +0000 Subject: [PATCH 03/13] fix(litert): ship the Tensor dispatch library; gate images once for text-only TPU loads Pre-PR review found: - litertlm-android does not ship libLiteRtDispatch_GoogleTensor.so (Google's Pixel 10 TPU sample bundles it), so getTpuSupport always reported dispatch_lib_missing and the TPU path never ran. Add litert-npu-runtime-google-tensor 2.3.0 (LiteRT-LM 0.17.1's LiteRT has the same dispatch API), non-transitive, without the on-device compiler plugin. - The text-only image guard only covered the no-tools path; with default tools on, images still reached native and were dropped. Move the check to the single send gate (localModelAcceptsImages) and the vision affordance (activeTextCapabilities) via liteRTService.loadedTextOnly(path). - The Pixel 10 banner promised a TPU build on phones (Pixel 10a, Tensor G4) that have none listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../liteRTReleaseBackends.rendered.test.tsx | 17 ++++++++++- .../unit/hooks/useChatModelActions.test.ts | 2 +- .../generationServiceHelpers.branches.test.ts | 29 ------------------- android/app/build.gradle | 12 ++++++++ src/screens/ModelsScreen/TextModelsTab.tsx | 5 +++- src/services/engines.ts | 10 +++++-- src/services/generationServiceHelpers.ts | 7 ----- src/services/litert.ts | 11 +++++-- 8 files changed, 49 insertions(+), 44 deletions(-) diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index 7be91d621..8e1bed6b5 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -44,10 +44,25 @@ it('adopts the media the native engine came up with when a TPU load drops to tex const { liteRTService } = require('../../../src/services/litert'); await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true, supportsAudio: true }); expect(liteRTService.getActiveBackend()).toBe('npu'); - expect(liteRTService.supportsVision()).toBe(false); + expect(liteRTService.loadedTextOnly(TENSOR_G5_FILE)).toBe(true); expect(liteRTService.supportsAudio()).toBe(false); }); +it('refuses an image for a vision model whose TPU engine came up text-only, before any generation path', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false }); + const { liteRTService } = require('../../../src/services/litert'); + const { localModelAcceptsImages, activeLocalTextCapabilities } = require('../../../src/services/engines'); + const model = createDownloadedModel({ id: 'g5', engine: 'litert', filePath: TENSOR_G5_FILE, fileName: 'gemma-4-E2B-it_Google_Tensor_G5.litertlm', liteRTVision: true }); + expect(localModelAcceptsImages(model)).toBe(true); // not loaded yet: the model's own flag decides + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true }); + expect(localModelAcceptsImages(model)).toBe(false); + expect(activeLocalTextCapabilities(model).vision).toBe(false); + await liteRTService.unloadModel(); + expect(localModelAcceptsImages(model)).toBe(true); +}); + it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { installNativeBoundary(); const React = require('react'); diff --git a/__tests__/unit/hooks/useChatModelActions.test.ts b/__tests__/unit/hooks/useChatModelActions.test.ts index eb08852bd..862559ea0 100644 --- a/__tests__/unit/hooks/useChatModelActions.test.ts +++ b/__tests__/unit/hooks/useChatModelActions.test.ts @@ -44,7 +44,7 @@ jest.mock('../../../src/services/llm', () => ({ // LiteRT boundary mocked as a dumb flag reader so engines.isModelReady/deriveEngineCapabilities // (which read this DIRECT module, not the barrel) resolve deterministically for the litert path. jest.mock('../../../src/services/litert', () => ({ - liteRTService: { isModelLoaded: jest.fn(() => false) }, + liteRTService: { isModelLoaded: jest.fn(() => false), loadedTextOnly: jest.fn(() => false) }, })); // Get mock references after hoisting diff --git a/__tests__/unit/services/generationServiceHelpers.branches.test.ts b/__tests__/unit/services/generationServiceHelpers.branches.test.ts index d6b6cc53d..bf9b50f0b 100644 --- a/__tests__/unit/services/generationServiceHelpers.branches.test.ts +++ b/__tests__/unit/services/generationServiceHelpers.branches.test.ts @@ -42,7 +42,6 @@ jest.mock('../../../src/services/llm', () => ({ jest.mock('../../../src/services/litert', () => ({ liteRTService: { isModelLoaded: jest.fn(() => false), - supportsVision: jest.fn(() => true), getActiveBackend: jest.fn(() => 'cpu'), prepareConversation: jest.fn(() => Promise.resolve()), sendMessage: jest.fn(() => Promise.resolve()), @@ -193,34 +192,6 @@ describe('runLiteRTResponseImpl — image support guard', () => { expect(mockedLiteRT.sendMessage).not.toHaveBeenCalled(); }); - it('rejects an image when the loaded engine came up text-only (a TPU load without its vision executor)', async () => { - mockedGetState.mockReturnValue(liteRTAppState({ liteRTVision: true })); - mockedLiteRT.supportsVision.mockReturnValueOnce(false); - const store = chatStoreMock(); - const svc = makeServiceSvc(); - - await expect( - generateResponseImpl(svc, { - conversationId: 'conv-1', - messages: [ - { - id: '1', - timestamp: 0, - role: 'user' as const, - content: 'look', - attachments: [ - { id: 'i', type: 'image' as const, uri: 'file:///pic.png' }, - ], - }, - ], - }), - ).rejects.toThrow(/Images are not available for this model on this device/); - - expect(store.clearStreamingMessage).toHaveBeenCalled(); - expect(svc.resetState).toHaveBeenCalled(); - expect(mockedLiteRT.sendMessage).not.toHaveBeenCalled(); - }); - it('accepts an image attachment when the model has vision enabled', async () => { mockedGetState.mockReturnValue(liteRTAppState({ liteRTVision: true })); chatStoreMock(); diff --git a/android/app/build.gradle b/android/app/build.gradle index 88875fc44..3b629302c 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -146,6 +146,12 @@ android { // stores .so files inside the APK zip — File.exists() returns false for them and // exec() fails with EACCES (error=13) because you can't fork-exec from a zip entry. useLegacyPackaging = true + // Google Tensor TPU runtime (see the dependency below). Off Grid loads only models + // compiled ahead of time for the TPU, which need the dispatch library alone, not the + // runtime's on-device compiler plugin. pickFirst guards against a second copy of the + // dispatch library arriving with a future litertlm-android. + excludes.add('**/libLiteRtCompilerPlugin_google_tensor.so') + pickFirsts.add('**/libLiteRtDispatch_GoogleTensor.so') } } aaptOptions { @@ -179,6 +185,12 @@ dependencies { // LiteRT-LM on-device LLM inference (pinned — do not use latest.release) implementation("com.google.ai.edge.litertlm:litertlm-android:0.17.1") + // Google Tensor TPU dispatch library (libLiteRtDispatch_GoogleTensor.so). LiteRT-LM's + // Backend.NPU loads it from nativeLibraryDir on a Pixel; litertlm-android does not ship it + // (Google's Pixel 10 TPU sample bundles it the same way). Its version must match the LiteRT + // runtime inside litertlm-android 0.17.1, whose dispatch API is identical to LiteRT 2.3.0's. + // Not transitive: only this .so, never a second LiteRT runtime beside litertlm's own. + implementation("com.google.ai.edge.litert:litert-npu-runtime-google-tensor:2.3.0") { transitive = false } // Download layer — Room + WorkManager + OkHttp def room_version = "2.8.2" diff --git a/src/screens/ModelsScreen/TextModelsTab.tsx b/src/screens/ModelsScreen/TextModelsTab.tsx index 52c796f87..019a9e324 100644 --- a/src/screens/ModelsScreen/TextModelsTab.tsx +++ b/src/screens/ModelsScreen/TextModelsTab.tsx @@ -31,6 +31,7 @@ import { modelDownloadService } from '../../services/modelDownloadService'; import { uniformDownloadId } from '../../services/modelDownloadService/uniformId'; import { fetchModelFiles } from '../../services/modelCatalogFiles'; import { predictGgufCapabilities } from '../../utils/ggufCapabilities'; +import { liteRTTensorTarget } from '../../utils/modelHelpers'; function hasNonSortFilters(fs: FilterState): boolean { return fs.orgs.length > 0 || fs.type !== 'all' || fs.source !== 'all' || fs.size !== 'all' || fs.quant !== 'all'; @@ -118,6 +119,8 @@ const ModelDetailView: React.FC = ({ const { colors } = useTheme(); const styles = useThemedStyles(createStyles); const tpuGeneration = useTensorTpuGeneration(); + // A build for THIS phone's TPU is listed (a Pixel 10a's Tensor G4 has none yet). + const tpuBuildListed = tpuGeneration !== null && modelFiles.some(f => liteRTTensorTarget(f.name) === tpuGeneration); // Pre-set the next pending (Download Manager icon) so it fires regardless of // how the user dismisses step 9 (button or backdrop tap). @@ -271,7 +274,7 @@ const ModelDetailView: React.FC = ({ {selectedModel.id === LITERT_PARENT_ID && Platform.OS === 'android' && DeviceInfo.getModel().toLowerCase().includes('pixel 10') && ( - {tpuGeneration === null + {!tpuBuildListed ? 'GPU acceleration is not yet supported on Pixel 10. Models will run on CPU. Support coming soon.' : "Pixel 10 can't run LiteRT models on its GPU. The Tensor TPU build runs on the TPU; the other files run on the CPU."} diff --git a/src/services/engines.ts b/src/services/engines.ts index 618192828..d9f8bd527 100644 --- a/src/services/engines.ts +++ b/src/services/engines.ts @@ -144,10 +144,16 @@ export function isModelReady(model: { engine?: string; filePath?: string } | nul * loads (used to gate image sends). llama needs a present projector (mmProjPath); LiteRT carries vision in * the bundle (the liteRTVision flag). A missing/absent projector means the native completion would throw * "Multimodal support not enabled" — so the send must be blocked here, not sent and crashed (device 2026-07-14). + * A LiteRT model whose engine came up text-only (a TPU load that dropped its vision executor) would have + * native discard the image silently, so it is refused here too — once, for every generation path. */ export function localModelAcceptsImages(model: DownloadedModel | null | undefined): boolean { if (!model) return false; - return isLiteRTModel(model) ? !!model.liteRTVision : !!model.mmProjPath; + return isLiteRTModel(model) ? liteRTAcceptsImages(model) : !!model.mmProjPath; +} + +function liteRTAcceptsImages(model: { liteRTVision?: boolean; filePath: string }): boolean { + return !!model.liteRTVision && !liteRTService.loadedTextOnly(model.filePath); } /** @@ -181,7 +187,7 @@ export function activeTextCapabilities(i: { isRemote: i.isRemote, remoteCaps: i.remoteCaps, engine: i.model?.engine, - liteRTVision: litert ? litert.liteRTVision : undefined, + liteRTVision: litert ? liteRTAcceptsImages(litert) : undefined, liteRTAudio: litert ? litert.liteRTAudio : undefined, liteRTLoaded: liteRTService.isModelLoaded(), llama: { diff --git a/src/services/generationServiceHelpers.ts b/src/services/generationServiceHelpers.ts index 3dabf84ea..e205fe4db 100644 --- a/src/services/generationServiceHelpers.ts +++ b/src/services/generationServiceHelpers.ts @@ -323,13 +323,6 @@ function assertLiteRTImageSupport( 'This model does not support images. Import it with vision enabled, or remove the image.', ); } - // The model takes images, but the engine came up text-only (a TPU load whose vision executor - // could not start). Say so instead of letting native drop the image silently. - if (liteRTService.isModelLoaded() && !liteRTService.supportsVision()) { - chatStore.clearStreamingMessage(); - svc.resetState(); - throw new Error('Images are not available for this model on this device. Remove the image to continue.'); - } } // assertLiteRTAudioSupport removed: audio is transcript-only (modelInputAudioUris always []), so it diff --git a/src/services/litert.ts b/src/services/litert.ts index 44cbaba87..78c5a4796 100644 --- a/src/services/litert.ts +++ b/src/services/litert.ts @@ -80,6 +80,7 @@ class LiteRTService { private loaded = false; private modelSupportsAudio = false; private modelSupportsVision = false; + private loadedModelPath: string | null = null; private activeBackend: LiteRTBackend | null = null; private readonly emitter: NativeEventEmitter | null = null; private subscriptions: EmitterSubscription[] = []; @@ -144,12 +145,14 @@ class LiteRTService { // Native reports what the engine came up with: a TPU load can drop to text-only. this.modelSupportsVision = typeof res === 'object' && typeof res.vision === 'boolean' ? res.vision : supportsVision; this.modelSupportsAudio = typeof res === 'object' && typeof res.audio === 'boolean' ? res.audio : supportsAudio; + this.loadedModelPath = modelPath; logger.log(TAG, `loadModel — loaded on ${this.activeBackend}`); } catch (e) { this.loaded = false; this.activeBackend = null; this.modelSupportsAudio = false; this.modelSupportsVision = false; + this.loadedModelPath = null; logger.log(TAG, `loadModel — failed: ${String(e)}`); throw e; } @@ -160,9 +163,10 @@ class LiteRTService { return this.loaded && this.modelSupportsAudio; } - /** Whether the loaded engine took images — false when a TPU load had to drop to text-only. */ - supportsVision(): boolean { - return this.loaded && this.modelSupportsVision; + /** True when `modelPath` is the loaded model and its engine came up without vision: a TPU load + * whose vision executor could not start dropped to text-only, so native would discard images. */ + loadedTextOnly(modelPath: string): boolean { + return this.loaded && this.loadedModelPath === modelPath && !this.modelSupportsVision; } // --------------------------------------------------------------------------- @@ -541,6 +545,7 @@ class LiteRTService { this.loaded = false; this.modelSupportsAudio = false; this.modelSupportsVision = false; + this.loadedModelPath = null; this.activeBackend = null; } } From 8d8ac7378f210bf82680857ff2ccb1a0f5852192 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 7 Oct 2026 23:36:11 +0000 Subject: [PATCH 04/13] test: give useChatModelStateSync's liteRTService mock loadedTextOnly Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- __tests__/unit/hooks/useChatModelStateSync.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/__tests__/unit/hooks/useChatModelStateSync.test.ts b/__tests__/unit/hooks/useChatModelStateSync.test.ts index 028387032..667d3a654 100644 --- a/__tests__/unit/hooks/useChatModelStateSync.test.ts +++ b/__tests__/unit/hooks/useChatModelStateSync.test.ts @@ -18,7 +18,7 @@ jest.mock('../../../src/services/llm', () => ({ }, })); jest.mock('../../../src/services/litert', () => ({ - liteRTService: { isModelLoaded: jest.fn(() => false) }, + liteRTService: { isModelLoaded: jest.fn(() => false), loadedTextOnly: jest.fn(() => false) }, })); const { llmService } = require('../../../src/services/llm'); From 975d50432365c642924e5dd03ab9c63a45f6f3ff Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 00:00:43 +0000 Subject: [PATCH 05/13] fix(litert): refuse images at the engine when a TPU load came up text-only The send gate runs before the lazy model load, so on the first image turn (and on resends and model fallback) it could not yet know the TPU load would drop vision, and native discarded the image silently. sendMessage, which generateRaw and the tool loop also use, now rejects image turns when the loaded engine has no vision. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../liteRTReleaseBackends.rendered.test.tsx | 15 +++++++++++++++ src/services/litert.ts | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index 8e1bed6b5..b45054f38 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -63,6 +63,21 @@ it('refuses an image for a vision model whose TPU engine came up text-only, befo expect(localModelAcceptsImages(model)).toBe(true); }); +it('refuses an image at the engine when the TPU load came up text-only, on the first turn after a lazy load', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + boundary.litert.module.loadModel.mockResolvedValueOnce({ backend: 'npu', maxNumTokens: 4096, vision: false, audio: false }); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu', { supportsVision: true }); + // The tool loop's entry point (generateRaw) and the plain path (sendMessage) both refuse. + await expect(liteRTService.generateRaw('what is this?', { imageUris: ['file:///pic.png'] })) + .rejects.toThrow(/Images are not available for this model on this device/); + const onError = jest.fn(); + await liteRTService.sendMessage('what is this?', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError }, { imageUris: ['file:///pic.png'] }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringMatching(/Images are not available/) })); + expect(boundary.litert.calls.sendMessageWithImages).toEqual([]); +}); + it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { installNativeBoundary(); const React = require('react'); diff --git a/src/services/litert.ts b/src/services/litert.ts index 78c5a4796..14429ab38 100644 --- a/src/services/litert.ts +++ b/src/services/litert.ts @@ -326,6 +326,14 @@ class LiteRTService { media?: { imageUris?: string[]; audioUris?: string[] }, ): Promise { if (!this.isAvailable() || !this.loaded) { callbacks.onError(new Error('No LiteRT model loaded')); return; } + // An engine without vision (a TPU load that dropped to text-only) would have native discard the + // image without a word. Every generation path sends through here, after any lazy load, so refuse + // it once, here. + if (media?.imageUris?.some(Boolean) && !this.modelSupportsVision) { + logger.log(TAG, 'sendMessage — refused images: the loaded engine has no vision'); + callbacks.onError(new Error('Images are not available for this model on this device. Remove the image to continue.')); + return; + } // Reset accumulators this.currentContent = ''; From 649e6757512ada550d13e54a8d43ef27e6cd3010 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 02:57:49 +0000 Subject: [PATCH 06/13] fix(litert): pin the Tensor G5 model and cap it at its compiled 4096-token context - Pin gemma-4-E2B-it_Google_Tensor_G5.litertlm to Hugging Face commit b3ca0d2f (3,113,545,589 bytes, sha256 af108298...). - Its TPU graph has a fixed 4096-token KV cache (every prefill_128 / decode / verify signature). LiteRT-LM's NPU executor silently clamps a larger maxNumTokens, so JS kept a larger budget than the engine had. Declare the limit on the entry, and have the LiteRT loader cap the requested context at a curated build's own limit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- ...activeModelService.loaders.branches.test.ts | 14 ++++++++++++++ .../services/curatedLiteRTContextLimit.test.ts | 18 ++++++------------ src/services/activeModelService/loaders.ts | 5 ++++- src/services/curatedLiteRTRegistry.ts | 10 +++++----- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/__tests__/unit/services/activeModelService.loaders.branches.test.ts b/__tests__/unit/services/activeModelService.loaders.branches.test.ts index 5d896fdb2..4844a30bb 100644 --- a/__tests__/unit/services/activeModelService.loaders.branches.test.ts +++ b/__tests__/unit/services/activeModelService.loaders.branches.test.ts @@ -190,6 +190,20 @@ describe('doLoadLiteRTModel branches', () => { expect(ToastAndroid.showWithGravity).toHaveBeenCalled(); }); + it('caps the context at a curated build\'s own limit (the Tensor TPU build is compiled for 4096 tokens)', async () => { + mockedLiteRT.loadModel.mockResolvedValue(undefined); + mockedLiteRT.getActiveBackend.mockReturnValue('npu'); + const fileName = 'gemma-4-E2B-it_Google_Tensor_G5.litertlm'; + const ctx = liteCtx({ + model: { id: `offgrid/litert-recommended/${fileName}`, engine: 'litert', fileName, filePath: `/models/${fileName}`, fileSize: 3113545589 }, + store: { settings: { ...makeStore().settings, liteRTBackend: 'gpu', liteRTMaxTokens: 8192 } }, + }); + + await doLoadTextModel(ctx); + + expect(mockedLiteRT.loadModel).toHaveBeenCalledWith(`/models/${fileName}`, 'gpu', expect.objectContaining({ maxNumTokens: 4096 })); + }); + it('runs warmup when active backend is gpu/npu', async () => { mockedLiteRT.loadModel.mockResolvedValue(undefined); mockedLiteRT.getActiveBackend.mockReturnValue('npu'); diff --git a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts index e031b8bbf..49621ba3f 100644 --- a/__tests__/unit/services/curatedLiteRTContextLimit.test.ts +++ b/__tests__/unit/services/curatedLiteRTContextLimit.test.ts @@ -1,27 +1,21 @@ import { CURATED_LITERT_ENTRIES, getCuratedLiteRTContextLimit, LITERT_PARENT_ID } from '../../../src/services/curatedLiteRTRegistry'; import { createDownloadedModel } from '../../utils/factories'; -const ENTRIES_WITH_LIMIT = CURATED_LITERT_ENTRIES.filter(e => e.maxContextTokens !== undefined); - -it.each(ENTRIES_WITH_LIMIT)('matches the pinned artifact for $displayName without changing saved model data', entry => { +it.each(CURATED_LITERT_ENTRIES)('matches the pinned artifact for $displayName without changing saved model data', entry => { const model = createDownloadedModel({ engine: 'litert', id: `${LITERT_PARENT_ID}/${entry.fileName}`, fileName: entry.fileName, fileSize: entry.sizeBytes, }); - expect(getCuratedLiteRTContextLimit(model)).toBe(32000); + expect(getCuratedLiteRTContextLimit(model)).toBe(entry.maxContextTokens); expect(getCuratedLiteRTContextLimit({ ...model, id: 'imported' })).toBeNull(); expect(getCuratedLiteRTContextLimit({ ...model, fileSize: 1 })).toBeNull(); const origin = { repoId: entry.hfRepoId, revision: entry.commitHash, path: entry.fileName }; - expect(getCuratedLiteRTContextLimit({ ...model, id: 'transferred', origin })).toBe(32000); + expect(getCuratedLiteRTContextLimit({ ...model, id: 'transferred', origin })).toBe(entry.maxContextTokens); expect(getCuratedLiteRTContextLimit({ ...model, origin: { ...origin, revision: 'main' } })).toBeNull(); expect(getCuratedLiteRTContextLimit({ ...model, engine: 'llama' })).toBeNull(); }); -it('imposes no catalog context limit on an entry that declares none (the Tensor TPU build)', () => { - const entry = CURATED_LITERT_ENTRIES.find(e => e.maxContextTokens === undefined)!; - const model = createDownloadedModel({ - engine: 'litert', id: `${LITERT_PARENT_ID}/${entry.fileName}`, - fileName: entry.fileName, fileSize: entry.sizeBytes, - }); - expect(getCuratedLiteRTContextLimit(model)).toBeNull(); +it('caps the Tensor TPU build at its compiled 4096-token KV cache', () => { + const entry = CURATED_LITERT_ENTRIES.find(e => e.fileName === 'gemma-4-E2B-it_Google_Tensor_G5.litertlm')!; + expect(entry.maxContextTokens).toBe(4096); }); diff --git a/src/services/activeModelService/loaders.ts b/src/services/activeModelService/loaders.ts index 294c7f724..2e5317955 100644 --- a/src/services/activeModelService/loaders.ts +++ b/src/services/activeModelService/loaders.ts @@ -8,6 +8,7 @@ import { useAppStore } from '../../stores'; import { DownloadedModel, LlamaDownloadedModel, ONNXImageModel, INFERENCE_BACKENDS } from '../../types'; import { llmService } from '../llm'; import { liteRTService } from '../litert'; +import { getCuratedLiteRTContextLimit } from '../curatedLiteRTRegistry'; import { unloadAllTextEngines } from '../engines'; import { localDreamGeneratorService as onnxImageGeneratorService } from '../localDreamGenerator'; import { modelManager } from '../modelManager'; @@ -106,7 +107,9 @@ async function doLoadLiteRTModel(ctx: TextLoadContext): Promise { const preferredBackend = ctx.store.settings.liteRTBackend; - const maxTokens = ctx.store.settings.liteRTMaxTokens ?? 4096; + // A curated build's own limit wins over a larger setting: native would clamp silently (the + // Tensor TPU build's KV cache is fixed at 4096) while JS kept the larger budget for compaction. + const maxTokens = Math.min(ctx.store.settings.liteRTMaxTokens ?? 4096, getCuratedLiteRTContextLimit(liteRTModel) ?? Infinity); const contextScalar = Math.max(1, maxTokens / 4096); const baseTimeoutMs = 90_000; const timeoutMs = Math.min(Math.ceil(baseTimeoutMs * contextScalar), 180_000); diff --git a/src/services/curatedLiteRTRegistry.ts b/src/services/curatedLiteRTRegistry.ts index 59401a12d..753420d7f 100644 --- a/src/services/curatedLiteRTRegistry.ts +++ b/src/services/curatedLiteRTRegistry.ts @@ -62,17 +62,17 @@ export const CURATED_LITERT_ENTRIES: readonly CuratedLiteRTEntry[] = [ // Gemma 4 E2B compiled for the Tensor G5 TPU (Pixel 10) — the model Google's Pixel 10 TPU // sample runs: https://github.com/google-ai-edge/litert-samples/tree/main/samples/litert/google/sample_app_tpu // Listed only on a Tensor G5 phone (liteRTFileRunsOnDevice) and loaded on the TPU - // (litert.ts resolveLiteRTBackend). Not yet pinned: the commit holding this file and its - // exact byte size could not be resolved when it was added, so it tracks `main` and the size - // is the Hugging Face listing's 3.11 GB. The downloader takes Content-Length as authoritative. + // (litert.ts resolveLiteRTBackend). fileName: 'gemma-4-E2B-it_Google_Tensor_G5.litertlm', hfRepoId: 'litert-community/gemma-4-E2B-it-litert-lm', - commitHash: 'main', - sizeBytes: 3110000000, + commitHash: 'b3ca0d2f076785a8f4b2219ddbd2bdb99954eae1', + sizeBytes: 3113545589, displayName: 'Gemma 4 E2B (Tensor TPU)', highlight: 'Runs on the Pixel 10 Tensor TPU', liteRTVision: true, liteRTAudio: false, + // Compiled for the TPU with a fixed 4096-token KV cache (every prefill/decode signature). + maxContextTokens: 4096, }, ]; From 1a92314d3cdd2ffd47df4019b012ba71d9c41ebd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 03:11:42 +0000 Subject: [PATCH 07/13] fix(litert): refuse audio on an audio-less engine; split the per-tier load attempts - CodeRabbit: after a text-only TPU fallback, an audio turn would reach the model as empty text. sendMessage now refuses audio when the loaded engine has none, like images. (The app sends no audio to LiteRT today; this keeps the engine honest if that changes.) - SonarCloud (kotlin:S3776): initializeWithFallback's cognitive complexity rose to 21. Move one tier's retries and the NPU text-only retry into tryTier; behavior unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../liteRTReleaseBackends.rendered.test.tsx | 4 ++ .../ai/offgridmobile/litert/LiteRTModule.kt | 51 ++++++++++--------- src/services/litert.ts | 11 ++-- 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index b45054f38..e950276e9 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -76,6 +76,10 @@ it('refuses an image at the engine when the TPU load came up text-only, on the f await liteRTService.sendMessage('what is this?', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError }, { imageUris: ['file:///pic.png'] }); expect(onError).toHaveBeenCalledWith(expect.objectContaining({ message: expect.stringMatching(/Images are not available/) })); expect(boundary.litert.calls.sendMessageWithImages).toEqual([]); + // Audio on an engine that came up without it would reach the model as empty text: refused too. + await expect(liteRTService.generateRaw('', { audioUris: ['file:///note.wav'] })) + .rejects.toThrow(/Audio input is not available for this model on this device/); + expect(boundary.litert.calls.sendMessageWithMedia).toEqual([]); }); it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index 08f556393..e2a581ff1 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -211,38 +211,41 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : private suspend fun initializeWithFallback(modelPath: String, requested: Backend, visionEnabled: Boolean, audioEnabled: Boolean): Backend { val chain = buildBackendChain(requested) + for (backend in chain) { + if (tryTier(modelPath, backend, visionEnabled, audioEnabled)) return backend + if (backend != chain.last()) { + Log.i(TAG, "initializeWithFallback — ${backendName(backend)} exhausted retries, falling back to next tier") + } + } + throw IllegalStateException("All backends failed") + } + /** Every attempt one backend tier gets before the chain moves to the next. */ + private suspend fun tryTier(modelPath: String, backend: Backend, visionEnabled: Boolean, audioEnabled: Boolean): Boolean { + val name = backendName(backend) // GPU/NPU failures can be transient (e.g. VRAM not yet released after a model switch). // Retry up to 2 extra times with backoff before giving up on a non-CPU backend. val gpuRetries = 2 val gpuRetryDelayMs = 600L - - for (backend in chain) { - val name = backendName(backend) - val maxAttempts = if (backend is Backend.CPU) 1 else gpuRetries + 1 - - for (attempt in 1..maxAttempts) { - if (attempt > 1) { - Log.i(TAG, "initializeWithFallback — $name retry $attempt/$maxAttempts after ${gpuRetryDelayMs}ms") - delay(gpuRetryDelayMs) - } else { - Log.i(TAG, "initializeWithFallback — trying $name vision=$visionEnabled audio=$audioEnabled") - } - if (tryInitBackend(modelPath, backend, name, visionEnabled, audioEnabled)) return backend - } - - // Google's Tensor TPU reference app keeps a model on the TPU text-only when its vision - // or audio executor can't start there, rather than dropping to a far slower tier. - if (backend is Backend.NPU && (visionEnabled || audioEnabled)) { - Log.i(TAG, "initializeWithFallback — $name retrying text-only") - if (tryInitBackend(modelPath, backend, name, visionEnabled = false, audioEnabled = false)) return backend + val maxAttempts = if (backend is Backend.CPU) 1 else gpuRetries + 1 + + for (attempt in 1..maxAttempts) { + if (attempt > 1) { + Log.i(TAG, "initializeWithFallback — $name retry $attempt/$maxAttempts after ${gpuRetryDelayMs}ms") + delay(gpuRetryDelayMs) + } else { + Log.i(TAG, "initializeWithFallback — trying $name vision=$visionEnabled audio=$audioEnabled") } + if (tryInitBackend(modelPath, backend, name, visionEnabled, audioEnabled)) return true + } - if (backend != chain.last()) { - Log.i(TAG, "initializeWithFallback — $name exhausted retries, falling back to next tier") - } + // Google's Tensor TPU reference app keeps a model on the TPU text-only when its vision + // or audio executor can't start there, rather than dropping to a far slower tier. + if (backend is Backend.NPU && (visionEnabled || audioEnabled)) { + Log.i(TAG, "initializeWithFallback — $name retrying text-only") + return tryInitBackend(modelPath, backend, name, visionEnabled = false, audioEnabled = false) } - throw IllegalStateException("All backends failed") + return false } // ------------------------------------------------------------------------- diff --git a/src/services/litert.ts b/src/services/litert.ts index 14429ab38..f1134747b 100644 --- a/src/services/litert.ts +++ b/src/services/litert.ts @@ -326,14 +326,19 @@ class LiteRTService { media?: { imageUris?: string[]; audioUris?: string[] }, ): Promise { if (!this.isAvailable() || !this.loaded) { callbacks.onError(new Error('No LiteRT model loaded')); return; } - // An engine without vision (a TPU load that dropped to text-only) would have native discard the - // image without a word. Every generation path sends through here, after any lazy load, so refuse - // it once, here. + // An engine without vision or audio (a TPU load that dropped to text-only) would have native + // discard the media without a word — an audio-only turn would even reach the model as empty + // text. Every generation path sends through here, after any lazy load, so refuse it once, here. if (media?.imageUris?.some(Boolean) && !this.modelSupportsVision) { logger.log(TAG, 'sendMessage — refused images: the loaded engine has no vision'); callbacks.onError(new Error('Images are not available for this model on this device. Remove the image to continue.')); return; } + if (media?.audioUris?.some(Boolean) && !this.modelSupportsAudio) { + logger.log(TAG, 'sendMessage — refused audio: the loaded engine has no audio input'); + callbacks.onError(new Error('Audio input is not available for this model on this device.')); + return; + } // Reset accumulators this.currentContent = ''; From 21d7b9bcb5cca65d6319910ac6ebe82532e32df3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 03:34:18 +0000 Subject: [PATCH 08/13] fix(litert): harden Tensor TPU detection and the TPU load path Findings from Pixel factory images and LiteRT sources: - Every Tensor G3+ Pixel reports SOC_MODEL "Tensor G" with SOC_MANUFACTURER "Google" (Pixel 10 family: "Tensor G5"; Pixel 10a: "Tensor G4"). The codename fallback never matched on real Pixels and false-matched other vendors' devices ("mustang", "rango"); drop it and require the Google manufacturer. - LiteRT's own NPU check excludes Android 16 builds starting BP2A; mirror it. - Field reports show the process aborting (SIGABRT) when the TPU runtime is unusable. Probe that libedgetpu_litert loads before offering the TPU, and mark TPU loads in flight: two loads in a row that never returned turn the TPU off for this install instead of crashing on every load. - A TPU build's graphs are TPU bytecode, so an NPU request no longer falls back to GPU/CPU (which cannot run it) or retries (each attempt re-maps a 3 GB model), and its fixed KV cache is not shrunk by the free-RAM clamp. - build.gradle: state the 0.17.1 / 2.3.0 dispatch compatibility precisely. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- android/app/build.gradle | 4 +- .../ai/offgridmobile/litert/LiteRTModule.kt | 97 +++++++++++++++---- .../java/ai/offgridmobile/litert/TensorTpu.kt | 38 +++++--- .../ai/offgridmobile/litert/TensorTpuTest.kt | 50 +++++++--- 4 files changed, 140 insertions(+), 49 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index 3b629302c..d2f8f478f 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -188,7 +188,9 @@ dependencies { // Google Tensor TPU dispatch library (libLiteRtDispatch_GoogleTensor.so). LiteRT-LM's // Backend.NPU loads it from nativeLibraryDir on a Pixel; litertlm-android does not ship it // (Google's Pixel 10 TPU sample bundles it the same way). Its version must match the LiteRT - // runtime inside litertlm-android 0.17.1, whose dispatch API is identical to LiteRT 2.3.0's. + // runtime inside litertlm-android: 0.17.1 embeds a pre-release LiteRT 2.3.0 with the same + // dispatch API version (0.1.0); its runtime function table lacks only append_metric, which the + // Google Tensor dispatch library never calls. 2.3.0 is the only published version. // Not transitive: only this .so, never a second LiteRT runtime beside litertlm's own. implementation("com.google.ai.edge.litert:litert-npu-runtime-google-tensor:2.3.0") { transitive = false } diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index e2a581ff1..0a5a1eabb 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -46,6 +46,12 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : const val EVENT_TOOL_CALL = "litert_tool_call" const val EVENT_DEBUG_LOG = "litert_debug_log" + /** Set while a TPU engine initializes; still set at launch means that load killed the app. */ + private const val TPU_LOAD_IN_FLIGHT = "tpu_load_in_flight" + private const val TPU_LOAD_STRIKES = "tpu_load_strikes" + private const val TPU_STRIKES_STAMP = "tpu_strikes_install" + private const val TPU_MAX_STRIKES = 2 + // Base timeouts per backend tier (for default 4096-token context). // Actual timeout scales up proportionally for larger context windows // because KV-cache allocation takes longer at higher token counts. @@ -120,15 +126,18 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : scope.launch { try { + val requestedBackend = parseBackend(backendStr) // Clamp the token budget to what free RAM can actually hold. The KV cache // grows with the budget, and an over-budget request aborts engine creation // (SIGABRT in nativeCreateEngine) or segfaults during inference. Degrading - // to a smaller context keeps the app working instead of crashing. - configuredMaxTokens = resolveSafeMaxTokens(modelPath, maxNumTokens) + // to a smaller context keeps the app working instead of crashing. Not on the + // TPU: a TPU build's KV cache is fixed at compile time, so a smaller budget + // saves no memory and only shrinks the usable context. + configuredMaxTokens = if (requestedBackend is Backend.NPU) maxNumTokens + else resolveSafeMaxTokens(modelPath, maxNumTokens) // Unload any existing engine first cleanupEngine() - val requestedBackend = parseBackend(backendStr) Log.i(TAG, "loadModel — attempting backend chain from $backendStr") // Sets supportsVision/supportsAudio to what the engine actually came up with — the @@ -155,7 +164,7 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : } } - // 3-tier fallback: NPU → GPU → CPU + // Fallback chain: GPU → CPU. NPU stands alone (see buildBackendChain). /** GPU init crashes on Pixel 10 (open LiteRT SDK bug) — skip the GPU delegate * there, on both the main backend chain and the vision delegate. */ private fun shouldSkipGpu(): Boolean = @@ -164,11 +173,10 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : private fun buildBackendChain(requested: Backend): List { val skipGpu = shouldSkipGpu() return when (requested) { - is Backend.NPU -> listOfNotNull( - Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir), - if (skipGpu) null else Backend.GPU(), - Backend.CPU(), - ) + // NPU is requested only for a file compiled for this phone's TPU, whose graphs are + // TPU bytecode (DISPATCH_OP): no GPU or CPU tier can run it, so fail with the TPU's + // own error instead of masking it with a doomed fallback. + is Backend.NPU -> listOf(Backend.NPU(nativeLibraryDir = reactContext.applicationInfo.nativeLibraryDir)) is Backend.GPU -> if (skipGpu) listOf(Backend.CPU()) else listOf(Backend.GPU(), Backend.CPU()) else -> listOf(Backend.CPU()) } @@ -193,7 +201,15 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : audioBackend = if (audioEnabled) Backend.CPU() else null, ) eng = Engine(cfg) - withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { eng.initialize() } + // A broken TPU runtime aborts the process (SIGABRT) rather than throwing. Mark the + // attempt so the next launch reports the TPU as unusable instead of crashing again. + if (backend is Backend.NPU) markTpuLoadInFlight(true) + try { + withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { eng.initialize() } + } finally { + if (backend is Backend.NPU) markTpuLoadInFlight(false) + } + if (backend is Backend.NPU) recordTpuLoadSucceeded() engine = eng supportsVision = visionEnabled supportsAudio = audioEnabled @@ -223,11 +239,12 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : /** Every attempt one backend tier gets before the chain moves to the next. */ private suspend fun tryTier(modelPath: String, backend: Backend, visionEnabled: Boolean, audioEnabled: Boolean): Boolean { val name = backendName(backend) - // GPU/NPU failures can be transient (e.g. VRAM not yet released after a model switch). - // Retry up to 2 extra times with backoff before giving up on a non-CPU backend. + // GPU failures can be transient (e.g. VRAM not yet released after a model switch). + // Retry up to 2 extra times with backoff. CPU and NPU failures are not, and each NPU + // attempt re-maps a multi-GB model. val gpuRetries = 2 val gpuRetryDelayMs = 600L - val maxAttempts = if (backend is Backend.CPU) 1 else gpuRetries + 1 + val maxAttempts = if (backend is Backend.GPU) gpuRetries + 1 else 1 for (attempt in 1..maxAttempts) { if (attempt > 1) { @@ -643,11 +660,19 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : @ReactMethod fun getTpuSupport(promise: Promise) { val safe = SafePromise(promise, TAG) - val socModel = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MODEL else "" - val generation = TensorTpu.generation(socModel, Build.HARDWARE ?: "", Build.BOARD ?: "", Build.DEVICE ?: "") + val sdk = Build.VERSION.SDK_INT + val socModel = if (sdk >= Build.VERSION_CODES.S) Build.SOC_MODEL else "" + val socManufacturer = if (sdk >= Build.VERSION_CODES.S) Build.SOC_MANUFACTURER else "" + val generation = TensorTpu.generation(socModel, socManufacturer) val dispatchLibPresent = File(reactContext.applicationInfo.nativeLibraryDir, TensorTpu.DISPATCH_LIB).exists() - val reason = TensorTpu.unsupportedReason(generation, Build.VERSION.SDK_INT, dispatchLibPresent) - Log.i(TAG, "getTpuSupport — soc=$socModel hardware=${Build.HARDWARE} tensorGeneration=$generation sdk=${Build.VERSION.SDK_INT} dispatchLib=$dispatchLibPresent reason=$reason") + // Only probe the vendor library on a phone that passed the cheap checks. + val eligible = generation != null && sdk >= TensorTpu.MIN_SDK && dispatchLibPresent + val reason = TensorTpu.unsupportedReason( + generation, sdk, Build.ID ?: "", dispatchLibPresent, + vendorLibLoads = !eligible || vendorLibLoads(), + previousInitCrashed = tpuLoadCrashedBefore(), + ) + Log.i(TAG, "getTpuSupport — soc=$socModel/$socManufacturer build=${Build.ID} tensorGeneration=$generation sdk=$sdk dispatchLib=$dispatchLibPresent reason=$reason") safe.resolve(Arguments.createMap().apply { putBoolean("supported", reason == null) if (generation != null) putInt("tensorGeneration", generation) else putNull("tensorGeneration") @@ -655,6 +680,44 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : }) } + private fun vendorLibLoads(): Boolean = try { + System.loadLibrary(TensorTpu.VENDOR_LIB) + true + } catch (e: Throwable) { + Log.w(TAG, "getTpuSupport — lib${TensorTpu.VENDOR_LIB}.so does not load: ${e.message}") + false + } + + // Strikes are keyed to this install: an app update gets a fresh start. + private val tpuPrefs by lazy { reactContext.getSharedPreferences("litert_tpu", Context.MODE_PRIVATE) } + private fun installStamp(): Long = try { + reactContext.packageManager.getPackageInfo(reactContext.packageName, 0).lastUpdateTime + } catch (e: Exception) { 0L } + + private fun markTpuLoadInFlight(inFlight: Boolean) { + // commit(), not apply(): the marker must be on disk before a native abort can kill us. + val edit = tpuPrefs.edit() + if (inFlight) edit.putBoolean(TPU_LOAD_IN_FLIGHT, true) else edit.remove(TPU_LOAD_IN_FLIGHT) + edit.commit() + } + + private fun recordTpuLoadSucceeded() { tpuPrefs.edit().remove(TPU_LOAD_STRIKES).commit() } + + /** + * A marker still set at launch means a TPU load never returned: the process died in it. Two + * such deaths in a row (one may just be the user closing the app mid-load) turn the TPU off + * for this install, so a crashing TPU runtime cannot crash the app on every load. + */ + private fun tpuLoadCrashedBefore(): Boolean { + val stamp = installStamp() + var strikes = if (tpuPrefs.getLong(TPU_STRIKES_STAMP, stamp) == stamp) tpuPrefs.getInt(TPU_LOAD_STRIKES, 0) else 0 + if (tpuPrefs.getBoolean(TPU_LOAD_IN_FLIGHT, false)) { + strikes++ + tpuPrefs.edit().remove(TPU_LOAD_IN_FLIGHT).putInt(TPU_LOAD_STRIKES, strikes).putLong(TPU_STRIKES_STAMP, stamp).commit() + } + return strikes >= TPU_MAX_STRIKES + } + /** Current free system RAM in MB. */ private fun availableRamMb(): Long { val am = reactContext.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager diff --git a/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt index ea5ff37e1..dc96ff27c 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/TensorTpu.kt @@ -2,37 +2,45 @@ package ai.offgridmobile.litert /** * Can this phone run a LiteRT model on its Google Tensor TPU? Pure (no Android deps) so it is - * unit-testable; [LiteRTModule.getTpuSupport] feeds it the Build properties. + * unit-testable; [LiteRTModule.getTpuSupport] feeds it the Build properties and runtime probes. */ internal object TensorTpu { /** LiteRT's Google Tensor dispatch library, shipped in the app's native lib dir. */ const val DISPATCH_LIB = "libLiteRtDispatch_GoogleTensor.so" + /** The Pixel vendor library the dispatch library opens (System.loadLibrary name). */ + const val VENDOR_LIB = "edgetpu_litert" /** LiteRT supports the Tensor TPU from Android 16 (API 36). */ const val MIN_SDK = 36 + /** LiteRT's own NPU check rejects this Android 16 build family: it shipped without NPU support. */ + private const val UNSUPPORTED_BUILD_PREFIX = "BP2A" - /** Pixel SoC and device codenames → Tensor generation, for builds whose SOC_MODEL doesn't name the chip. */ - private val CODENAMES = mapOf( - "gs101" to 1, "gs201" to 2, "zuma" to 3, "zumapro" to 4, "laguna" to 5, "malibu" to 6, - // Pixel 10, 10 Pro, 10 Pro XL, 10 Pro Fold - "frankel" to 5, "blazer" to 5, "mustang" to 5, "rango" to 5, - ) - private val SOC_MODEL = Regex("""tensor\s*g?(\d+)""") + private val SOC_MODEL = Regex("""^tensor\s*g(\d+)$""") /** - * Tensor generation (5 for a Pixel 10's "Tensor G5"), or null when this is not a Tensor SoC. - * [socModel] is Build.SOC_MODEL; [codenames] are Build.HARDWARE / BOARD / DEVICE. + * Tensor generation (5 for a Pixel 10's "Tensor G5"), or null when this is not a Google Tensor + * SoC. Every Tensor G3+ Pixel reports SOC_MODEL "Tensor G" with SOC_MANUFACTURER "Google" + * (factory images); device codenames are not used — other vendors reuse them. */ - fun generation(socModel: String, vararg codenames: String): Int? { - SOC_MODEL.find(socModel.lowercase())?.let { return it.groupValues[1].toIntOrNull() } - // Older Pixels report the SoC codename itself as SOC_MODEL ("GS101"). - return (listOf(socModel) + codenames).firstNotNullOfOrNull { CODENAMES[it.lowercase()] } + fun generation(socModel: String, socManufacturer: String): Int? { + if (!socManufacturer.equals("Google", ignoreCase = true)) return null + return SOC_MODEL.find(socModel.trim().lowercase())?.groupValues?.get(1)?.toIntOrNull() } /** Why the TPU can't be used here, or null when it can. */ - fun unsupportedReason(generation: Int?, sdkInt: Int, dispatchLibPresent: Boolean): String? = when { + fun unsupportedReason( + generation: Int?, + sdkInt: Int, + buildId: String, + dispatchLibPresent: Boolean, + vendorLibLoads: Boolean, + previousInitCrashed: Boolean, + ): String? = when { generation == null -> "not_tensor" sdkInt < MIN_SDK -> "android_too_old" + buildId.startsWith(UNSUPPORTED_BUILD_PREFIX) -> "android_build_unsupported" !dispatchLibPresent -> "dispatch_lib_missing" + !vendorLibLoads -> "vendor_lib_unavailable" + previousInitCrashed -> "previous_tpu_load_crashed" else -> null } } diff --git a/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt index 9b35d86eb..e51272059 100644 --- a/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt +++ b/android/app/src/test/java/ai/offgridmobile/litert/TensorTpuTest.kt @@ -4,38 +4,56 @@ import org.junit.Assert.assertEquals import org.junit.Assert.assertNull import org.junit.Test -/** Tests for TensorTpu — whether a phone can run a LiteRT model on its Google Tensor TPU. */ +/** + * Tests for TensorTpu — whether a phone can run a LiteRT model on its Google Tensor TPU. + * SOC_MODEL / SOC_MANUFACTURER values are the ones Google's factory images report (vendor build.prop). + */ @Suppress("kotlin:S100") // Backtick test names are idiomatic Kotlin class TensorTpuTest { @Test - fun `reads the generation from a Pixel SOC_MODEL`() { - assertEquals(5, TensorTpu.generation("Tensor G5", "blazer", "blazer", "blazer")) - assertEquals(4, TensorTpu.generation("Tensor G4", "caiman", "caiman", "caiman")) + fun `reads the generation from a Pixel's SOC_MODEL`() { + assertEquals(5, TensorTpu.generation("Tensor G5", "Google")) // Pixel 10 family + assertEquals(4, TensorTpu.generation("Tensor G4", "Google")) // Pixel 9 family, Pixel 10a + assertEquals(3, TensorTpu.generation("Tensor G3", "Google")) // Pixel 8 family + assertEquals(6, TensorTpu.generation("Tensor G6", "Google")) // Pixel 11 family } @Test - fun `falls back to the SoC or device codename when SOC_MODEL does not name the chip`() { - assertEquals(5, TensorTpu.generation("", "laguna")) - assertEquals(5, TensorTpu.generation("", "unknown", "unknown", "mustang")) - assertEquals(1, TensorTpu.generation("GS101", "oriole", "oriole", "oriole")) + fun `is not a Tensor TPU when the chip carries no generation`() { + assertNull(TensorTpu.generation("Tensor", "Google")) // Pixel 6 + assertNull(TensorTpu.generation("GS201", "Google")) // Pixel 7 } @Test - fun `is not a Tensor phone on Snapdragon or MediaTek`() { - assertNull(TensorTpu.generation("SM8750", "qcom", "sun", "pa3q")) - assertNull(TensorTpu.generation("MT6989", "mt6989", "k6989v1_64", "duchamp")) + fun `is not a Tensor TPU on other vendors, even with a look-alike SOC_MODEL`() { + assertNull(TensorTpu.generation("SM8750", "QTI")) + assertNull(TensorTpu.generation("MT6989", "Mediatek")) + assertNull(TensorTpu.generation("Tensor G5", "Other")) + assertNull(TensorTpu.generation("", "")) } @Test - fun `allows the TPU on a Pixel 10 on Android 16 with the dispatch library`() { - assertNull(TensorTpu.unsupportedReason(5, 36, dispatchLibPresent = true)) + fun `allows the TPU on a Pixel 10 on Android 16 with a working runtime`() { + assertNull(reason()) } @Test fun `explains why the TPU cannot be used`() { - assertEquals("not_tensor", TensorTpu.unsupportedReason(null, 36, dispatchLibPresent = true)) - assertEquals("android_too_old", TensorTpu.unsupportedReason(5, 35, dispatchLibPresent = true)) - assertEquals("dispatch_lib_missing", TensorTpu.unsupportedReason(5, 36, dispatchLibPresent = false)) + assertEquals("not_tensor", reason(generation = null)) + assertEquals("android_too_old", reason(sdkInt = 35)) + assertEquals("android_build_unsupported", reason(buildId = "BP2A.250605.031")) + assertEquals("dispatch_lib_missing", reason(dispatchLibPresent = false)) + assertEquals("vendor_lib_unavailable", reason(vendorLibLoads = false)) + assertEquals("previous_tpu_load_crashed", reason(previousInitCrashed = true)) } + + private fun reason( + generation: Int? = 5, + sdkInt: Int = 36, + buildId: String = "CP3A.261005.005", + dispatchLibPresent: Boolean = true, + vendorLibLoads: Boolean = true, + previousInitCrashed: Boolean = false, + ) = TensorTpu.unsupportedReason(generation, sdkInt, buildId, dispatchLibPresent, vendorLibLoads, previousInitCrashed) } From c7541fa2a64b34ac6737b1753ccb383796d97ad5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 13:24:04 +0000 Subject: [PATCH 09/13] fix(litert): start the TPU vision and audio encoders during the load LiteRT-LM 0.17.1 creates the vision and audio executors when the first conversation is registered, not in Engine.initialize(). A TPU encoder that could not start therefore passed the load, skipped the text-only retry, and failed every chat turn instead. Open and close one conversation inside the TPU load (and its crash marker) so that failure lands where the retry is. Manifest comment: stock Pixel 10 firmware lists the four declared libraries in /vendor/etc/public.libraries.txt, and libedgetpu_litert.so loads the other three. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- android/app/src/main/AndroidManifest.xml | 5 +++-- .../src/main/java/ai/offgridmobile/litert/LiteRTModule.kt | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index af76c58eb..34f791d1f 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -77,8 +77,9 @@ - + diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index 0a5a1eabb..e73943704 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -206,6 +206,12 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : if (backend is Backend.NPU) markTpuLoadInFlight(true) try { withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { eng.initialize() } + // LiteRT-LM starts the vision and audio encoders with the first conversation, not in + // initialize(). Open one now so an encoder that can't run on the TPU fails this + // attempt, and the text-only retry below, instead of every chat turn. + if (backend is Backend.NPU && (visionEnabled || audioEnabled)) { + eng.createConversation().close() + } } finally { if (backend is Backend.NPU) markTpuLoadInFlight(false) } From 74d25ac53fdff4bbc17ef95d4ce7c6646e2b0a82 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 13:24:04 +0000 Subject: [PATCH 10/13] fix(litert): treat LiteRT-LM context-limit errors as a full context The chat compacts and retries when an engine reports a full context, but only llama.cpp's wording matched. LiteRT-LM's limit errors surfaced as a failed turn instead. This bites hardest on the Tensor TPU: the 4096-token context is compiled into the model, and LiteRT-LM 0.17.1 refuses a prefill once the history passes 4096 - 128 tokens. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../liteRTReleaseBackends.rendered.test.tsx | 15 +++++++++++++++ __tests__/unit/services/contextCompaction.test.ts | 5 +++++ src/services/contextCompaction.ts | 6 ++++++ 3 files changed, 26 insertions(+) diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index e950276e9..ac9a98ee0 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -82,6 +82,21 @@ it('refuses an image at the engine when the TPU load came up text-only, on the f expect(boundary.litert.calls.sendMessageWithMedia).toEqual([]); }); +it('reports a spent TPU context as context-full, so the chat compacts and retries instead of failing', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + const { liteRTService } = require('../../../src/services/litert'); + const { contextCompactionService } = require('../../../src/services/contextCompaction'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu'); + // LiteRT-LM's TPU executor refuses a prefill that would pass the context compiled into the model. + boundary.litert.scriptError('Prefill length (128) plus current step (3990) exceeds max sequence length (4096).'); + const onError = jest.fn(); + await liteRTService.sendMessage('hi', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError }); + await new Promise(resolve => setTimeout(resolve, 0)); + expect(onError).toHaveBeenCalledTimes(1); + expect(contextCompactionService.isContextFullError(onError.mock.calls[0][0])).toBe(true); +}); + it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { installNativeBoundary(); const React = require('react'); diff --git a/__tests__/unit/services/contextCompaction.test.ts b/__tests__/unit/services/contextCompaction.test.ts index 39273b3a3..747618864 100644 --- a/__tests__/unit/services/contextCompaction.test.ts +++ b/__tests__/unit/services/contextCompaction.test.ts @@ -73,6 +73,11 @@ describe('isContextFullError', () => { ['context window exceeded', true], ['context length exceeded', true], ['context is full', true], + // LiteRT-LM's own limit errors, as the native module forwards them + ['Max number of tokens reached, context window out of bounds', true], + ['Input token ids are too long. Exceeding the maximum number of tokens allowed: 4100 >= 4096', true], + ['Reached maximum number of tokens.', true], + ['Generation failed: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).', true], ])('"%s" → %s', (msg, expected) => { const input = typeof msg === 'string' ? new Error(msg) : msg; expect(contextCompactionService.isContextFullError(input)).toBe(expected); diff --git a/src/services/contextCompaction.ts b/src/services/contextCompaction.ts index 3dba50c86..69d6f7a34 100644 --- a/src/services/contextCompaction.ts +++ b/src/services/contextCompaction.ts @@ -28,6 +28,12 @@ const CONTEXT_FULL_PATTERNS = [ 'not enough context space', 'context window exceeded', 'context length exceeded', + // LiteRT-LM (any backend) + 'context window out of bounds', + 'exceeding the maximum number of tokens', + // LiteRT-LM on the Tensor TPU, whose context is fixed when the model is compiled + 'reached maximum number of tokens', + 'exceeds max sequence length', ]; /** Fraction of context allocated to the summary */ From c4455d511cba6e8159bafd77aeb59e5f78cfc229 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 13:39:57 +0000 Subject: [PATCH 11/13] fix(litert): gate the SoC reads where Android lint can see the check Lint's NewApi check does not follow SDK_INT through a local, so :app:lintDebug failed on Build.SOC_MODEL / SOC_MANUFACTURER (API 31). Compare Build.VERSION.SDK_INT directly; behaviour is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index e73943704..4c107f92e 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -667,8 +667,8 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : fun getTpuSupport(promise: Promise) { val safe = SafePromise(promise, TAG) val sdk = Build.VERSION.SDK_INT - val socModel = if (sdk >= Build.VERSION_CODES.S) Build.SOC_MODEL else "" - val socManufacturer = if (sdk >= Build.VERSION_CODES.S) Build.SOC_MANUFACTURER else "" + val socModel = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MODEL else "" + val socManufacturer = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) Build.SOC_MANUFACTURER else "" val generation = TensorTpu.generation(socModel, socManufacturer) val dispatchLibPresent = File(reactContext.applicationInfo.nativeLibraryDir, TensorTpu.DISPATCH_LIB).exists() // Only probe the vendor library on a phone that passed the cheap checks. From 9d39899f2f890f700f680ed8502b9a12aa53e184 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 14:35:11 +0000 Subject: [PATCH 12/13] fix(litert): let the full-context retry actually rebuild the conversation Review of the previous commit found the compact-and-retry it enabled for LiteRT could not succeed: - A failed native send leaves the refused turn in LiteRT-LM's Conversation (and, on the TPU, the prefill chunks that ran before the limit), but the service kept the conversation as active. The retry's prepareConversation saw the same id and prompt and sent into that same overfull session, so the retry, and every later turn in the chat, failed the same way. Invalidate the conversation on any native send error; the next prepare rebuilds it from the JS history, and LiteRT's own 65% compaction applies. - The tool loop's native-parse fallback matches any "Status Code: 3", which LiteRT-LM also uses for a full context. It retried without tools on the full history instead of compacting. Rethrow context-full errors first. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../liteRTReleaseBackends.rendered.test.tsx | 20 +++++++++++++++++++ .../generationToolLoop.branches.test.ts | 16 +++++++++++++++ src/services/generationToolLoop.ts | 2 ++ src/services/litert.ts | 5 +++++ 4 files changed, 43 insertions(+) diff --git a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx index ac9a98ee0..e0ddfc465 100644 --- a/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx +++ b/__tests__/integration/settings/liteRTReleaseBackends.rendered.test.tsx @@ -97,6 +97,26 @@ it('reports a spent TPU context as context-full, so the chat compacts and retrie expect(contextCompactionService.isContextFullError(onError.mock.calls[0][0])).toBe(true); }); +it('rebuilds the native conversation from the compacted history when the chat retries a full context', async () => { + const boundary = installNativeBoundary(); + boundary.litert.module.getTpuSupport.mockResolvedValue(PIXEL_10_TPU); + const { liteRTService } = require('../../../src/services/litert'); + await liteRTService.loadModel(TENSOR_G5_FILE, 'gpu'); + const history = [ + { role: 'user', content: 'first question' }, { role: 'assistant', content: 'first answer' }, + { role: 'user', content: 'second question' }, { role: 'assistant', content: 'second answer' }, + ]; + await liteRTService.prepareConversation('chat-1', 'sys', { history }); + boundary.litert.scriptError('Status Code: 3. Message: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).'); + await liteRTService.sendMessage('third question', { onToken: jest.fn(), onReasoning: jest.fn(), onComplete: jest.fn(), onError: jest.fn() }); + await new Promise(resolve => setTimeout(resolve, 0)); + // The native conversation still holds the refused turn, so the retry must not reuse it. + const compacted = [{ role: 'user', content: 'Summary: two questions answered.' }, { role: 'assistant', content: 'Noted.' }]; + await liteRTService.prepareConversation('chat-1', 'sys', { history: compacted }); + const historyJson = boundary.litert.calls.resetConversation.at(-1)?.[5]; + expect(historyJson).toBe(JSON.stringify(compacted)); +}); + it('offers only CPU and GPU in Chat and Model Settings even with a saved NPU preference', async () => { installNativeBoundary(); const React = require('react'); diff --git a/__tests__/unit/services/generationToolLoop.branches.test.ts b/__tests__/unit/services/generationToolLoop.branches.test.ts index e8385c8da..0a69d617c 100644 --- a/__tests__/unit/services/generationToolLoop.branches.test.ts +++ b/__tests__/unit/services/generationToolLoop.branches.test.ts @@ -46,6 +46,7 @@ jest.mock('../../../src/stores', () => ({ addMessage: mockAddMessage, setStreamingMessage: mockSetStreamingMessage, setIsThinking: mockSetIsThinking, + conversations: [], }), }, useRemoteServerStore: { getState: () => ({ activeServerId: null }) }, @@ -400,6 +401,21 @@ describe('runToolLoop — LiteRT loop branches', () => { expect(ctx.onFinalResponse).toHaveBeenCalledWith('recovered answer'); }); + it('leaves a full context (also Status Code 3) to compaction instead of dropping the tools', async () => { + mockedLiteRT.generateRaw.mockRejectedValue(new Error( + 'Status Code: 3. Message: Prefill length (128) plus current step (3990) exceeds max sequence length (4096).', + )); + + const ctx = createContext({ + messages: [makeMessage({ role: 'user', content: 'hi' })], + }); + await expect(runToolLoop(ctx)).rejects.toThrow(/exceeds max sequence length/); + const toolless = mockedLiteRT.prepareConversation.mock.calls.filter( + (c: any[]) => Array.isArray(c[2]?.tools) && c[2].tools.length === 0, + ); + expect(toolless).toEqual([]); + }); + it('rethrows a non-parse error without retrying (line 427 negative branch)', async () => { mockedLiteRT.generateRaw.mockRejectedValue(new Error('out of memory')); diff --git a/src/services/generationToolLoop.ts b/src/services/generationToolLoop.ts index 25eba489c..ae9c86ac8 100644 --- a/src/services/generationToolLoop.ts +++ b/src/services/generationToolLoop.ts @@ -884,6 +884,8 @@ async function callLiteRTForLoop( // The litertlm native FC parser hard-fails (Status Code 3) when a small model emits // a malformed tool call. Rather than surface a raw "Generation Error", retry once // WITHOUT tools so the user still gets a text answer instead of a crashed turn. + // A full context is also Status Code 3; the caller compacts for that instead. + if (contextCompactionService.isContextFullError(msg)) throw e; if (!/parse (tool|FC) calls|Status Code: 3/i.test(msg)) throw e; logger.warn( `[ToolLoop] LiteRT tool-call parse failed; retrying without tools: ${msg.slice( diff --git a/src/services/litert.ts b/src/services/litert.ts index f1134747b..24f21710e 100644 --- a/src/services/litert.ts +++ b/src/services/litert.ts @@ -419,6 +419,10 @@ class LiteRTService { this.emitter!.addListener(EVENT_ERROR, (message: string) => { logger.log(TAG, `sendMessage — error: ${message}`); this.clearSubscriptions(); + // The native conversation keeps the failed turn (and, on the TPU, the prefill that ran + // before the limit), so the next prepareConversation must rebuild it — e.g. the + // compaction retry after a full context. + this.invalidateConversation(); this.currentToolCallHandler = null; callbacks.onError(new Error(message)); @@ -458,6 +462,7 @@ class LiteRTService { } } catch (e) { this.clearSubscriptions(); + this.invalidateConversation(); const err = e instanceof Error ? e : new Error(String(e)); logger.log(TAG, `sendMessage — native error: ${err.message}`); callbacks.onError(err); From 144b76e62878e3fba250a5e97e8f481d5f4e088f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 17:26:47 +0000 Subject: [PATCH 13/13] fix(litert): count the TPU encoder start against the load's time budget CodeRabbit: the conversation opened to start the vision/audio encoders ran outside the init timeout. Run it inside the same withTimeout as initialize(), so a slow encoder start fails the attempt and leads to the text-only retry like a slow initialize() does. (Both are blocking native calls, so the timeout fails a load that finishes late; it cannot cut one short. The encoder start previously happened untimed on the first turn.) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017BbwQk6htC5Q6UNdKk2m8H --- .../java/ai/offgridmobile/litert/LiteRTModule.kt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt index 4c107f92e..1d8281004 100644 --- a/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt +++ b/android/app/src/main/java/ai/offgridmobile/litert/LiteRTModule.kt @@ -205,12 +205,15 @@ class LiteRTModule(private val reactContext: ReactApplicationContext) : // attempt so the next launch reports the TPU as unusable instead of crashing again. if (backend is Backend.NPU) markTpuLoadInFlight(true) try { - withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { eng.initialize() } - // LiteRT-LM starts the vision and audio encoders with the first conversation, not in - // initialize(). Open one now so an encoder that can't run on the TPU fails this - // attempt, and the text-only retry below, instead of every chat turn. - if (backend is Backend.NPU && (visionEnabled || audioEnabled)) { - eng.createConversation().close() + withTimeout(initTimeoutMs(backend, configuredMaxTokens)) { + eng.initialize() + // LiteRT-LM starts the vision and audio encoders with the first conversation, not + // in initialize(). Open one now, within the same time budget, so an encoder that + // can't run on the TPU fails this attempt, and the text-only retry below, instead + // of every chat turn. + if (backend is Backend.NPU && (visionEnabled || audioEnabled)) { + eng.createConversation().close() + } } } finally { if (backend is Backend.NPU) markTpuLoadInFlight(false)