diff --git a/harness/core-ai/availability.device.harness.ts b/harness/core-ai/availability.device.harness.ts index 85f9cf4a..fae2d01d 100644 --- a/harness/core-ai/availability.device.harness.ts +++ b/harness/core-ai/availability.device.harness.ts @@ -21,7 +21,7 @@ describe('core-ai availability', () => { it('rejects a model that is not on disk', async () => { await expect( - CoreAI.generateImage({ source: '/missing' }, 'a cat', {}) + CoreAI.generateImage('missing-model', { source: '/missing' }, 'a cat', {}) ).rejects.toMatchObject({ code: expect.any(String) }) }) diff --git a/harness/core-ai/image-generation.device.harness.ts b/harness/core-ai/image-generation.device.harness.ts index d2b46bcc..a17fbaa6 100644 --- a/harness/core-ai/image-generation.device.harness.ts +++ b/harness/core-ai/image-generation.device.harness.ts @@ -1,3 +1,4 @@ +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' import { generateImage } from 'ai' import { describe, expect, it } from 'react-native-harness' @@ -80,4 +81,49 @@ describe('core-ai image generation', () => { expect.objectContaining({ type: 'unsupported', feature: 'size' }), ]) }) + + it('reports each denoising step in order', async () => { + const progress: CoreAIGenerationProgress[] = [] + await generateImage({ + model, + prompt, + seed: 1, + providerOptions: { + 'core-ai': { + stepCount: 2, + // providerOptions is typed as JSON, so a callback needs a cast + onProgress: ((update: CoreAIGenerationProgress) => + progress.push(update)) as any, + }, + }, + }) + + expect(progress).toEqual([ + { phase: 'denoising', step: 1, totalSteps: 2 }, + { phase: 'denoising', step: 2, totalSteps: 2 }, + ]) + }) + + it('stops after the current step when aborted', async () => { + const controller = new AbortController() + const progress: CoreAIGenerationProgress[] = [] + const result = generateImage({ + model, + prompt, + abortSignal: controller.signal, + providerOptions: { + 'core-ai': { + stepCount: 4, + onProgress: ((update: CoreAIGenerationProgress) => { + progress.push(update) + controller.abort() + }) as any, + }, + }, + }) + + await expect(result).rejects.toThrow() + // A step already running when the cancel lands still finishes + expect(progress.length).toBeLessThan(4) + }) }) diff --git a/harness/core-ai/video-generation.device.harness.ts b/harness/core-ai/video-generation.device.harness.ts index 6fc81dc3..a2f0a458 100644 --- a/harness/core-ai/video-generation.device.harness.ts +++ b/harness/core-ai/video-generation.device.harness.ts @@ -1,3 +1,4 @@ +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' import { experimental_generateVideo as generateVideo } from 'ai' import { describe, expect, it } from 'react-native-harness' @@ -43,6 +44,32 @@ describe('core-ai video generation', () => { ) }) + it('reports each phase and denoising step in order', async () => { + const progress: CoreAIGenerationProgress[] = [] + await generateVideo({ + model, + prompt: 'a red ball rolling across a wooden table', + seed: 1, + providerOptions: { + 'core-ai': { + stepCount: 2, + cfgCutoff: 1, + // providerOptions is typed as JSON, so a callback needs a cast + onProgress: ((update: CoreAIGenerationProgress) => + progress.push(update)) as any, + }, + }, + }) + + expect(progress.map(({ phase, step }) => `${phase} ${step}`)).toEqual([ + 'encoding 0', + 'denoising 0', + 'denoising 1', + 'decoding 2', + 'assembling 2', + ]) + }) + it('stops when aborted', async () => { const controller = new AbortController() const result = generate({}, controller.signal) diff --git a/packages/core-ai/ios/CoreAI.mm b/packages/core-ai/ios/CoreAI.mm index 6f0f3e6c..cde7e812 100644 --- a/packages/core-ai/ios/CoreAI.mm +++ b/packages/core-ai/ios/CoreAI.mm @@ -237,15 +237,32 @@ - (void)generateVideo:(nonnull NSString *)requestId options:(nonnull NSDictionary *)options resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { - [_coreAI generateVideo:requestId config:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; + [_coreAI generateVideo:requestId + config:CoreAIConfigToDictionary(config) + prompt:prompt + options:options + onProgress:^(NSDictionary *progress) { + [self emitOnGenerationProgress:progress]; + } + resolve:resolve + reject:reject]; } -- (void)generateImage:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config +- (void)generateImage:(nonnull NSString *)requestId + config:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config prompt:(nonnull NSString *)prompt options:(nonnull NSDictionary *)options resolve:(nonnull RCTPromiseResolveBlock)resolve reject:(nonnull RCTPromiseRejectBlock)reject { - [_coreAI generateImage:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; + [_coreAI generateImage:requestId + config:CoreAIConfigToDictionary(config) + prompt:prompt + options:options + onProgress:^(NSDictionary *progress) { + [self emitOnGenerationProgress:progress]; + } + resolve:resolve + reject:reject]; } - (void)runTask:(nonnull NSString *)task diff --git a/packages/core-ai/ios/CoreAIImpl.swift b/packages/core-ai/ios/CoreAIImpl.swift index 292bfa67..d103bfe3 100644 --- a/packages/core-ai/ios/CoreAIImpl.swift +++ b/packages/core-ai/ios/CoreAIImpl.swift @@ -649,6 +649,7 @@ public class CoreAIImpl: NSObject { config: [String: Any], prompt: String, options: [String: Any]?, + onProgress: @escaping @Sendable ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -659,7 +660,9 @@ public class CoreAIImpl: NSObject { startTask(requestId, onError: { _, code, message in reject(code, message, nil) }) { - resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:])) + resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:]) { + onProgress($0.merging(["requestId": requestId]) { current, _ in current }) + }) } } catch { reject(errorCode(error), error.localizedDescription, error) @@ -672,11 +675,15 @@ public class CoreAIImpl: NSObject { #endif } + /// Generates images, reporting each denoising step. cancelStream(requestId) + /// stops it after the current step @objc public func generateImage( - _ config: [String: Any], + _ requestId: String, + config: [String: Any], prompt: String, options: [String: Any]?, + onProgress: @escaping @Sendable ([String: Any]) -> Void, resolve: @escaping (Any?) -> Void, reject: @escaping (String, String, Error?) -> Void ) { @@ -684,34 +691,43 @@ public class CoreAIImpl: NSObject { if #available(iOS 27, macOS 27, *) { do { let sourceURL = try resolveSourceURL(config) - Task { - do { - let startingImage = try Self.startingImage(from: options) - let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) - // Exports without a VAE encoder generate from the prompt alone, - // which the metadata reports so JS can warn - let editsImage = startingImage != nil && pipeline.supportsImageToImage - let configuration = try self.makePipelineConfiguration( - prompt: prompt, - pipeline: pipeline, - options: options, - startingImage: editsImage ? startingImage : nil - ) - let result = try await pipeline.generateImages(configuration: configuration) { _ in true } - let images = try result.images.map(Self.pngBase64) - var metadata: [String: Any] = [ - "count": images.count, - "imageFormat": "image/png", - "width": result.images.first?.width ?? 0, - "height": result.images.first?.height ?? 0 - ] - if startingImage != nil { - metadata["imageToImage"] = editsImage - } - resolve(["images": images, "metadata": metadata]) - } catch { - reject(errorCode(error), error.localizedDescription, error) + startTask(requestId, onError: { _, code, message in + reject(code, message, nil) + }) { + let startingImage = try Self.startingImage(from: options) + let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) + // Exports without a VAE encoder generate from the prompt alone, + // which the metadata reports so JS can warn + let editsImage = startingImage != nil && pipeline.supportsImageToImage + let configuration = try self.makePipelineConfiguration( + prompt: prompt, + pipeline: pipeline, + options: options, + startingImage: editsImage ? startingImage : nil + ) + let result = try await pipeline.generateImages(configuration: configuration) { progress in + // Steps after a cancel aren't reported, since generation stops + guard !Task.isCancelled else { return false } + onProgress([ + "requestId": requestId, + "phase": "denoising", + "step": progress.step, + "totalSteps": progress.totalSteps + ]) + return true + } + try Task.checkCancellation() + let images = try result.images.map(Self.pngBase64) + var metadata: [String: Any] = [ + "count": images.count, + "imageFormat": "image/png", + "width": result.images.first?.width ?? 0, + "height": result.images.first?.height ?? 0 + ] + if startingImage != nil { + metadata["imageToImage"] = editsImage } + resolve(["images": images, "metadata": metadata]) } } catch { reject(errorCode(error), error.localizedDescription, error) diff --git a/packages/core-ai/ios/CoreAIVideoGeneration.swift b/packages/core-ai/ios/CoreAIVideoGeneration.swift index fd6e6bda..7889fcc7 100644 --- a/packages/core-ai/ios/CoreAIVideoGeneration.swift +++ b/packages/core-ai/ios/CoreAIVideoGeneration.swift @@ -15,11 +15,23 @@ enum VideoGeneration { // only while it runs nonisolated(unsafe) private static var loaded: (key: String, load: Task<(WanPipeline, ExportedShape), Error>)? - static func generate(prompt: String, sourceURL: URL, options: [String: Any]) async throws -> [String: Any] { + static func generate( + prompt: String, + sourceURL: URL, + options: [String: Any], + onProgress: @escaping @Sendable ([String: Any]) -> Void + ) async throws -> [String: Any] { let (pipeline, shape) = try await self.pipeline(at: sourceURL, options: options) let configuration = try makeConfiguration(prompt: prompt, pipeline: pipeline, shape: shape, options: options) - let result = try await pipeline.generateVideo(configuration: configuration) { _ in - !Task.isCancelled + let result = try await pipeline.generateVideo(configuration: configuration) { progress in + // Phases after a cancel aren't reported, since generation stops + guard !Task.isCancelled else { return false } + onProgress([ + "phase": phaseName(progress.phase), + "step": progress.step, + "totalSteps": progress.totalSteps + ]) + return true } try Task.checkCancellation() @@ -41,6 +53,15 @@ enum VideoGeneration { ] } + private static func phaseName(_ phase: VideoProgress.Phase) -> String { + switch phase { + case .encoding: return "encoding" + case .denoising: return "denoising" + case .decoding: return "decoding" + case .assembling: return "assembling" + } + } + // Concurrent calls for the same bundle share one load private static func pipeline(at url: URL, options: [String: Any]) async throws -> (WanPipeline, ExportedShape) { let lazy = options["lazyModelLoading"] as? Bool ?? true diff --git a/packages/core-ai/src/NativeCoreAI.ts b/packages/core-ai/src/NativeCoreAI.ts index 9b0a38f7..5bd79300 100644 --- a/packages/core-ai/src/NativeCoreAI.ts +++ b/packages/core-ai/src/NativeCoreAI.ts @@ -60,6 +60,13 @@ export type CoreAITranscriptionUpdateEvent = { endSecond: number } +export type CoreAIGenerationProgressEvent = { + requestId: string + phase: string + step: number + totalSteps: number +} + export interface Spec extends TurboModule { getCapabilities(): Promise @@ -138,7 +145,9 @@ export interface Spec extends TurboModule { prompt: string, options: CodegenTypes.UnsafeObject ): Promise + // Cancel with cancelStream(requestId) generateImage( + requestId: string, config: NativeCoreAIModelConfig, prompt: string, options: CodegenTypes.UnsafeObject @@ -161,6 +170,8 @@ export interface Spec extends TurboModule { onStreamComplete: CodegenTypes.EventEmitter onStreamError: CodegenTypes.EventEmitter onTranscriptionUpdate: CodegenTypes.EventEmitter + // Progress of generateImage and generateVideo requests + onGenerationProgress: CodegenTypes.EventEmitter } export default TurboModuleRegistry.getEnforcing('NativeCoreAI') diff --git a/packages/core-ai/src/__tests__/ai-sdk.test.ts b/packages/core-ai/src/__tests__/ai-sdk.test.ts index 308deaf0..664a3207 100644 --- a/packages/core-ai/src/__tests__/ai-sdk.test.ts +++ b/packages/core-ai/src/__tests__/ai-sdk.test.ts @@ -70,6 +70,7 @@ const generateVideo = mock( ) const generateImage = mock( async ( + _requestId: string, _config: Record, _prompt: string, options: Record @@ -108,6 +109,7 @@ const onStreamUpdate = createEmitter() const onStreamComplete = createEmitter() const onStreamError = createEmitter() const onTranscriptionUpdate = createEmitter() +const onGenerationProgress = createEmitter() const transcriptionResult = { text: 'hello world', @@ -151,6 +153,7 @@ mock.module('react-native', () => ({ appendTranscriptionAudio, finishTranscriptionStream, onTranscriptionUpdate, + onGenerationProgress, }), }, })) @@ -1238,7 +1241,7 @@ describe('core ai image editing', () => { ...overrides, }) - const lastOptions = () => generateImage.mock.calls.at(-1)![2] + const lastOptions = () => generateImage.mock.calls.at(-1)![3] test('passes the first image as base64', async () => { const { warnings } = await imageModel().doGenerate( @@ -1314,3 +1317,84 @@ describe('core ai image editing', () => { ]) }) }) + +describe('core ai generation progress', () => { + const imageModel = () => coreAI.imageModel({ id: 'flux', source: '/flux' }) + const videoModel = () => coreAI.videoModel({ id: 'wan', source: '/wan' }) + + test('reports progress of its own image request', async () => { + generateImage.mockImplementationOnce(async (requestId) => { + onGenerationProgress.emit({ + requestId: 'other', + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + onGenerationProgress.emit({ + requestId, + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + return { images: ['AAAA'] } + }) + const onProgress = mock(() => {}) + + await imageModel().doGenerate({ + prompt: 'a cat', + n: 1, + size: undefined, + aspectRatio: undefined, + seed: undefined, + files: undefined, + mask: undefined, + providerOptions: { 'core-ai': { onProgress: onProgress as any } }, + }) + + expect(onProgress.mock.calls).toEqual([ + [{ phase: 'denoising', step: 1, totalSteps: 2 }], + ]) + // The callback stays in JS + expect(generateImage.mock.calls.at(-1)![3]).not.toHaveProperty('onProgress') + }) + + test('stops reporting video progress once the call finishes', async () => { + let requestId = '' + generateVideo.mockImplementationOnce(async (id) => { + requestId = id + onGenerationProgress.emit({ + requestId, + phase: 'encoding', + step: 0, + totalSteps: 2, + }) + return { video: 'AAAA', mediaType: 'video/mp4' } + }) + const onProgress = mock(() => {}) + + await videoModel().generate('a cat', { onProgress }) + onGenerationProgress.emit({ + requestId, + phase: 'denoising', + step: 1, + totalSteps: 2, + }) + + expect(onProgress.mock.calls).toEqual([ + [{ phase: 'encoding', step: 0, totalSteps: 2 }], + ]) + }) + + test('cancels the native image request on abort', async () => { + const controller = new AbortController() + generateImage.mockImplementationOnce(async (requestId) => { + controller.abort() + expect(cancelStream).toHaveBeenCalledWith(requestId) + throw new Error('cancelled') + }) + + await expect( + imageModel().generate('a cat', {}, controller.signal) + ).rejects.toMatchObject({ name: 'AbortError' }) + }) +}) diff --git a/packages/core-ai/src/ai-sdk.ts b/packages/core-ai/src/ai-sdk.ts index d6fdf9b3..db2805f0 100644 --- a/packages/core-ai/src/ai-sdk.ts +++ b/packages/core-ai/src/ai-sdk.ts @@ -38,6 +38,7 @@ import type { CoreAIEmbeddingResult, CoreAIGenerationOptions, CoreAIGenerationPart, + CoreAIGenerationProgress, CoreAIGenerationResult, CoreAIImageGenerationOptions, CoreAIImageGenerationResult, @@ -440,26 +441,33 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { this.modelId = getModelId(config) } + // The abort signal stops generation after the current denoising step async generate( prompt: string, - options: CoreAIImageGenerationOptions = {} + { onProgress, ...options }: CoreAIImageGenerationOptions = {}, + abortSignal?: AbortSignal ): Promise { - return NativeCoreAI.generateImage( - this.config, - prompt, - options - ) as Promise + return runGeneration( + (requestId) => + NativeCoreAI.generateImage(requestId, this.config, prompt, options), + onProgress, + abortSignal + ) } async doGenerate(options: ImageModelV4CallOptions) { - const result = await this.generate(options.prompt ?? '', { - n: options.n, - size: options.size, - aspectRatio: options.aspectRatio, - seed: options.seed, - ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), - ...(options.providerOptions?.['core-ai'] ?? {}), - }) + const result = await this.generate( + options.prompt ?? '', + { + n: options.n, + size: options.size, + aspectRatio: options.aspectRatio, + seed: options.seed, + ...(options.files?.[0] ? await toImageInput(options.files[0]) : {}), + ...(options.providerOptions?.['core-ai'] ?? {}), + }, + options.abortSignal + ) return { images: result.images, @@ -484,6 +492,40 @@ export class CoreAIImageGenerationModel implements ImageModelV4 { } } +// Runs a native image or video request, forwards its progress events, and +// cancels it when the signal aborts +async function runGeneration( + run: (requestId: string) => Promise, + onProgress: ((progress: CoreAIGenerationProgress) => void) | undefined, + abortSignal: AbortSignal | undefined +): Promise { + // React Native's AbortSignal has no throwIfAborted() + if (abortSignal?.aborted) { + throw toAbortError(abortSignal) + } + const requestId = generateId() + const onAbort = () => NativeCoreAI.cancelStream(requestId) + abortSignal?.addEventListener('abort', onAbort) + const subscription = + onProgress && + NativeCoreAI.onGenerationProgress(({ requestId: id, ...progress }) => { + if (id === requestId) { + onProgress(progress as CoreAIGenerationProgress) + } + }) + try { + return (await run(requestId)) as Result + } catch (error) { + if (abortSignal?.aborted) { + throw toAbortError(abortSignal) + } + throw error + } finally { + subscription?.remove() + abortSignal?.removeEventListener('abort', onAbort) + } +} + // Wan 2.1 sizes for the aspect ratios it is trained on const VIDEO_SIZES: Record = { '16:9': { width: 832, height: 480 }, @@ -507,31 +549,15 @@ export class CoreAIVideoModel implements VideoModelV4 { // signal stops it after the current denoising step async generate( prompt: string, - options: CoreAIVideoGenerationOptions = {}, + { onProgress, ...options }: CoreAIVideoGenerationOptions = {}, abortSignal?: AbortSignal ): Promise { - // React Native's AbortSignal has no throwIfAborted() - if (abortSignal?.aborted) { - throw toAbortError(abortSignal) - } - const requestId = generateId() - const onAbort = () => NativeCoreAI.cancelStream(requestId) - abortSignal?.addEventListener('abort', onAbort) - try { - return (await NativeCoreAI.generateVideo( - requestId, - this.config, - prompt, - options - )) as CoreAIVideoGenerationResult - } catch (error) { - if (abortSignal?.aborted) { - throw toAbortError(abortSignal) - } - throw error - } finally { - abortSignal?.removeEventListener('abort', onAbort) - } + return runGeneration( + (requestId) => + NativeCoreAI.generateVideo(requestId, this.config, prompt, options), + onProgress, + abortSignal + ) } async doGenerate( diff --git a/packages/core-ai/src/index.ts b/packages/core-ai/src/index.ts index 90f615e2..e8c95eef 100644 --- a/packages/core-ai/src/index.ts +++ b/packages/core-ai/src/index.ts @@ -18,6 +18,7 @@ export type { CoreAIEmbeddingResult, CoreAIGenerationOptions, CoreAIGenerationPart, + CoreAIGenerationProgress, CoreAIImageGenerationOptions, CoreAIImageGenerationResult, CoreAIImageInput, diff --git a/packages/core-ai/src/types.ts b/packages/core-ai/src/types.ts index c0cc700d..024e2670 100644 --- a/packages/core-ai/src/types.ts +++ b/packages/core-ai/src/types.ts @@ -184,6 +184,15 @@ export interface CoreAITranscriptionResult { metadata?: CodegenTypes.UnsafeObject } +export interface CoreAIGenerationProgress { + // Images report each denoising step once it is done, from step 1. Videos + // report encoding the prompt, each denoising step as it starts, from step + // 0, then decoding and assembling the frames + phase: 'encoding' | 'denoising' | 'decoding' | 'assembling' + step: number + totalSteps: number +} + export interface CoreAIImageGenerationOptions { n?: number size?: `${number}x${number}` @@ -203,6 +212,8 @@ export interface CoreAIImageGenerationOptions { // How many tokens of the input image FLUX.2 attends to: more follows the // image more closely and runs slower. Defaults to the finest grid the export has referenceGrid?: 'full' | 'half' | 'quarter' + // Called after each denoising step + onProgress?: (progress: CoreAIGenerationProgress) => void } export interface CoreAIImageGenerationResult { @@ -369,6 +380,8 @@ export interface CoreAIVideoGenerationOptions { vaeTileSize?: number // Loads each component only while it runs, true by default lazyModelLoading?: boolean + // Called as each phase and denoising step starts + onProgress?: (progress: CoreAIGenerationProgress) => void } export interface CoreAIVideoGenerationResult { diff --git a/website/src/content/docs/docs/core-ai/images-and-video.md b/website/src/content/docs/docs/core-ai/images-and-video.md index 05a14d90..686072be 100644 --- a/website/src/content/docs/docs/core-ai/images-and-video.md +++ b/website/src/content/docs/docs/core-ai/images-and-video.md @@ -52,6 +52,31 @@ const { image } = await generateImage({ - One image is edited per call, and there's no inpainting: extra images and a `mask` are left out with a warning. - `coreAI.imageModel(config).generate(prompt, { imageUri })` (or `imageBase64`) edits outside the AI SDK. +## Progress + +Generation takes seconds per image and minutes per video. Pass `onProgress` in `providerOptions['core-ai']` to follow it. The callback gets `{ phase, step, totalSteps }`: + +```typescript +import type { CoreAIGenerationProgress } from '@react-native-ai/core-ai' + +const onProgress = ({ phase, step, totalSteps }: CoreAIGenerationProgress) => + setStatus(`${phase} ${step}/${totalSteps}`) + +const { video } = await generateVideo({ + model: coreAI.videoModel({ id: 'wan2.1-t2v-1.3b', source: wanDirectory }), + prompt: 'a red ball rolling across a wooden table', + providerOptions: { + // providerOptions is typed as JSON, so a callback needs a cast + 'core-ai': { onProgress: onProgress as any }, + }, +}) +``` + +- Images report `denoising` after each step, from step 1. +- Videos report `encoding` as the prompt is encoded, `denoising` as each step starts (from step 0), then `decoding` and `assembling` the frames. +- The `generate(prompt, { onProgress })` methods of `coreAI.imageModel(...)` and `coreAI.videoModel(...)` take the callback without a cast. +- An `abortSignal` stops image and video generation after the current step. + ## Video Generation Pass `coreAI.videoModel(config)` to `experimental_generateVideo` to get a short MP4 clip.