Repository navigation
feat(core-ai): report progress for image and video generation #373
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: feat/core-ai-image-to-image
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -237,15 +237,32 @@ | |
| options:(nonnull NSDictionary *)options | ||
| resolve:(nonnull RCTPromiseResolveBlock)resolve | ||
| reject:(nonnull RCTPromiseRejectBlock)reject { | ||
| [_coreAI generateVideo:requestId config:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; | ||
| [_coreAI generateVideo:requestId | ||
| config:CoreAIConfigToDictionary(config) | ||
| prompt:prompt | ||
| options:options | ||
| onProgress:^(NSDictionary *progress) { | ||
| [self emitOnGenerationProgress:progress]; | ||
| } | ||
| resolve:resolve | ||
| reject:reject]; | ||
| } | ||
|
|
||
| - (void)generateImage:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config | ||
| - (void)generateImage:(nonnull NSString *)requestId | ||
| config:(JS::NativeCoreAI::NativeCoreAIModelConfig &)config | ||
| prompt:(nonnull NSString *)prompt | ||
| options:(nonnull NSDictionary *)options | ||
| resolve:(nonnull RCTPromiseResolveBlock)resolve | ||
| reject:(nonnull RCTPromiseRejectBlock)reject { | ||
| [_coreAI generateImage:CoreAIConfigToDictionary(config) prompt:prompt options:options resolve:resolve reject:reject]; | ||
| [_coreAI generateImage:requestId | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · Native progress block is invoked with no JS subscriber, which throws on the generation thread The ObjC bridge now passes an unconditional Suggested change Only emit progress when someone is listening: either subscribe unconditionally in Details
The same line already exists for Also raised hereAlso raised at this location by swift-ios: requestId injection for video progress keeps the wrong side of the merge — persona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. The codegen EventEmitter (new architecture) is a no-op when nothing is subscribed, the same as the existing stream events. It isn't the old 🤖 Reply from Claude Code |
||
| config:CoreAIConfigToDictionary(config) | ||
| prompt:prompt | ||
| options:options | ||
| onProgress:^(NSDictionary *progress) { | ||
| [self emitOnGenerationProgress:progress]; | ||
| } | ||
| resolve:resolve | ||
| reject:reject]; | ||
|
Check notice on line 265 in packages/core-ai/ios/CoreAI.mm
|
||
| } | ||
|
|
||
| - (void)runTask:(nonnull NSString *)task | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -649,6 +649,7 @@ | |
| config: [String: Any], | ||
| prompt: String, | ||
| options: [String: Any]?, | ||
| onProgress: @escaping @Sendable ([String: Any]) -> Void, | ||
| resolve: @escaping (Any?) -> Void, | ||
| reject: @escaping (String, String, Error?) -> Void | ||
| ) { | ||
|
|
@@ -659,7 +660,9 @@ | |
| startTask(requestId, onError: { _, code, message in | ||
| reject(code, message, nil) | ||
| }) { | ||
| resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:])) | ||
| resolve(try await VideoGeneration.generate(prompt: prompt, sourceURL: sourceURL, options: options ?? [:]) { | ||
| onProgress($0.merging(["requestId": requestId]) { current, _ in current }) | ||
| }) | ||
| } | ||
| } catch { | ||
| reject(errorCode(error), error.localizedDescription, error) | ||
|
|
@@ -672,46 +675,59 @@ | |
| #endif | ||
| } | ||
|
|
||
| /// Generates images, reporting each denoising step. cancelStream(requestId) | ||
| /// stops it after the current step | ||
| @objc | ||
| public func generateImage( | ||
| _ config: [String: Any], | ||
| _ requestId: String, | ||
| config: [String: Any], | ||
| prompt: String, | ||
| options: [String: Any]?, | ||
| onProgress: @escaping @Sendable ([String: Any]) -> Void, | ||
| resolve: @escaping (Any?) -> Void, | ||
| reject: @escaping (String, String, Error?) -> Void | ||
| ) { | ||
| #if canImport(CoreAIDiffusionPipeline) | ||
| if #available(iOS 27, macOS 27, *) { | ||
| do { | ||
| let sourceURL = try resolveSourceURL(config) | ||
| Task { | ||
| do { | ||
| let startingImage = try Self.startingImage(from: options) | ||
| let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) | ||
| // Exports without a VAE encoder generate from the prompt alone, | ||
| // which the metadata reports so JS can warn | ||
| let editsImage = startingImage != nil && pipeline.supportsImageToImage | ||
| let configuration = try self.makePipelineConfiguration( | ||
| prompt: prompt, | ||
| pipeline: pipeline, | ||
| options: options, | ||
| startingImage: editsImage ? startingImage : nil | ||
| ) | ||
| let result = try await pipeline.generateImages(configuration: configuration) { _ in true } | ||
| let images = try result.images.map(Self.pngBase64) | ||
| var metadata: [String: Any] = [ | ||
| "count": images.count, | ||
| "imageFormat": "image/png", | ||
| "width": result.images.first?.width ?? 0, | ||
| "height": result.images.first?.height ?? 0 | ||
| ] | ||
| if startingImage != nil { | ||
| metadata["imageToImage"] = editsImage | ||
| } | ||
| resolve(["images": images, "metadata": metadata]) | ||
| } catch { | ||
| reject(errorCode(error), error.localizedDescription, error) | ||
| startTask(requestId, onError: { _, code, message in | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🟡 Minor · Native image errors now go through the language-model error mapping and lose the underlying Error The image path moved from its own Suggested change Preserve the previous error surface for image generation: reject with errorCode(error) and the original Error, mapping only Task cancellation to CoreAIError.cancelled, rather than reusing the language-model error mapper via startTask. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. 🤖 Reply from Claude Code |
||
| reject(code, message, nil) | ||
| }) { | ||
|
Check warning on line 696 in packages/core-ai/ios/CoreAIImpl.swift
|
||
| let startingImage = try Self.startingImage(from: options) | ||
| let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options) | ||
| // Exports without a VAE encoder generate from the prompt alone, | ||
| // which the metadata reports so JS can warn | ||
| let editsImage = startingImage != nil && pipeline.supportsImageToImage | ||
| let configuration = try self.makePipelineConfiguration( | ||
| prompt: prompt, | ||
| pipeline: pipeline, | ||
| options: options, | ||
| startingImage: editsImage ? startingImage : nil | ||
| ) | ||
| let result = try await pipeline.generateImages(configuration: configuration) { progress in | ||
| // Steps after a cancel aren't reported, since generation stops | ||
| guard !Task.isCancelled else { return false } | ||
| onProgress([ | ||
| "requestId": requestId, | ||
| "phase": "denoising", | ||
| "step": progress.step, | ||
| "totalSteps": progress.totalSteps | ||
| ]) | ||
| return true | ||
| } | ||
| try Task.checkCancellation() | ||
| let images = try result.images.map(Self.pngBase64) | ||
| var metadata: [String: Any] = [ | ||
| "count": images.count, | ||
| "imageFormat": "image/png", | ||
| "width": result.images.first?.width ?? 0, | ||
| "height": result.images.first?.height ?? 0 | ||
| ] | ||
| if startingImage != nil { | ||
| metadata["imageToImage"] = editsImage | ||
| } | ||
| resolve(["images": images, "metadata": metadata]) | ||
| } | ||
| } catch { | ||
| reject(errorCode(error), error.localizedDescription, error) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,11 +15,23 @@ | |
| // only while it runs | ||
| nonisolated(unsafe) private static var loaded: (key: String, load: Task<(WanPipeline, ExportedShape), Error>)? | ||
|
|
||
| static func generate(prompt: String, sourceURL: URL, options: [String: Any]) async throws -> [String: Any] { | ||
| static func generate( | ||
|
grabbou marked this conversation as resolved.
Resolved
|
||
| prompt: String, | ||
| sourceURL: URL, | ||
| options: [String: Any], | ||
| onProgress: @escaping @Sendable ([String: Any]) -> Void | ||
| ) async throws -> [String: Any] { | ||
| let (pipeline, shape) = try await self.pipeline(at: sourceURL, options: options) | ||
| let configuration = try makeConfiguration(prompt: prompt, pipeline: pipeline, shape: shape, options: options) | ||
| let result = try await pipeline.generateVideo(configuration: configuration) { _ in | ||
| !Task.isCancelled | ||
| let result = try await pipeline.generateVideo(configuration: configuration) { progress in | ||
| // Phases after a cancel aren't reported, since generation stops | ||
| guard !Task.isCancelled else { return false } | ||
| onProgress([ | ||
| "phase": phaseName(progress.phase), | ||
| "step": progress.step, | ||
| "totalSteps": progress.totalSteps | ||
| ]) | ||
| return true | ||
| } | ||
| try Task.checkCancellation() | ||
|
|
||
|
|
@@ -41,6 +53,15 @@ | |
| ] | ||
| } | ||
|
|
||
| private static func phaseName(_ phase: VideoProgress.Phase) -> String { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🆕 🔵 Nit · Phase names duplicated between Swift and the TS union with nothing linking them
Suggested change Keep one list of phase names: at minimum name the Swift strings via a shared constant/enum comment pointing at CoreAIGenerationProgress, so the two sides are changed together. Detailspersona: Reply Useful? Reply
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Leaving this as is. Swift's 🤖 Reply from Claude Code |
||
| switch phase { | ||
| case .encoding: return "encoding" | ||
| case .denoising: return "denoising" | ||
| case .decoding: return "decoding" | ||
| case .assembling: return "assembling" | ||
| } | ||
| } | ||
|
|
||
|
Check notice on line 64 in packages/core-ai/ios/CoreAIVideoGeneration.swift
|
||
| // Concurrent calls for the same bundle share one load | ||
| private static func pipeline(at url: URL, options: [String: Any]) async throws -> (WanPipeline, ExportedShape) { | ||
| let lazy = options["lazyModelLoading"] as? Bool ?? true | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.