Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 65 additions & 15 deletions packages/core-ai/ios/CoreAIImpl.swift
Original file line number Diff line number Diff line change
Expand Up @@ -686,19 +686,29 @@ public class CoreAIImpl: NSObject {
let sourceURL = try resolveSourceURL(config)
Task {
do {
let startingImage = try Self.startingImage(from: options)
let pipeline = try await self.makeDiffusionPipeline(at: sourceURL, options: options)
let configuration = try self.makePipelineConfiguration(prompt: prompt, pipeline: pipeline, options: options)
// Exports without a VAE encoder generate from the prompt alone,
// which the metadata reports so JS can warn
let editsImage = startingImage != nil && pipeline.supportsImageToImage
let configuration = try self.makePipelineConfiguration(
prompt: prompt,
pipeline: pipeline,
options: options,
startingImage: editsImage ? startingImage : nil
)
let result = try await pipeline.generateImages(configuration: configuration) { _ in true }
let images = try result.images.map(Self.pngBase64)
resolve([
"images": images,
"metadata": [
"count": images.count,
"imageFormat": "image/png",
"width": result.images.first?.width ?? 0,
"height": result.images.first?.height ?? 0
]
])
var metadata: [String: Any] = [
"count": images.count,
"imageFormat": "image/png",
"width": result.images.first?.width ?? 0,
"height": result.images.first?.height ?? 0
]
if startingImage != nil {
metadata["imageToImage"] = editsImage
}
resolve(["images": images, "metadata": metadata])
} catch {
reject(errorCode(error), error.localizedDescription, error)
}
Expand Down Expand Up @@ -1279,7 +1289,8 @@ extension CoreAIImpl {
private func makePipelineConfiguration(
prompt: String,
pipeline: FlowTransformerPipeline,
options: [String: Any]?
options: [String: Any]?,
startingImage: CGImage? = nil
) throws -> PipelineConfiguration {
let defaults = pipeline.config
let scheduler = SchedulerType(rawValue: options?["schedulerType"] as? String ?? "") ?? .discreteFlow
Expand All @@ -1295,14 +1306,47 @@ extension CoreAIImpl {
stepCount: stepCount,
guidanceScale: guidanceScale,
schedulerType: scheduler,
startingImage: startingImage,
strength: Float(options?["strength"] as? Double ?? 1.0),
referenceGrid: try Self.referenceGrid(from: options?["referenceGrid"], pipeline: pipeline),
encoderScaleFactor: defaults.encoderScaleFactor ?? DiffusionDefaults.Image.scaleFactor,
decoderScaleFactor: defaults.decoderScaleFactor ?? DiffusionDefaults.Image.scaleFactor,
decoderShiftFactor: defaults.decoderShiftFactor ?? DiffusionDefaults.Image.decoderShiftFactor,
decodeResolution: decodeResolution,
lazyModelLoading: options?["lazyModelLoading"] as? Bool ?? true
)
}

/// The requested reference grid, or the finest one the export has. iOS
/// exports carry the half grid only
private static func referenceGrid(
from option: Any?,
pipeline: FlowTransformerPipeline
) throws -> ReferenceGrid {
if let name = option as? String {
guard let grid = ReferenceGrid(rawValue: name) else {
throw CoreAIError.invalidInput("referenceGrid must be full, half or quarter, got \(name).")
}
return grid
}
return [.full, .half, .quarter].first { pipeline.img2imgRoutes[$0] != nil } ?? .full
}

/// The image to edit, decoded with its EXIF orientation applied. The
/// pipeline resizes it to the model's size
private static func startingImage(from options: [String: Any]?) throws -> CGImage? {
if let base64 = options?["imageBase64"] as? String {
guard let data = Data(base64Encoded: base64),
let source = CGImageSourceCreateWithData(data as CFData, nil) else {
throw CoreAIError.invalidInput("Could not decode imageBase64.")
}
return try decodeCGImage(source, description: "imageBase64")
}
if let imageUri = options?["imageUri"] as? String {
return try loadCGImage(from: ["imageUri": imageUri])
}
return nil
}
}
#endif

Expand Down Expand Up @@ -1804,11 +1848,17 @@ extension CoreAIImpl {
url = URL(fileURLWithPath: imageUri)
}

guard let source = CGImageSourceCreateWithURL(url as CFURL, nil),
let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any],
guard let source = CGImageSourceCreateWithURL(url as CFURL, nil) else {
throw CoreAIError.invalidInput("Could not load image at \(url.path).")
}
return try decodeCGImage(source, description: "image at \(url.path)")
}

static func decodeCGImage(_ source: CGImageSource, description: String) throws -> CGImage {
guard let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any],
let width = properties[kCGImagePropertyPixelWidth] as? Int,
let height = properties[kCGImagePropertyPixelHeight] as? Int else {
throw CoreAIError.invalidInput("Could not load image at \(url.path).")
throw CoreAIError.invalidInput("Could not load \(description).")
}
// Decode with the EXIF orientation applied, so pixel coordinates match
// the image as it is displayed
Expand All @@ -1818,7 +1868,7 @@ extension CoreAIImpl {
kCGImageSourceThumbnailMaxPixelSize: max(width, height)
]
guard let image = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) else {
throw CoreAIError.invalidInput("Could not load image at \(url.path).")
throw CoreAIError.invalidInput("Could not load \(description).")
}
return image
}
Expand Down
111 changes: 111 additions & 0 deletions packages/core-ai/src/__tests__/ai-sdk.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,24 @@ const generateVideo = mock(
seed: 1,
})
)
const generateImage = mock(
async (
_config: Record<string, unknown>,
_prompt: string,
options: Record<string, unknown>
): Promise<Record<string, unknown>> => ({
images: ['AAAA'],
metadata: {
count: 1,
imageFormat: 'image/png',
width: 512,
height: 512,
...(options.imageUri || options.imageBase64
? { imageToImage: true }
: {}),
},
})
)
const embed = mock(
async (
_config: Record<string, unknown>,
Expand Down Expand Up @@ -122,6 +140,7 @@ mock.module('react-native', () => ({
streamText,
cancelStream,
generateVideo,
generateImage,
embed,
runTask,
createLanguageSession,
Expand Down Expand Up @@ -1203,3 +1222,95 @@ describe('core ai video generation', () => {
)
})
})

describe('core ai image editing', () => {
const imageModel = () => coreAI.imageModel({ id: 'flux', source: '/flux' })

const callOptions = (overrides: object = {}) => ({
prompt: 'make it blue',
n: 1,
size: undefined,
aspectRatio: undefined,
seed: undefined,
files: undefined,
mask: undefined,
providerOptions: {},
...overrides,
})

const lastOptions = () => generateImage.mock.calls.at(-1)![2]

test('passes the first image as base64', async () => {
const { warnings } = await imageModel().doGenerate(
callOptions({
files: [
{
type: 'file',
mediaType: 'image/png',
data: new Uint8Array([1, 2, 3]),
},
],
providerOptions: { 'core-ai': { referenceGrid: 'half' } },
})
)

expect(lastOptions()).toMatchObject({
imageBase64: 'AQID',
referenceGrid: 'half',
})
expect(warnings).toEqual([])
})

test('passes data urls as base64 and local urls as uris', async () => {
await imageModel().doGenerate(
callOptions({
files: [{ type: 'url', url: 'data:image/png;base64,AQID' }],
})
)
expect(lastOptions()).toMatchObject({ imageBase64: 'AQID' })

await imageModel().doGenerate(
callOptions({ files: [{ type: 'url', url: 'file:///tmp/a.png' }] })
)
expect(lastOptions()).toMatchObject({ imageUri: 'file:///tmp/a.png' })
})

test('warns about extra images and masks', async () => {
const file = { type: 'file', mediaType: 'image/png', data: 'AQID' }
const { warnings } = await imageModel().doGenerate(
callOptions({ files: [file, file], mask: file })
)

expect(warnings).toEqual([
expect.objectContaining({ type: 'unsupported', feature: 'files' }),
expect.objectContaining({ type: 'unsupported', feature: 'mask' }),
])
})

test('warns when the export cannot edit images', async () => {
generateImage.mockImplementationOnce(async () => ({
images: ['AAAA'],
metadata: {
count: 1,
imageFormat: 'image/png',
width: 512,
height: 512,
imageToImage: false,
},
}))

const { warnings } = await imageModel().doGenerate(
callOptions({
files: [{ type: 'file', mediaType: 'image/png', data: 'AQID' }],
})
)

expect(warnings).toEqual([
expect.objectContaining({
type: 'unsupported',
feature: 'files',
details: expect.stringContaining('no image-to-image components'),
}),
])
})
})
57 changes: 56 additions & 1 deletion packages/core-ai/src/ai-sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type {
Experimental_VideoModelV4Result as VideoModelV4Result,
ImageModelV4,
ImageModelV4CallOptions,
ImageModelV4File,
LanguageModelV4,
LanguageModelV4CallOptions,
LanguageModelV4Content,
Expand Down Expand Up @@ -456,12 +457,16 @@ export class CoreAIImageGenerationModel implements ImageModelV4 {
size: options.size,
aspectRatio: options.aspectRatio,
seed: options.seed,
...(options.files?.[0] ? await toImageInput(options.files[0]) : {}),
...(options.providerOptions?.['core-ai'] ?? {}),
})

return {
images: result.images,
warnings: getImageWarnings(options, result.metadata),
warnings: [
...getImageInputWarnings(options, result.metadata),
...getImageWarnings(options, result.metadata),
],
response: {
timestamp: new Date(),
modelId: this.modelId,
Expand Down Expand Up @@ -781,6 +786,56 @@ export class CoreAITranscriptionModel implements TranscriptionModelV4 {
}
}

// The image to edit, as base64 data or a local file. Remote images are
// downloaded first, since native reads local files only
async function toImageInput(
file: ImageModelV4File
): Promise<Pick<CoreAIImageGenerationOptions, 'imageUri' | 'imageBase64'>> {
if (file.type === 'file') {
return { imageBase64: toBase64(file.data) }
}
if (file.url.startsWith('data:')) {
return { imageBase64: file.url.slice(file.url.indexOf(',') + 1) }
}
if (/^https?:/.test(file.url)) {
const response = await fetch(file.url)
const data = new Uint8Array(await response.arrayBuffer())
return { imageBase64: toBase64(data) }
}
return { imageUri: file.url }
}

// Core AI edits one image at a time and has no inpainting
function getImageInputWarnings(
options: ImageModelV4CallOptions,
metadata: CoreAIImageGenerationResult['metadata']
): SharedV4Warning[] {
const warnings: SharedV4Warning[] = []
if (metadata?.imageToImage === false) {
warnings.push({
type: 'unsupported',
feature: 'files',
details:
'This export has no image-to-image components, so the image was generated from the prompt alone.',
})
} else if (options.files && options.files.length > 1) {
warnings.push({
type: 'unsupported',
feature: 'files',
details: 'Core AI image models edit the first image only.',
})
}
if (options.mask) {
warnings.push({
type: 'unsupported',
feature: 'mask',
details:
'Core AI image models have no inpainting, so the mask was left out.',
})
}
return warnings
}

// Exported models generate one fixed size, which the metadata reports
function getImageWarnings(
options: ImageModelV4CallOptions,
Expand Down
10 changes: 10 additions & 0 deletions packages/core-ai/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,13 @@ export interface CoreAIImageGenerationOptions {
decodeResolution?: 'auto' | 'full' | 'half' | 'tiled'
strength?: number
lazyModelLoading?: boolean
// An image to edit, as a local file path or `file://` URI, or as base64
// data. Needs an export with image-to-image components
imageUri?: string
imageBase64?: string
// How many tokens of the input image FLUX.2 attends to: more follows the
// image more closely and runs slower. Defaults to the finest grid the export has
referenceGrid?: 'full' | 'half' | 'quarter'
}

export interface CoreAIImageGenerationResult {
Expand All @@ -205,6 +212,9 @@ export interface CoreAIImageGenerationResult {
imageFormat: 'image/png'
width: number
height: number
// False when an input image was given but the export can't edit images,
// so the result was generated from the prompt alone
imageToImage?: boolean
}
}

Expand Down
19 changes: 19 additions & 0 deletions website/src/content/docs/docs/core-ai/images-and-video.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,25 @@ const { image } = await generateImage({
- `providerOptions['core-ai']` tunes the pipeline: `stepCount`, `guidanceScale`, `negativePrompt`, `schedulerType`, `decodeResolution` (`auto`, `full`, `half` or `tiled`) and `strength`.
- The pipeline loads one part at a time, so a 4 GB model fits in an iPhone app's memory. `lazyModelLoading: false` loads it all at once.

## Image Editing

Pass an image with the prompt to edit it. FLUX.2 starts from noise and attends to the image's tokens, so the result keeps its layout and follows the prompt:

```typescript
const { image } = await generateImage({
model,
prompt: { images: [photo], text: 'make the apple green' },
seed: 1,
providerOptions: { 'core-ai': { stepCount: 4 } },
})
```

- `photo` is a `Uint8Array`, base64 string, `file://` URL or `https://` URL, which is downloaded first. Images are decoded with their EXIF orientation and resized to the model's size.
- The export needs a VAE encoder and an image-to-image transformer: `vae_encoder_half` and `transformer_512_img2img_half` in the iOS recipe. Without them the image is left out with a warning, and the result comes from the prompt alone.
- `providerOptions['core-ai'].referenceGrid` (`full`, `half` or `quarter`) sets how many of the image's tokens the model attends to: more follows the image more closely and runs slower. It defaults to the finest grid the export has. `strength` has no effect on FLUX.2.
- One image is edited per call, and there's no inpainting: extra images and a `mask` are left out with a warning.
- `coreAI.imageModel(config).generate(prompt, { imageUri })` (or `imageBase64`) edits outside the AI SDK.

## Video Generation

Pass `coreAI.videoModel(config)` to `experimental_generateVideo` to get a short MP4 clip.
Expand Down
Loading