From 175c0bae5a62a0f92d24679efc474f2829999064 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:32:26 +0100 Subject: [PATCH 01/48] test: a lease's copy swaps every recorded foreign folder for its own (#1570) --- src/auth/admin-secret.test.ts | 12 +++++ src/drivers/ios/foreign-folders.ts | 10 ++++ src/drivers/ios/index.ts | 6 ++- src/drivers/ios/source-data.ts | 52 ++++++++++++++++--- src/drivers/ios/source-swap.test.ts | 79 +++++++++++++++++++++++++++++ src/drivers/ios/testing.ts | 5 +- src/ports/filesystem.ts | 36 +++++++++++++ 7 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 src/drivers/ios/foreign-folders.ts create mode 100644 src/drivers/ios/source-swap.test.ts diff --git a/src/auth/admin-secret.test.ts b/src/auth/admin-secret.test.ts index 85dd1effc..678a752b2 100644 --- a/src/auth/admin-secret.test.ts +++ b/src/auth/admin-secret.test.ts @@ -78,6 +78,12 @@ describe("AdminSecretManager", () => { copyTree: () => { throw new Error("verify() must not touch the filesystem"); }, + readlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, + replaceSymlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, mkdirp: () => { throw new Error("verify() must not touch the filesystem"); }, @@ -165,6 +171,12 @@ describe("AdminSecretManager", () => { copyTree: () => { throw new Error("verify() must not touch the filesystem"); }, + readlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, + replaceSymlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, mkdirp: () => { throw new Error("must not touch the filesystem"); }, diff --git a/src/drivers/ios/foreign-folders.ts b/src/drivers/ios/foreign-folders.ts new file mode 100644 index 000000000..3639c5970 --- /dev/null +++ b/src/drivers/ios/foreign-folders.ts @@ -0,0 +1,10 @@ +export interface ForeignFolders { + readonly deviceSet: string; + readonly sourcesSet: string; + readonly logsRoot: string; + readonly ownUdid: string; +} + +export function swapForeignFolders(bytes: Uint8Array, _folders: ForeignFolders): Uint8Array { + return bytes; +} diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 78148a415..cec90e1d3 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -249,6 +249,8 @@ export interface IosSimctlDriverOptions { /** why: the daemon's pid is named in the staging directory of a new root. */ readonly pid?: number; readonly coreSimulatorRoot?: string; + /** why: `/Library/Logs/CoreSimulator`: a simulator's log folder is named in its files beside its device folder, and a copy's swap covers both (ADR 0076 §3). */ + readonly logsRoot?: string; // why: where launchd_sim keeps each simulator's overrides folder; a test points it elsewhere. readonly launchdOverridesRoot?: string; readonly onSlimmed?: (fact: SlimmedFact) => void; @@ -888,7 +890,7 @@ export class IosSimctlDriver implements Driver { await this.#copyOverrides(data.udid, cloneUdid); await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes); + this.#sources.markReady(sourceId, sizeBytes, []); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; } catch (error: unknown) { if (cloneUdid !== undefined && cloneUdid !== "") { @@ -1079,7 +1081,7 @@ export class IosSimctlDriver implements Driver { await this.#deleteSeedSimulator(udid); throw error; } - this.#sources.markReady(sourceId, await this.#sizeOf(udid)); + this.#sources.markReady(sourceId, await this.#sizeOf(udid), []); return { sourceId, setPath: this.#sourcesSet, udid, runtimeBuild: resolved.runtime.build }; } catch (error: unknown) { this.#sources.deleteRow(sourceId); diff --git a/src/drivers/ios/source-data.ts b/src/drivers/ios/source-data.ts index 2fafffdd5..c6c183f25 100644 --- a/src/drivers/ios/source-data.ts +++ b/src/drivers/ios/source-data.ts @@ -1,4 +1,4 @@ -import type { SqliteDatabase, SqliteSchemaOwner } from "../../ports/index.js"; +import { inTransaction, type SqliteDatabase, type SqliteSchemaOwner } from "../../ports/index.js"; export const iosSourceDataOwner: SqliteSchemaOwner = { migrations: [ @@ -28,11 +28,25 @@ export const iosSourceDataOwner: SqliteSchemaOwner = { OR (newer.created_at = ios_source_data.created_at AND newer.source_id > ios_source_data.source_id))); CREATE UNIQUE INDEX ios_source_data_ready_seed ON ios_source_data (spec_key) WHERE kind = 'seed' AND state = 'ready';`, + // why: ADR 0076 §2: a source records every file and link that names a device or log folder (`indexed` tells a source that names none from one made before this step); a copy swaps only those. + `ALTER TABLE ios_source_data ADD COLUMN indexed INTEGER NOT NULL DEFAULT 0; + CREATE TABLE ios_source_files ( + source_id TEXT NOT NULL, + relative_path TEXT NOT NULL CHECK (length(CAST(relative_path AS BLOB)) <= 4096), + kind TEXT NOT NULL CHECK (kind IN ('link', 'file')), + PRIMARY KEY (source_id, relative_path) + ) STRICT`, ], name: "ios-source-data", - version: 2, + version: 3, }; +/** why: a file or link of a source that names a device or log folder, its path under the source's own UDID folder, `/`-separated. */ +export interface SourceFile { + readonly relativePath: string; + readonly kind: "link" | "file"; +} + export interface NewSourceRow { readonly sourceId: string; readonly kind: "seed" | "baseline"; @@ -105,14 +119,38 @@ export class IosSourceTable { .run(udid, sourceId); } - markReady(sourceId: string, sizeBytes: number | undefined): void { - this.#database - .prepare("UPDATE ios_source_data SET state = 'ready', size_bytes = ? WHERE source_id = ?") - .run(sizeBytes ?? null, sourceId); + /** why: the row turns `ready`, indexed, and holds its files in one step, so a reader never sees a ready source with half its record. */ + markReady(sourceId: string, sizeBytes: number | undefined, files: readonly SourceFile[]): void { + inTransaction(this.#database, () => { + this.#database + .prepare( + "UPDATE ios_source_data SET state = 'ready', indexed = 1, size_bytes = ? WHERE source_id = ?", + ) + .run(sizeBytes ?? null, sourceId); + const insert = this.#database.prepare( + "INSERT INTO ios_source_files (source_id, relative_path, kind) VALUES (?, ?, ?)", + ); + for (const file of files) insert.run(sourceId, file.relativePath, file.kind); + }); + } + + filesOf(sourceId: string): readonly SourceFile[] { + return this.#database + .prepare( + "SELECT relative_path, kind FROM ios_source_files WHERE source_id = ? ORDER BY relative_path", + ) + .all(sourceId) + .map((row) => ({ + relativePath: row["relative_path"] as string, + kind: row["kind"] as "link" | "file", + })); } deleteRow(sourceId: string): void { - this.#database.prepare("DELETE FROM ios_source_data WHERE source_id = ?").run(sourceId); + inTransaction(this.#database, () => { + this.#database.prepare("DELETE FROM ios_source_files WHERE source_id = ?").run(sourceId); + this.#database.prepare("DELETE FROM ios_source_data WHERE source_id = ?").run(sourceId); + }); } findReadySeed(specKey: string): ReadySeed | undefined { diff --git a/src/drivers/ios/source-swap.test.ts b/src/drivers/ios/source-swap.test.ts new file mode 100644 index 000000000..a1f3ca42e --- /dev/null +++ b/src/drivers/ios/source-swap.test.ts @@ -0,0 +1,79 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { memorySourceDatabase, plantReadySeed } from "./testing.js"; +import { + deviceRoot, + driverData, + instanceId, + plantManagedDevice, + simctl, + sourcesRoot, +} from "./test-slim.js"; + +const listFixture = readFileSync(new URL("./fixtures/simctl-list.json", import.meta.url), "utf8"); +const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; +const logsRoot = "/home/Library/Logs/CoreSimulator"; +const seedUdid = "00000000-0000-0000-0000-0000000000AA"; +const copyUdid = driverData.udid; + +async function provisionCopy( + setup: (filesystem: MemoryFilesystem) => Promise, + files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], +): Promise { + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await setup(filesystem); + const database = plantReadySeed(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + files, + runtimeId: driverData.runtimeId, + udid: seedUdid, + }); + const runner = new ScriptedProcessRunner([ + { + match: simctl("list", "-j", "devicetypes", "runtimes"), + result: { code: 0, stderr: "", stdout: listFixture }, + }, + { + match: { + args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, deviceRoot], + command: "xcrun", + }, + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + ]); + const driver = await IosSimctlDriver.create({ + clock: new FakeClock(), + database, + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + logsRoot, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + await driver.resolveSpec(spec); + await driver.provision(spec); + return filesystem; +} + +describe("a lease's copy before its first boot", () => { + it("swaps every recorded foreign folder for its own, length unchanged", async () => { + const link = `${deviceRoot}/${copyUdid}/data/Library/Logs`; + + const filesystem = await provisionCopy( + async (fs) => { + await fs.mkdirp(`${deviceRoot}/${copyUdid}/data/Library`); + fs.defineSymlink(link, `${logsRoot}/${seedUdid}`); + }, + [{ kind: "link", relativePath: "data/Library/Logs" }], + ); + + expect(await filesystem.readlink(link)).toBe(`${logsRoot}/${copyUdid}`); + }); +}); diff --git a/src/drivers/ios/testing.ts b/src/drivers/ios/testing.ts index 025ce7256..375d5851f 100644 --- a/src/drivers/ios/testing.ts +++ b/src/drivers/ios/testing.ts @@ -1,5 +1,5 @@ import { NodeSqliteOpener, type SqliteDatabase } from "../../ports/index.js"; -import { iosSourceDataOwner, IosSourceTable } from "./source-data.js"; +import { iosSourceDataOwner, IosSourceTable, type SourceFile } from "./source-data.js"; export async function memorySourceDatabase(): Promise { const database = await new NodeSqliteOpener().open(":memory:", { readOnly: false }); @@ -9,6 +9,7 @@ export async function memorySourceDatabase(): Promise { export interface PlantedSource { readonly deviceTypeId: string; + readonly files?: readonly SourceFile[]; readonly kind?: "seed" | "baseline"; readonly maker?: string; readonly model?: string; @@ -39,7 +40,7 @@ export function plantSource(database: SqliteDatabase, source: PlantedSource): Sq specKey: `${source.deviceTypeId}\u0000${source.runtimeId}`, }); if (source.udid !== undefined) table.setUdid(sourceId, source.udid); - if (source.state !== "making") table.markReady(sourceId, source.sizeBytes); + if (source.state !== "making") table.markReady(sourceId, source.sizeBytes, source.files ?? []); return database; } diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index 835a32b7d..c1d953c28 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -9,10 +9,12 @@ import { open, readdir, readFile, + readlink, realpath, rename, rm, stat, + symlink, statfs, writeFile, } from "node:fs/promises"; @@ -121,6 +123,10 @@ export interface Filesystem { diskFree(path: string): Promise; diskSpace(path: string): Promise<{ readonly freeBytes: number; readonly totalBytes: number }>; directorySize(path: string): Promise; + /** why: what a symbolic link points at, without following it. */ + readlink(path: string): Promise; + /** why: the link at `path` points at `target` from one step to the next, with no moment when it is missing. */ + replaceSymlink(path: string, target: string): Promise; } // why: only the directories that are absent get the mode, so one that is already there keeps its own. @@ -406,6 +412,21 @@ export class NodeFilesystem implements Filesystem { } } + async readlink(path: string): Promise { + return readlink(path); + } + + async replaceSymlink(path: string, target: string): Promise { + const staged = `${path}.simlock-swap-${String(process.pid)}`; + await symlink(target, staged); + try { + await rename(staged, path); + } catch (error: unknown) { + await rm(staged, { force: true }); + throw error; + } + } + async directorySize(path: string): Promise { let total = 0; for (const entry of await readdir(path, { withFileTypes: true })) { @@ -774,6 +795,21 @@ export class MemoryFilesystem implements Filesystem { this.#failures.set(path, code); } + async readlink(path: string): Promise { + this.#failIfDefined(path); + const entry = this.#rawEntryAt(this.#underResolvedParent(path)); + if (entry.kind !== "symlink") throw errnoError("EINVAL", `Not a symbolic link: ${path}`); + return entry.target; + } + + async replaceSymlink(path: string, target: string): Promise { + this.#failIfDefined(path); + const at = this.#underResolvedParent(path); + const entry = this.#rawEntryAt(at); + if (entry.kind !== "symlink") throw errnoError("EINVAL", `Not a symbolic link: ${path}`); + this.#entries.set(at, { ...this.#newEntry({ kind: "symlink", target }), uid: entry.uid }); + } + defineSymlink(path: string, target: string): void { this.#entries.set(path, this.#newEntry({ kind: "symlink", target })); } From 6024065fa2c15fc73fbbedd743040c3b3da1fc4a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:33:13 +0100 Subject: [PATCH 02/48] feat(ios): a lease's copy has its links' foreign folders swapped for its own before the first boot (#1570) --- src/daemon/driver-discovery.ts | 1 + src/drivers/ios/foreign-folders.ts | 31 +++++++++++++++++++++++++++--- src/drivers/ios/index.ts | 30 +++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/daemon/driver-discovery.ts b/src/daemon/driver-discovery.ts index 5de91b527..4a3ce350e 100644 --- a/src/daemon/driver-discovery.ts +++ b/src/daemon/driver-discovery.ts @@ -197,6 +197,7 @@ async function discoverIosDriver( idGenerator: options.idGenerator, instanceId: options.instanceId, logger: logger.child("ios"), + logsRoot: `${homedir()}/Library/Logs/CoreSimulator`, onSilenceFailed: logSilenceFailure(logger), onWindowFailed: logWindowFailure(logger), onSlimmed: emitSlimDiagnostic(options.eventBus), diff --git a/src/drivers/ios/foreign-folders.ts b/src/drivers/ios/foreign-folders.ts index 3639c5970..88292add6 100644 --- a/src/drivers/ios/foreign-folders.ts +++ b/src/drivers/ios/foreign-folders.ts @@ -1,10 +1,35 @@ export interface ForeignFolders { readonly deviceSet: string; readonly sourcesSet: string; - readonly logsRoot: string; + /** why: absent when the daemon names no log folder; only device folders are then swapped. */ + readonly logsRoot?: string | undefined; readonly ownUdid: string; } -export function swapForeignFolders(bytes: Uint8Array, _folders: ForeignFolders): Uint8Array { - return bytes; +const UDID = "[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"; + +function pattern(roots: readonly string[]): RegExp { + const alternatives = roots.map((root) => root.split("/").map(escapeRegExp).join("/")); + return new RegExp(`(?:${alternatives.join("|")})/(${UDID})`, "g"); +} + +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * why: ADR 0076 §3: every device or log folder of another simulator becomes the copy's own, byte for byte, + * so a file's length never changes and the swap needs to understand no format. Bytes map one to one to + * `latin1` characters, so a binary file passes through untouched outside the paths. + */ +export function swapForeignFolders(bytes: Uint8Array, folders: ForeignFolders): Uint8Array { + const text = Buffer.from(bytes).toString("latin1"); + const devices = pattern([folders.deviceSet, folders.sourcesSet]); + const own = (root: string) => (match: string, udid: string) => + udid === folders.ownUdid ? match : `${root}/${folders.ownUdid}`; + let swapped = text.replace(devices, own(folders.deviceSet)); + if (folders.logsRoot !== undefined) { + swapped = swapped.replace(pattern([folders.logsRoot]), own(folders.logsRoot)); + } + return Buffer.from(swapped, "latin1"); } diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index cec90e1d3..6e483ac35 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -85,6 +85,7 @@ import { IosPhysicalHandler } from "./physical.js"; import { createIosDeviceInput } from "./input/index.js"; import { iosRuntimeVersionFromId } from "./runtime-version.js"; import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; +import { swapForeignFolders } from "./foreign-folders.js"; import { seedIsStale, sourceStaleReason } from "./source-staleness.js"; import { fitSourcesFolders, sourcesFolderOf, sourcesSetCheck } from "./sources-folder.js"; import { createIosScreenCapture } from "./screen-capture.js"; @@ -336,6 +337,7 @@ export class IosSimctlDriver implements Driver { readonly #sources: IosSourceTable; readonly #sourcesSet: string; readonly #ownsSourcesSet: (path: string) => boolean; + readonly #logsRoot: string | undefined; #sourceMaker: string | undefined; #swept = false; readonly #seedMakes = new Map>(); @@ -376,6 +378,7 @@ export class IosSimctlDriver implements Driver { this.#slimByDefault = options.slimByDefault === true; this.#sources = new IosSourceTable(options.database); this.#sourcesSet = sourcesSet; + this.#logsRoot = options.logsRoot; this.#ownsSourcesSet = sourcesSetCheck( sourcesSet, options.simlockHome, @@ -977,6 +980,7 @@ export class IosSimctlDriver implements Driver { try { await this.#writeMark(udid); await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); + await this.#swapForeignFolders(seed.sourceId, udid); } catch (error: unknown) { // why: a clone whose mark or database drop failed is not a device Simlock hands out, and // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. @@ -997,14 +1001,16 @@ export class IosSimctlDriver implements Driver { }; } - async #namedSource(sourceId: string): Promise<{ readonly udid: string }> { + async #namedSource( + sourceId: string, + ): Promise<{ readonly sourceId: string; readonly udid: string }> { const row = this.#sources.findRow(sourceId); if (row?.state !== "ready" || row.udid === undefined) { throw new DriverCrashError(`iOS keeps no ready source ${sourceId}`); } const reason = sourceStaleReason(row, await this.#loadCatalog(), this.#sourcesSet); if (reason !== undefined) throw new DriverCrashError(`source ${sourceId} is stale: ${reason}`); - return { udid: row.udid }; + return { sourceId, udid: row.udid }; } /** why: two provisions of one kind at once make one seed, the second waiting for the first's make. */ @@ -1149,6 +1155,26 @@ export class IosSimctlDriver implements Driver { } } + /** + * why: ADR 0076 §3: a copy's files and links still name its source's device and log folders, and + * the copy would write into them; before its first boot each recorded one names the copy's own + * (a swap of the same length, so no format is parsed). A copy whose swap fails is deleted by the caller. + */ + async #swapForeignFolders(sourceId: string, copyUdid: string): Promise { + const folders = { + deviceSet: this.#deviceRoot, + logsRoot: this.#logsRoot, + ownUdid: copyUdid, + sourcesSet: this.#sourcesSet, + }; + for (const file of this.#sources.filesOf(sourceId)) { + const path = join(this.#deviceRoot, copyUdid, file.relativePath); + const target = await this.#filesystem.readlink(path); + const swapped = Buffer.from(swapForeignFolders(Buffer.from(target), folders)).toString(); + if (swapped !== target) await this.#filesystem.replaceSymlink(path, swapped); + } + } + async #destroySeed(seed: ReadySeed): Promise { await this.#deleteSeedInTurn(seed.udid, seed.setPath); this.#sources.deleteRow(seed.sourceId); From d25612c52ab35bcd95701e2021e1f5421ae2ae48 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:34:06 +0100 Subject: [PATCH 03/48] test: the same-length swap keeps every spelling and separator it found (#1570) --- src/drivers/ios/foreign-folders.test.ts | 64 +++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/drivers/ios/foreign-folders.test.ts diff --git a/src/drivers/ios/foreign-folders.test.ts b/src/drivers/ios/foreign-folders.test.ts new file mode 100644 index 000000000..1d200c4e6 --- /dev/null +++ b/src/drivers/ios/foreign-folders.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { swapForeignFolders } from "./foreign-folders.js"; + +const folders = { + deviceSet: "/private/tmp/h/devices/ios", + logsRoot: "/Users/al/Library/Logs/CoreSimulator", + ownUdid: "11111111-1111-1111-1111-111111111111", + sourcesSet: "/private/tmp/h/devices/src", +}; +const seed = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; +const usedCopy = "BBBBBBBB-BBBB-BBBB-BBBB-BBBBBBBBBBBB"; +const own = folders.ownUdid; + +// why: the swap is a pure function with many byte-level cases (spellings, separators, binary neighbours) that every driver verb leans on; the driver's own test proves the files and links it is applied to. +describe("swapForeignFolders", () => { + it.each([ + { + name: "a seed's folder becomes the copy's own", + before: `x ${folders.sourcesSet}/${seed}/data y`, + after: `x ${folders.deviceSet}/${own}/data y`, + }, + { + name: "the folder a baseline was saved from becomes the copy's own", + before: `${folders.deviceSet}/${usedCopy}/data/Media`, + after: `${folders.deviceSet}/${own}/data/Media`, + }, + { + name: "a log folder becomes the copy's own log folder", + before: `${folders.logsRoot}/${seed}`, + after: `${folders.logsRoot}/${own}`, + }, + { + name: "the spelling without /private keeps that spelling", + before: `/tmp/h/devices/src/${seed}/x`, + after: `/tmp/h/devices/ios/${own}/x`, + }, + { + name: "a JSON path with escaped separators keeps them escaped", + before: `{"p":"\\/private\\/tmp\\/h\\/devices\\/src\\/${seed}\\/x"}`, + after: `{"p":"\\/private\\/tmp\\/h\\/devices\\/ios\\/${own}\\/x"}`, + }, + { + name: "a path already naming the copy is unchanged", + before: `${folders.deviceSet}/${own}/data ${folders.logsRoot}/${own}`, + after: `${folders.deviceSet}/${own}/data ${folders.logsRoot}/${own}`, + }, + { + name: "bytes around a path, binary ones included, are kept", + before: `\u0000ÿ\u0080${folders.sourcesSet}/${seed}\u0001þ`, + after: `\u0000ÿ\u0080${folders.deviceSet}/${own}\u0001þ`, + }, + { + name: "a path of some other folder is unchanged", + before: `/private/tmp/other/devices/src/${seed}`, + after: `/private/tmp/other/devices/src/${seed}`, + }, + ])("$name, the length unchanged", ({ before, after }) => { + const swapped = swapForeignFolders(Buffer.from(before, "latin1"), folders); + + expect(Buffer.from(swapped).toString("latin1")).toBe(after); + expect(swapped.length).toBe(before.length); + }); +}); From f53683ca5abf0fbadc2d40a7c7126243c382718e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:34:22 +0100 Subject: [PATCH 04/48] feat(ios): the swap keeps the /private spelling and escaped separators it found (#1570) --- src/drivers/ios/foreign-folders.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/drivers/ios/foreign-folders.ts b/src/drivers/ios/foreign-folders.ts index 88292add6..ac47e35b9 100644 --- a/src/drivers/ios/foreign-folders.ts +++ b/src/drivers/ios/foreign-folders.ts @@ -7,16 +7,37 @@ export interface ForeignFolders { } const UDID = "[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"; +const PRIVATE = "/private"; +const SEPARATOR = "(?:\\\\/|/)"; + +/** why: iOS writes a folder under `/private` in two spellings (`/private/tmp/x` and `/tmp/x`), and each is matched and kept. */ +function spellings(root: string): readonly string[] { + return root.startsWith(`${PRIVATE}/`) ? [root, root.slice(PRIVATE.length)] : [root]; +} function pattern(roots: readonly string[]): RegExp { - const alternatives = roots.map((root) => root.split("/").map(escapeRegExp).join("/")); - return new RegExp(`(?:${alternatives.join("|")})/(${UDID})`, "g"); + const alternatives = roots + .flatMap(spellings) + .sort((a, b) => b.length - a.length) + .map((root) => root.split("/").map(escapeRegExp).join(SEPARATOR)); + return new RegExp(`(?:${alternatives.join("|")})${SEPARATOR}(${UDID})`, "g"); } function escapeRegExp(text: string): string { return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } +/** why: the copy's own folder in the spelling and with the separators of the path it replaces. */ +function ownFolder(root: string, udid: string, found: string): string { + const escaped = found.includes("\\/"); + const plain = escaped ? found.replaceAll("\\/", "/") : found; + const spelling = + spellings(root).find( + (candidate) => !plain.startsWith(PRIVATE) === !candidate.startsWith(PRIVATE), + ) ?? root; + return `${spelling}/${udid}`.replaceAll("/", escaped ? "\\/" : "/"); +} + /** * why: ADR 0076 §3: every device or log folder of another simulator becomes the copy's own, byte for byte, * so a file's length never changes and the swap needs to understand no format. Bytes map one to one to @@ -26,7 +47,7 @@ export function swapForeignFolders(bytes: Uint8Array, folders: ForeignFolders): const text = Buffer.from(bytes).toString("latin1"); const devices = pattern([folders.deviceSet, folders.sourcesSet]); const own = (root: string) => (match: string, udid: string) => - udid === folders.ownUdid ? match : `${root}/${folders.ownUdid}`; + udid === folders.ownUdid ? match : ownFolder(root, folders.ownUdid, match); let swapped = text.replace(devices, own(folders.deviceSet)); if (folders.logsRoot !== undefined) { swapped = swapped.replace(pattern([folders.logsRoot]), own(folders.logsRoot)); From d3c5344aa44cf1d966cd4155049ffa6827a3ea51 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:35:32 +0100 Subject: [PATCH 05/48] test: the filesystem reads links and bytes and writes them back at their own length (#1570) --- src/auth/admin-secret.test.ts | 12 ++++++++ src/ports/filesystem.test.ts | 58 +++++++++++++++++++++++++++++++++++ src/ports/filesystem.ts | 16 ++++++++++ 3 files changed, 86 insertions(+) diff --git a/src/auth/admin-secret.test.ts b/src/auth/admin-secret.test.ts index 678a752b2..582b51482 100644 --- a/src/auth/admin-secret.test.ts +++ b/src/auth/admin-secret.test.ts @@ -84,6 +84,12 @@ describe("AdminSecretManager", () => { replaceSymlink: () => { throw new Error("verify() must not touch the filesystem"); }, + readBytes: () => { + throw new Error("verify() must not touch the filesystem"); + }, + overwriteSameLength: () => { + throw new Error("verify() must not touch the filesystem"); + }, mkdirp: () => { throw new Error("verify() must not touch the filesystem"); }, @@ -177,6 +183,12 @@ describe("AdminSecretManager", () => { replaceSymlink: () => { throw new Error("verify() must not touch the filesystem"); }, + readBytes: () => { + throw new Error("verify() must not touch the filesystem"); + }, + overwriteSameLength: () => { + throw new Error("verify() must not touch the filesystem"); + }, mkdirp: () => { throw new Error("must not touch the filesystem"); }, diff --git a/src/ports/filesystem.test.ts b/src/ports/filesystem.test.ts index 61ea824a2..77d1570b5 100644 --- a/src/ports/filesystem.test.ts +++ b/src/ports/filesystem.test.ts @@ -235,6 +235,49 @@ describe.each(implementations)("Filesystem contract: $name", ({ create, link }) ); }); + it("reads a link's target without following it and replaces it, and reads a file's bytes and overwrites them in place with as many", async () => { + const filesystem = create(); + const dir = `${temporaryDirectory}/bytes`; + const file = `${dir}/state.bin`; + const bytes = Uint8Array.from([0, 255, 128, 65, 10]); + await filesystem.mkdirp(dir); + await filesystem.writeFileStream( + file, + new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + ); + await filesystem.chmod(file, 0o640); + await link(filesystem, `${dir}/logs`, `${dir}/old`); + + expect(await filesystem.readlink(`${dir}/logs`), "a link is read, not followed").toBe( + `${dir}/old`, + ); + await filesystem.replaceSymlink(`${dir}/logs`, `${dir}/new`); + expect(await filesystem.readlink(`${dir}/logs`), "the link points at the new target").toBe( + `${dir}/new`, + ); + expect((await filesystem.lstat(`${dir}/logs`)).kind, "it is still a link").toBe("symlink"); + expect(await filesystem.readdir(dir), "no staged link is left").toEqual(["logs", "state.bin"]); + + expect(Array.from(await filesystem.readBytes(file)), "every byte, binary ones too").toEqual( + Array.from(bytes), + ); + await filesystem.overwriteSameLength(file, Uint8Array.from([1, 2, 3, 4, 5])); + expect(Array.from(await filesystem.readBytes(file))).toEqual([1, 2, 3, 4, 5]); + expect((await filesystem.lstat(file)).mode, "the mode is kept").toBe(0o640); + await expect( + filesystem.overwriteSameLength(file, Uint8Array.from([1, 2, 3])), + "another length is refused", + ).rejects.toThrow(); + expect(Array.from(await filesystem.readBytes(file)), "and the file is left as it was").toEqual([ + 1, 2, 3, 4, 5, + ]); + }); + it("refuses to answer for a path that runs through a file", async () => { // why: A mistyped device root -- `/notes.txt/ios` -- is a different problem from one // that is simply not there yet, and callers can only tell them apart by the errno. @@ -325,6 +368,21 @@ describe.each(implementations)("Filesystem ownership contract: $name", ({ create }); }); +describe("NodeFilesystem.overwriteSameLength", () => { + it("keeps the file's modification time", async () => { + const filesystem = new NodeFilesystem(); + const file = `${temporaryDirectory}/stamp.bin`; + await filesystem.mkdirp(temporaryDirectory); + await writeFile(file, "abc"); + await utimes(file, new Date(1_000_000), new Date(1_000_000)); + + await filesystem.overwriteSameLength(file, Buffer.from("xyz")); + + expect((await filesystem.stat(file)).modifiedAtMs).toBe(1_000_000); + await expect(filesystem.readFile(file)).resolves.toBe("xyz"); + }); +}); + describe("NodeFilesystem.directorySize", () => { it("counts a file that vanishes between the listing and its stat as 0 bytes, and fails on any other error", async () => { const filesystem = new NodeFilesystem(); diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index c1d953c28..015fc5385 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -127,6 +127,10 @@ export interface Filesystem { readlink(path: string): Promise; /** why: the link at `path` points at `target` from one step to the next, with no moment when it is missing. */ replaceSymlink(path: string, target: string): Promise; + /** why: a file's bytes as they are, for a caller that must not read them as text. */ + readBytes(path: string): Promise; + /** why: a file written in place with bytes of its own length, keeping its mode and modification time; another length is refused and the file left as it was. */ + overwriteSameLength(path: string, bytes: Uint8Array): Promise; } // why: only the directories that are absent get the mode, so one that is already there keeps its own. @@ -416,6 +420,12 @@ export class NodeFilesystem implements Filesystem { return readlink(path); } + async readBytes(_path: string): Promise { + return new Uint8Array(); + } + + async overwriteSameLength(_path: string, _bytes: Uint8Array): Promise {} + async replaceSymlink(path: string, target: string): Promise { const staged = `${path}.simlock-swap-${String(process.pid)}`; await symlink(target, staged); @@ -795,6 +805,12 @@ export class MemoryFilesystem implements Filesystem { this.#failures.set(path, code); } + async readBytes(_path: string): Promise { + return new Uint8Array(); + } + + async overwriteSameLength(_path: string, _bytes: Uint8Array): Promise {} + async readlink(path: string): Promise { this.#failIfDefined(path); const entry = this.#rawEntryAt(this.#underResolvedParent(path)); From 4313667a8aa42ba6c5d5615b424022689dc33c02 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:35:44 +0100 Subject: [PATCH 06/48] feat(ports): the filesystem reads a file's bytes and overwrites them at their own length (#1570) --- src/ports/filesystem.ts | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index 015fc5385..840fc91dd 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -420,11 +420,25 @@ export class NodeFilesystem implements Filesystem { return readlink(path); } - async readBytes(_path: string): Promise { - return new Uint8Array(); + async readBytes(path: string): Promise { + return readFile(path); } - async overwriteSameLength(_path: string, _bytes: Uint8Array): Promise {} + async overwriteSameLength(path: string, bytes: Uint8Array): Promise { + const handle = await open(path, "r+"); + try { + const details = await handle.stat(); + if (details.size !== bytes.length) { + throw new Error( + `Refusing to write ${String(bytes.length)} bytes over ${String(details.size)} in ${path}`, + ); + } + await handle.write(bytes, 0, bytes.length, 0); + await handle.utimes(details.atime, details.mtime); + } finally { + await handle.close(); + } + } async replaceSymlink(path: string, target: string): Promise { const staged = `${path}.simlock-swap-${String(process.pid)}`; @@ -805,11 +819,25 @@ export class MemoryFilesystem implements Filesystem { this.#failures.set(path, code); } - async readBytes(_path: string): Promise { - return new Uint8Array(); + async readBytes(path: string): Promise { + this.#failIfDefined(path); + const entry = this.#entryAt(path); + if (entry.kind !== "file") throw errnoError("EISDIR", `Is a directory: ${path}`); + return fileBytes(entry); } - async overwriteSameLength(_path: string, _bytes: Uint8Array): Promise {} + async overwriteSameLength(path: string, bytes: Uint8Array): Promise { + this.#failIfDefined(path); + const entry = this.#entryAt(path); + if (entry.kind !== "file") throw errnoError("EISDIR", `Is a directory: ${path}`); + if (fileBytes(entry).byteLength !== bytes.length) { + throw new Error( + `Refusing to write ${String(bytes.length)} bytes over ${path}: another length`, + ); + } + entry.bytes = Uint8Array.from(bytes); + entry.contents = Buffer.from(bytes).toString("utf8"); + } async readlink(path: string): Promise { this.#failIfDefined(path); From 177eca1d3df5d55d3b28639224d9d3abea9cb521 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:36:21 +0100 Subject: [PATCH 07/48] test: a copy's recorded files are swapped too, and a swap that changes a length is refused (#1570) --- src/drivers/ios/source-swap.test.ts | 154 +++++++++++++++++++++++----- 1 file changed, 126 insertions(+), 28 deletions(-) diff --git a/src/drivers/ios/source-swap.test.ts b/src/drivers/ios/source-swap.test.ts index a1f3ca42e..a19c68b23 100644 --- a/src/drivers/ios/source-swap.test.ts +++ b/src/drivers/ios/source-swap.test.ts @@ -2,31 +2,73 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import { DriverCrashError, OWNED_ROOT_MARKER_FILE } from "../../core/index.js"; +import { + FakeClock, + MemoryFilesystem, + ScriptedProcessRunner, + type Logger, +} from "../../ports/index.js"; import { IosSimctlDriver } from "./index.js"; import { memorySourceDatabase, plantReadySeed } from "./testing.js"; -import { - deviceRoot, - driverData, - instanceId, - plantManagedDevice, - simctl, - sourcesRoot, -} from "./test-slim.js"; +import { deviceRoot, driverData, instanceId, sourcesRoot } from "./test-slim.js"; const listFixture = readFileSync(new URL("./fixtures/simctl-list.json", import.meta.url), "utf8"); const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; const logsRoot = "/home/Library/Logs/CoreSimulator"; const seedUdid = "00000000-0000-0000-0000-0000000000AA"; +const usedUdid = "00000000-0000-0000-0000-0000000000BB"; const copyUdid = driverData.udid; -async function provisionCopy( - setup: (filesystem: MemoryFilesystem) => Promise, +/** why: a device set of another depth than the sources folder, as long as it, so an escaped path swapped into it changes length. */ +const deepRoot = "/home/.simlock/a/b/c/devices-d"; + +/** why: the clock moves when a file is read, so the swap's logged time is what the clock measured. */ +class SlowReadFilesystem extends MemoryFilesystem { + constructor(private readonly clock: FakeClock) { + super(); + } + + override async readBytes(path: string): Promise { + this.clock.advance(7); + return super.readBytes(path); + } +} + +class LineLogger implements Logger { + readonly lines: string[] = []; + + debug(message: string): void { + this.lines.push(message); + } + info(): void {} + warn(): void {} + error(): void {} + child(): Logger { + return this; + } +} + +interface Rig { + readonly filesystem: MemoryFilesystem; + readonly logger: LineLogger; + provision(): Promise; +} + +async function rigWith( + root: string, files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], -): Promise { - const filesystem = new MemoryFilesystem(); - await plantManagedDevice(filesystem); - await setup(filesystem); + plant: (filesystem: MemoryFilesystem, copy: string) => Promise, +): Promise { + const clock = new FakeClock(); + const filesystem = new SlowReadFilesystem(clock); + const logger = new LineLogger(); + await filesystem.mkdirp(`${root}/${copyUdid}`); + await filesystem.writeFileAtomic( + `${root}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), + ); + await plant(filesystem, `${root}/${copyUdid}`); const database = plantReadySeed(await memorySourceDatabase(), { deviceTypeId: driverData.deviceTypeId, files, @@ -35,45 +77,101 @@ async function provisionCopy( }); const runner = new ScriptedProcessRunner([ { - match: simctl("list", "-j", "devicetypes", "runtimes"), + match: { + args: ["simctl", "--set", root, "list", "-j", "devicetypes", "runtimes"], + command: "xcrun", + }, result: { code: 0, stderr: "", stdout: listFixture }, }, { match: { - args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, deviceRoot], + args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, root], command: "xcrun", }, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, }, + { match: { args: ["simctl", "--set", root, "delete", copyUdid], command: "xcrun" } }, ]); const driver = await IosSimctlDriver.create({ - clock: new FakeClock(), + clock, database, - driverConfig: { deviceRoot }, + driverConfig: { deviceRoot: root }, filesystem, idGenerator: { generate: () => "device-1" }, instanceId, + logger, logsRoot, processRunner: runner, simlockHome: "/home/.simlock", }); await driver.resolveSpec(spec); - await driver.provision(spec); - return filesystem; + return { filesystem, logger, provision: () => driver.provision(spec) }; } describe("a lease's copy before its first boot", () => { - it("swaps every recorded foreign folder for its own, length unchanged", async () => { - const link = `${deviceRoot}/${copyUdid}/data/Library/Logs`; + it("swaps every recorded foreign folder for its own, length unchanged, and logs how long it took", async () => { + const copy = `${deviceRoot}/${copyUdid}`; + const rig = await rigWith( + deviceRoot, + [ + { kind: "link", relativePath: "data/Library/Logs" }, + { kind: "file", relativePath: "data/prefs.bin" }, + { kind: "file", relativePath: "data/Media/state.json" }, + ], + async (fs) => { + await fs.mkdirp(`${copy}/data/Library`); + await fs.mkdirp(`${copy}/data/Media`); + fs.defineSymlink(`${copy}/data/Library/Logs`, `${logsRoot}/${seedUdid}`); + await fs.writeFileAtomic( + `${copy}/data/prefs.bin`, + `\u0001bplist\u0000${sourcesRoot}/${seedUdid}/data\u0002`, + ); + await fs.writeFileAtomic( + `${copy}/data/Media/state.json`, + `{"from":"${deviceRoot}/${usedUdid}/data"}`, + ); + await fs.writeFileAtomic(`${copy}/data/unrecorded.txt`, `${sourcesRoot}/${seedUdid}`); + }, + ); + + await rig.provision(); - const filesystem = await provisionCopy( + expect(await rig.filesystem.readlink(`${copy}/data/Library/Logs`), "a link").toBe( + `${logsRoot}/${copyUdid}`, + ); + expect(await rig.filesystem.readFile(`${copy}/data/prefs.bin`), "a binary file").toBe( + `\u0001bplist\u0000${deviceRoot}/${copyUdid}/data\u0002`, + ); + expect( + await rig.filesystem.readFile(`${copy}/data/Media/state.json`), + "a saved-from folder", + ).toBe(`{"from":"${deviceRoot}/${copyUdid}/data"}`); + expect( + await rig.filesystem.readFile(`${copy}/data/unrecorded.txt`), + "only recorded files", + ).toBe(`${sourcesRoot}/${seedUdid}`); + expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 14 ms for ${copyUdid}`); + }); + + it("refuses a file whose swap would change its length", async () => { + const copy = `${deepRoot}/${copyUdid}`; + const escaped = (root: string, udid: string): string => + `${root}/${udid}`.replaceAll("/", "\\/"); + const rig = await rigWith( + deepRoot, + [{ kind: "file", relativePath: "data/escaped.json" }], async (fs) => { - await fs.mkdirp(`${deviceRoot}/${copyUdid}/data/Library`); - fs.defineSymlink(link, `${logsRoot}/${seedUdid}`); + await fs.mkdirp(`${copy}/data`); + await fs.writeFileAtomic( + `${copy}/data/escaped.json`, + `"${escaped(sourcesRoot, seedUdid)}"`, + ); }, - [{ kind: "link", relativePath: "data/Library/Logs" }], ); - expect(await filesystem.readlink(link)).toBe(`${logsRoot}/${copyUdid}`); + const failed = await rig.provision().catch((error: unknown) => error); + + expect(failed).toBeInstanceOf(DriverCrashError); + expect((failed as Error).message).toContain("data/escaped.json"); }); }); From ea166656fdb319dd29a1a48abd712bc80e5fec9a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:36:41 +0100 Subject: [PATCH 08/48] feat(ios): a copy's recorded files are swapped before its first boot and the swap is timed (#1570) --- src/drivers/ios/index.ts | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 6e483ac35..9d31c845c 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -1167,12 +1167,31 @@ export class IosSimctlDriver implements Driver { ownUdid: copyUdid, sourcesSet: this.#sourcesSet, }; + const startedAt = this.#clock.now(); + let files = 0; + let links = 0; for (const file of this.#sources.filesOf(sourceId)) { const path = join(this.#deviceRoot, copyUdid, file.relativePath); - const target = await this.#filesystem.readlink(path); - const swapped = Buffer.from(swapForeignFolders(Buffer.from(target), folders)).toString(); - if (swapped !== target) await this.#filesystem.replaceSymlink(path, swapped); + const before = + file.kind === "link" + ? Buffer.from(await this.#filesystem.readlink(path)) + : await this.#filesystem.readBytes(path); + const after = swapForeignFolders(before, folders); + if (after.length !== before.length) { + throw new DriverCrashError(`swapping the folders in ${path} would change its length`); + } + if (Buffer.compare(before, after) === 0) continue; + if (file.kind === "link") { + await this.#filesystem.replaceSymlink(path, Buffer.from(after).toString()); + links += 1; + } else { + await this.#filesystem.overwriteSameLength(path, after); + files += 1; + } } + this.#logger?.debug( + `swapped ${String(files)} files and ${String(links)} links in ${String(this.#clock.now() - startedAt)} ms for ${copyUdid}`, + ); } async #destroySeed(seed: ReadySeed): Promise { From e2c386e8b26e13db4fcff7be9858648cf712d60a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:38:13 +0100 Subject: [PATCH 09/48] test: making a source flushes its WALs and records the files and links that name a folder (#1570) --- src/drivers/ios/index.ts | 3 + src/drivers/ios/source-isolation.test.ts | 351 +++++++++++++++++++++++ src/drivers/ios/source-swap.test.ts | 177 ------------ src/drivers/ios/testing.ts | 34 ++- 4 files changed, 387 insertions(+), 178 deletions(-) create mode 100644 src/drivers/ios/source-isolation.test.ts delete mode 100644 src/drivers/ios/source-swap.test.ts diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 9d31c845c..4b7e4be3d 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -57,6 +57,7 @@ import type { ProcessResult, ProcessRunner, SqliteDatabase, + SqliteOpener, TcpListenerFactory, TcpProbe, TcpStreamConnector, @@ -252,6 +253,8 @@ export interface IosSimctlDriverOptions { readonly coreSimulatorRoot?: string; /** why: `/Library/Logs/CoreSimulator`: a simulator's log folder is named in its files beside its device folder, and a copy's swap covers both (ADR 0076 §3). */ readonly logsRoot?: string; + /** why: a made source has the write-ahead log of each of its databases flushed before its files are indexed (ADR 0076 §2); with none, such a log is left out of the index and logged. */ + readonly sqlite?: SqliteOpener; // why: where launchd_sim keeps each simulator's overrides folder; a test points it elsewhere. readonly launchdOverridesRoot?: string; readonly onSlimmed?: (fact: SlimmedFact) => void; diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts new file mode 100644 index 000000000..d51359211 --- /dev/null +++ b/src/drivers/ios/source-isolation.test.ts @@ -0,0 +1,351 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; + +import { DriverCrashError, OWNED_ROOT_MARKER_FILE } from "../../core/index.js"; +import { + FakeClock, + MemoryFilesystem, + ScriptedProcessRunner, + type Logger, +} from "../../ports/index.js"; +import { IosSimctlDriver } from "./index.js"; +import { FakeSqliteOpener, memorySourceDatabase, plantReadySeed } from "./testing.js"; +import { deviceRoot, driverData, instanceId, sourcesRoot } from "./test-slim.js"; + +const listFixture = readFileSync(new URL("./fixtures/simctl-list.json", import.meta.url), "utf8"); +const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; +const logsRoot = "/home/Library/Logs/CoreSimulator"; +const seedUdid = "00000000-0000-0000-0000-0000000000AA"; +const usedUdid = "00000000-0000-0000-0000-0000000000BB"; +const copyUdid = driverData.udid; + +/** why: a device set of another depth than the sources folder, as long as it, so an escaped path swapped into it changes length. */ +const deepRoot = "/home/.simlock/a/b/c/devices-d"; + +/** why: the clock moves when a file is read, so the swap's logged time is what the clock measured. */ +class SlowReadFilesystem extends MemoryFilesystem { + constructor(private readonly clock: FakeClock) { + super(); + } + + override async readBytes(path: string): Promise { + this.clock.advance(7); + return super.readBytes(path); + } +} + +class LineLogger implements Logger { + readonly lines: string[] = []; + + debug(message: string): void { + this.lines.push(message); + } + info(): void {} + warn(): void {} + error(): void {} + child(): Logger { + return this; + } +} + +interface Rig { + readonly filesystem: MemoryFilesystem; + readonly logger: LineLogger; + provision(): Promise; +} + +async function rigWith( + root: string, + files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], + plant: (filesystem: MemoryFilesystem, copy: string) => Promise, +): Promise { + const clock = new FakeClock(); + const filesystem = new SlowReadFilesystem(clock); + const logger = new LineLogger(); + await filesystem.mkdirp(`${root}/${copyUdid}`); + await filesystem.writeFileAtomic( + `${root}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), + ); + await plant(filesystem, `${root}/${copyUdid}`); + const database = plantReadySeed(await memorySourceDatabase(), { + deviceTypeId: driverData.deviceTypeId, + files, + runtimeId: driverData.runtimeId, + udid: seedUdid, + }); + const runner = new ScriptedProcessRunner([ + { + match: { + args: ["simctl", "--set", root, "list", "-j", "devicetypes", "runtimes"], + command: "xcrun", + }, + result: { code: 0, stderr: "", stdout: listFixture }, + }, + { + match: { + args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, root], + command: "xcrun", + }, + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + { match: { args: ["simctl", "--set", root, "delete", copyUdid], command: "xcrun" } }, + ]); + const driver = await IosSimctlDriver.create({ + clock, + database, + driverConfig: { deviceRoot: root }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + logger, + logsRoot, + processRunner: runner, + simlockHome: "/home/.simlock", + }); + await driver.resolveSpec(spec); + return { filesystem, logger, provision: () => driver.provision(spec) }; +} + +describe("a lease's copy before its first boot", () => { + it("swaps every recorded foreign folder for its own, length unchanged, and logs how long it took", async () => { + const copy = `${deviceRoot}/${copyUdid}`; + const rig = await rigWith( + deviceRoot, + [ + { kind: "link", relativePath: "data/Library/Logs" }, + { kind: "file", relativePath: "data/prefs.bin" }, + { kind: "file", relativePath: "data/Media/state.json" }, + ], + async (fs) => { + await fs.mkdirp(`${copy}/data/Library`); + await fs.mkdirp(`${copy}/data/Media`); + fs.defineSymlink(`${copy}/data/Library/Logs`, `${logsRoot}/${seedUdid}`); + await fs.writeFileAtomic( + `${copy}/data/prefs.bin`, + `\u0001bplist\u0000${sourcesRoot}/${seedUdid}/data\u0002`, + ); + await fs.writeFileAtomic( + `${copy}/data/Media/state.json`, + `{"from":"${deviceRoot}/${usedUdid}/data"}`, + ); + await fs.writeFileAtomic(`${copy}/data/unrecorded.txt`, `${sourcesRoot}/${seedUdid}`); + }, + ); + + await rig.provision(); + + expect(await rig.filesystem.readlink(`${copy}/data/Library/Logs`), "a link").toBe( + `${logsRoot}/${copyUdid}`, + ); + expect(await rig.filesystem.readFile(`${copy}/data/prefs.bin`), "a binary file").toBe( + `\u0001bplist\u0000${deviceRoot}/${copyUdid}/data\u0002`, + ); + expect( + await rig.filesystem.readFile(`${copy}/data/Media/state.json`), + "a saved-from folder", + ).toBe(`{"from":"${deviceRoot}/${copyUdid}/data"}`); + expect( + await rig.filesystem.readFile(`${copy}/data/unrecorded.txt`), + "only recorded files", + ).toBe(`${sourcesRoot}/${seedUdid}`); + expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 14 ms for ${copyUdid}`); + }); + + it("refuses a file whose swap would change its length", async () => { + const copy = `${deepRoot}/${copyUdid}`; + const escaped = (root: string, udid: string): string => + `${root}/${udid}`.replaceAll("/", "\\/"); + const rig = await rigWith( + deepRoot, + [{ kind: "file", relativePath: "data/escaped.json" }], + async (fs) => { + await fs.mkdirp(`${copy}/data`); + await fs.writeFileAtomic( + `${copy}/data/escaped.json`, + `"${escaped(sourcesRoot, seedUdid)}"`, + ); + }, + ); + + const failed = await rig.provision().catch((error: unknown) => error); + + expect(failed).toBeInstanceOf(DriverCrashError); + expect((failed as Error).message).toContain("data/escaped.json"); + }); +}); + +describe("a source made", () => { + const cloneUdid = "00000000-0000-0000-0000-0000000000CC"; + const sourceFolder = (udid: string): string => `${sourcesRoot}/${udid}`; + const device = { + address: copyUdid, + deviceId: copyUdid, + driverData, + physical: false, + } as const; + const scoped = (...args: readonly string[]) => ({ + args: ["simctl", "--set", sourcesRoot, ...args], + command: "xcrun", + }); + const own = (...args: readonly string[]) => ({ + args: ["simctl", "--set", deviceRoot, ...args], + command: "xcrun", + }); + const ok = { code: 0, stderr: "", stdout: "" }; + + it.each([ + { + name: "a seed with two WALs: both flushed before its files are read", + make: "seed", + failing: [], + tree: { + "data/a.sqlite": (udid: string) => `${sourceFolder(udid)}/data`, + "data/a.sqlite-wal": (udid: string) => `${sourceFolder(udid)}/data`, + "data/b.sqlite": () => "plain", + "data/b.sqlite-wal": (udid: string) => `${sourceFolder(udid)}/x`, + "data/note.txt": () => "no folder here", + }, + link: true, + recorded: [ + ["data/Library/Logs", "link"], + ["data/a.sqlite", "file"], + ], + flushed: ["data/a.sqlite", "data/b.sqlite"], + warned: [], + }, + { + name: "a baseline saved from a used copy names that copy's folder", + make: "baseline", + failing: [], + tree: { "data/Media/state.plist": () => `${deviceRoot}/${copyUdid}/data/Media` }, + link: false, + recorded: [["data/Media/state.plist", "file"]], + flushed: [], + warned: [], + }, + { + name: "a WAL that cannot be flushed is logged and left out, and the source is still made", + make: "baseline", + failing: ["data/b.sqlite"], + tree: { + "data/b.sqlite": () => "plain", + "data/b.sqlite-wal": (udid: string) => `${sourceFolder(udid)}/x`, + }, + link: false, + recorded: [], + flushed: [], + warned: ["data/b.sqlite"], + }, + { + name: "a source that names no folder has no rows and is indexed", + make: "baseline", + failing: [], + tree: { "data/note.txt": () => "nothing" }, + link: false, + recorded: [], + flushed: [], + warned: [], + }, + ] as const)( + "flushes its WALs and records the files and links that name a folder: $name", + async ({ make, failing, tree, link, recorded, flushed, warned }) => { + const udid = make === "seed" ? seedUdid : cloneUdid; + const filesystem = new MemoryFilesystem(); + const sqlite = new FakeSqliteOpener(filesystem); + const warnings: string[] = []; + const logger = Object.assign(new LineLogger(), { + warn: (message: string) => void warnings.push(message), + }); + await filesystem.mkdirp(sourcesRoot, { mode: 0o700 }); + await filesystem.writeFileAtomic( + `${sourcesRoot}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), + ); + await filesystem.mkdirp(deviceRoot); + await filesystem.writeFileAtomic( + `${deviceRoot}/${OWNED_ROOT_MARKER_FILE}`, + JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), + ); + for (const folder of [sourceFolder(udid), `${deviceRoot}/${copyUdid}`]) { + for (const [path, contents] of Object.entries(tree)) { + const at = `${folder}/${path}`; + await filesystem.mkdirp(at.slice(0, at.lastIndexOf("/"))); + await filesystem.writeFileAtomic(at, contents(udid)); + } + if (link) { + await filesystem.mkdirp(`${folder}/data/Library`); + filesystem.defineSymlink(`${folder}/data/Library/Logs`, `${logsRoot}/${udid}`); + } + } + for (const path of failing) sqlite.failing.add(`${sourceFolder(udid)}/${path}`); + const database = await memorySourceDatabase(); + const runner = new ScriptedProcessRunner([ + { + match: own("list", "-j", "devicetypes", "runtimes"), + result: { ...ok, stdout: listFixture }, + }, + ...(make === "seed" + ? [ + { + match: scoped( + "create", + "simlock-seed-device-1", + driverData.deviceTypeId, + driverData.runtimeId, + ), + result: { ...ok, stdout: `${udid}\n` }, + }, + { match: scoped("boot", udid) }, + { match: scoped("bootstatus", udid, "-b") }, + { match: scoped("listapps", udid) }, + { match: scoped("shutdown", udid) }, + { + match: scoped("clone", udid, driverData.name, deviceRoot), + result: { ...ok, stdout: `${copyUdid}\n` }, + }, + ] + : [ + { match: own("shutdown", copyUdid) }, + { + match: own("clone", copyUdid, "simlock-baseline-device-1", sourcesRoot), + result: { ...ok, stdout: `${udid}\n` }, + }, + ]), + ]); + const driver = await IosSimctlDriver.create({ + clock: new FakeClock(), + database, + driverConfig: { deviceRoot }, + filesystem, + idGenerator: { generate: () => "device-1" }, + instanceId, + logger, + logsRoot, + processRunner: runner, + simlockHome: "/home/.simlock", + sqlite, + }); + if (make === "seed") await driver.resolveSpec(spec); + + await (make === "seed" ? driver.provision(spec) : driver.saveSource(device)); + + expect( + database + .prepare("SELECT relative_path, kind FROM ios_source_files ORDER BY relative_path") + .all(), + "the files and links that name a folder", + ).toEqual(recorded.map(([relativePath, kind]) => ({ kind, relative_path: relativePath }))); + expect(database.prepare("SELECT indexed FROM ios_source_data").all()).toEqual([ + { indexed: 1 }, + ]); + expect(sqlite.flushed, "the databases whose WAL was flushed").toEqual( + flushed.map((path) => `${sourceFolder(udid)}/${path}`), + ); + expect(warnings).toEqual( + warned.map((path) => `could not flush ${sourceFolder(udid)}/${path}: database is locked`), + ); + }, + ); +}); diff --git a/src/drivers/ios/source-swap.test.ts b/src/drivers/ios/source-swap.test.ts deleted file mode 100644 index a19c68b23..000000000 --- a/src/drivers/ios/source-swap.test.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { readFileSync } from "node:fs"; - -import { describe, expect, it } from "vitest"; - -import { DriverCrashError, OWNED_ROOT_MARKER_FILE } from "../../core/index.js"; -import { - FakeClock, - MemoryFilesystem, - ScriptedProcessRunner, - type Logger, -} from "../../ports/index.js"; -import { IosSimctlDriver } from "./index.js"; -import { memorySourceDatabase, plantReadySeed } from "./testing.js"; -import { deviceRoot, driverData, instanceId, sourcesRoot } from "./test-slim.js"; - -const listFixture = readFileSync(new URL("./fixtures/simctl-list.json", import.meta.url), "utf8"); -const spec = { physical: false, model: "iPhone 16", osVersion: "26.5", platform: "ios" } as const; -const logsRoot = "/home/Library/Logs/CoreSimulator"; -const seedUdid = "00000000-0000-0000-0000-0000000000AA"; -const usedUdid = "00000000-0000-0000-0000-0000000000BB"; -const copyUdid = driverData.udid; - -/** why: a device set of another depth than the sources folder, as long as it, so an escaped path swapped into it changes length. */ -const deepRoot = "/home/.simlock/a/b/c/devices-d"; - -/** why: the clock moves when a file is read, so the swap's logged time is what the clock measured. */ -class SlowReadFilesystem extends MemoryFilesystem { - constructor(private readonly clock: FakeClock) { - super(); - } - - override async readBytes(path: string): Promise { - this.clock.advance(7); - return super.readBytes(path); - } -} - -class LineLogger implements Logger { - readonly lines: string[] = []; - - debug(message: string): void { - this.lines.push(message); - } - info(): void {} - warn(): void {} - error(): void {} - child(): Logger { - return this; - } -} - -interface Rig { - readonly filesystem: MemoryFilesystem; - readonly logger: LineLogger; - provision(): Promise; -} - -async function rigWith( - root: string, - files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], - plant: (filesystem: MemoryFilesystem, copy: string) => Promise, -): Promise { - const clock = new FakeClock(); - const filesystem = new SlowReadFilesystem(clock); - const logger = new LineLogger(); - await filesystem.mkdirp(`${root}/${copyUdid}`); - await filesystem.writeFileAtomic( - `${root}/${OWNED_ROOT_MARKER_FILE}`, - JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), - ); - await plant(filesystem, `${root}/${copyUdid}`); - const database = plantReadySeed(await memorySourceDatabase(), { - deviceTypeId: driverData.deviceTypeId, - files, - runtimeId: driverData.runtimeId, - udid: seedUdid, - }); - const runner = new ScriptedProcessRunner([ - { - match: { - args: ["simctl", "--set", root, "list", "-j", "devicetypes", "runtimes"], - command: "xcrun", - }, - result: { code: 0, stderr: "", stdout: listFixture }, - }, - { - match: { - args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, root], - command: "xcrun", - }, - result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, - }, - { match: { args: ["simctl", "--set", root, "delete", copyUdid], command: "xcrun" } }, - ]); - const driver = await IosSimctlDriver.create({ - clock, - database, - driverConfig: { deviceRoot: root }, - filesystem, - idGenerator: { generate: () => "device-1" }, - instanceId, - logger, - logsRoot, - processRunner: runner, - simlockHome: "/home/.simlock", - }); - await driver.resolveSpec(spec); - return { filesystem, logger, provision: () => driver.provision(spec) }; -} - -describe("a lease's copy before its first boot", () => { - it("swaps every recorded foreign folder for its own, length unchanged, and logs how long it took", async () => { - const copy = `${deviceRoot}/${copyUdid}`; - const rig = await rigWith( - deviceRoot, - [ - { kind: "link", relativePath: "data/Library/Logs" }, - { kind: "file", relativePath: "data/prefs.bin" }, - { kind: "file", relativePath: "data/Media/state.json" }, - ], - async (fs) => { - await fs.mkdirp(`${copy}/data/Library`); - await fs.mkdirp(`${copy}/data/Media`); - fs.defineSymlink(`${copy}/data/Library/Logs`, `${logsRoot}/${seedUdid}`); - await fs.writeFileAtomic( - `${copy}/data/prefs.bin`, - `\u0001bplist\u0000${sourcesRoot}/${seedUdid}/data\u0002`, - ); - await fs.writeFileAtomic( - `${copy}/data/Media/state.json`, - `{"from":"${deviceRoot}/${usedUdid}/data"}`, - ); - await fs.writeFileAtomic(`${copy}/data/unrecorded.txt`, `${sourcesRoot}/${seedUdid}`); - }, - ); - - await rig.provision(); - - expect(await rig.filesystem.readlink(`${copy}/data/Library/Logs`), "a link").toBe( - `${logsRoot}/${copyUdid}`, - ); - expect(await rig.filesystem.readFile(`${copy}/data/prefs.bin`), "a binary file").toBe( - `\u0001bplist\u0000${deviceRoot}/${copyUdid}/data\u0002`, - ); - expect( - await rig.filesystem.readFile(`${copy}/data/Media/state.json`), - "a saved-from folder", - ).toBe(`{"from":"${deviceRoot}/${copyUdid}/data"}`); - expect( - await rig.filesystem.readFile(`${copy}/data/unrecorded.txt`), - "only recorded files", - ).toBe(`${sourcesRoot}/${seedUdid}`); - expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 14 ms for ${copyUdid}`); - }); - - it("refuses a file whose swap would change its length", async () => { - const copy = `${deepRoot}/${copyUdid}`; - const escaped = (root: string, udid: string): string => - `${root}/${udid}`.replaceAll("/", "\\/"); - const rig = await rigWith( - deepRoot, - [{ kind: "file", relativePath: "data/escaped.json" }], - async (fs) => { - await fs.mkdirp(`${copy}/data`); - await fs.writeFileAtomic( - `${copy}/data/escaped.json`, - `"${escaped(sourcesRoot, seedUdid)}"`, - ); - }, - ); - - const failed = await rig.provision().catch((error: unknown) => error); - - expect(failed).toBeInstanceOf(DriverCrashError); - expect((failed as Error).message).toContain("data/escaped.json"); - }); -}); diff --git a/src/drivers/ios/testing.ts b/src/drivers/ios/testing.ts index 375d5851f..d966d5261 100644 --- a/src/drivers/ios/testing.ts +++ b/src/drivers/ios/testing.ts @@ -1,4 +1,9 @@ -import { NodeSqliteOpener, type SqliteDatabase } from "../../ports/index.js"; +import { + NodeSqliteOpener, + type Filesystem, + type SqliteDatabase, + type SqliteOpener, +} from "../../ports/index.js"; import { iosSourceDataOwner, IosSourceTable, type SourceFile } from "./source-data.js"; export async function memorySourceDatabase(): Promise { @@ -50,3 +55,30 @@ export function plantReadySeed( ): SqliteDatabase { return plantSource(database, seed); } + +/** why: stands in for the opener that flushes a source's databases: a checkpoint empties the database's `-wal` file in the filesystem, as SQLite's does, and a path in `failing` cannot be opened. */ +export class FakeSqliteOpener implements SqliteOpener { + readonly flushed: string[] = []; + readonly failing = new Set(); + readonly #filesystem: Filesystem; + + constructor(filesystem: Filesystem) { + this.#filesystem = filesystem; + } + + async open(path: string, options: { readonly readOnly: boolean }): Promise { + if (options.readOnly) throw new Error(`${path} is opened read-write to flush it`); + if (this.failing.has(path)) throw new Error("database is locked"); + return { + close: () => undefined, + exec: (sql) => { + if (sql !== "PRAGMA wal_checkpoint(TRUNCATE)") throw new Error(`unexpected ${sql}`); + this.flushed.push(path); + void this.#filesystem.writeFileAtomic(`${path}-wal`, ""); + }, + prepare: () => { + throw new Error("a flush runs one pragma and nothing else"); + }, + }; + } +} From 14a9f5d200cdc4d0032fa4d6d577febabc2ebb4d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:39:01 +0100 Subject: [PATCH 10/48] feat(ios): a made source has its WALs flushed and its files and links that name a folder recorded (#1570) --- src/daemon/driver-discovery.ts | 2 + src/drivers/ios/foreign-folders.ts | 12 ++++ src/drivers/ios/index.ts | 30 +++++++++- src/drivers/ios/source-index.ts | 92 ++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 src/drivers/ios/source-index.ts diff --git a/src/daemon/driver-discovery.ts b/src/daemon/driver-discovery.ts index 4a3ce350e..a3ed45b7a 100644 --- a/src/daemon/driver-discovery.ts +++ b/src/daemon/driver-discovery.ts @@ -36,6 +36,7 @@ import { type IdGenerator, type Logger, NodeBytePipeLauncher, + NodeSqliteOpener, NodeTcpListenerFactory, NodeTcpStreamConnector, type ProcessRunner, @@ -198,6 +199,7 @@ async function discoverIosDriver( instanceId: options.instanceId, logger: logger.child("ios"), logsRoot: `${homedir()}/Library/Logs/CoreSimulator`, + sqlite: new NodeSqliteOpener(), onSilenceFailed: logSilenceFailure(logger), onWindowFailed: logWindowFailure(logger), onSlimmed: emitSlimDiagnostic(options.eventBus), diff --git a/src/drivers/ios/foreign-folders.ts b/src/drivers/ios/foreign-folders.ts index ac47e35b9..d01cc1598 100644 --- a/src/drivers/ios/foreign-folders.ts +++ b/src/drivers/ios/foreign-folders.ts @@ -38,6 +38,18 @@ function ownFolder(root: string, udid: string, found: string): string { return `${spelling}/${udid}`.replaceAll("/", escaped ? "\\/" : "/"); } +/** why: ADR 0076 §2: a source's index lists the files and links that name any simulator's device or log folder, its own included. */ +export function namesAnyFolder( + bytes: Uint8Array, + folders: Omit, +): boolean { + const text = Buffer.from(bytes).toString("latin1"); + return ( + pattern([folders.deviceSet, folders.sourcesSet]).test(text) || + (folders.logsRoot !== undefined && pattern([folders.logsRoot]).test(text)) + ); +} + /** * why: ADR 0076 §3: every device or log folder of another simulator becomes the copy's own, byte for byte, * so a file's length never changes and the swap needs to understand no format. Bytes map one to one to diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 4b7e4be3d..af762b456 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -85,8 +85,9 @@ import { fillSimctlDevice } from "./simctl-device.js"; import { IosPhysicalHandler } from "./physical.js"; import { createIosDeviceInput } from "./input/index.js"; import { iosRuntimeVersionFromId } from "./runtime-version.js"; -import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; +import { IosSourceTable, type ReadySeed, type SourceFile, type SourceRow } from "./source-data.js"; import { swapForeignFolders } from "./foreign-folders.js"; +import { indexSource } from "./source-index.js"; import { seedIsStale, sourceStaleReason } from "./source-staleness.js"; import { fitSourcesFolders, sourcesFolderOf, sourcesSetCheck } from "./sources-folder.js"; import { createIosScreenCapture } from "./screen-capture.js"; @@ -341,6 +342,7 @@ export class IosSimctlDriver implements Driver { readonly #sourcesSet: string; readonly #ownsSourcesSet: (path: string) => boolean; readonly #logsRoot: string | undefined; + readonly #sqlite: SqliteOpener | undefined; #sourceMaker: string | undefined; #swept = false; readonly #seedMakes = new Map>(); @@ -382,6 +384,7 @@ export class IosSimctlDriver implements Driver { this.#sources = new IosSourceTable(options.database); this.#sourcesSet = sourcesSet; this.#logsRoot = options.logsRoot; + this.#sqlite = options.sqlite; this.#ownsSourcesSet = sourcesSetCheck( sourcesSet, options.simlockHome, @@ -895,8 +898,9 @@ export class IosSimctlDriver implements Driver { this.#sources.setUdid(sourceId, cloneUdid); await this.#copyOverrides(data.udid, cloneUdid); await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); + const files = await this.#indexSource(cloneUdid); const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes, []); + this.#sources.markReady(sourceId, sizeBytes, files); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; } catch (error: unknown) { if (cloneUdid !== undefined && cloneUdid !== "") { @@ -1090,7 +1094,14 @@ export class IosSimctlDriver implements Driver { await this.#deleteSeedSimulator(udid); throw error; } - this.#sources.markReady(sourceId, await this.#sizeOf(udid), []); + let files: readonly SourceFile[]; + try { + files = await this.#indexSource(udid); + } catch (error: unknown) { + await this.#deleteSeedSimulator(udid); + throw error; + } + this.#sources.markReady(sourceId, await this.#sizeOf(udid), files); return { sourceId, setPath: this.#sourcesSet, udid, runtimeBuild: resolved.runtime.build }; } catch (error: unknown) { this.#sources.deleteRow(sourceId); @@ -1158,6 +1169,19 @@ export class IosSimctlDriver implements Driver { } } + #indexSource(udid: string): Promise { + return indexSource(join(this.#sourcesSet, udid), { + filesystem: this.#filesystem, + folders: { + deviceSet: this.#deviceRoot, + logsRoot: this.#logsRoot, + sourcesSet: this.#sourcesSet, + }, + logger: this.#logger, + sqlite: this.#sqlite, + }); + } + /** * why: ADR 0076 §3: a copy's files and links still name its source's device and log folders, and * the copy would write into them; before its first boot each recorded one names the copy's own diff --git a/src/drivers/ios/source-index.ts b/src/drivers/ios/source-index.ts new file mode 100644 index 000000000..a4320b0e4 --- /dev/null +++ b/src/drivers/ios/source-index.ts @@ -0,0 +1,92 @@ +import { join } from "node:path"; + +import type { Filesystem, Logger, SqliteOpener } from "../../ports/index.js"; +import { namesAnyFolder, type ForeignFolders } from "./foreign-folders.js"; +import type { SourceFile } from "./source-data.js"; + +const WAL_SUFFIX = "-wal"; +const CHECKPOINT = "PRAGMA wal_checkpoint(TRUNCATE)"; + +export interface SourceIndexOptions { + readonly filesystem: Filesystem; + readonly folders: Omit; + readonly logger: Logger | undefined; + readonly sqlite: SqliteOpener | undefined; +} + +interface Entry extends SourceFile { + readonly path: string; +} + +/** + * why: ADR 0076 §2: a WAL frame carries a checksum a swap would break, so every WAL of a made source is + * flushed into its database first; then the files and links that name a device or log folder are found + * once, and each copy swaps only those. A WAL that cannot be flushed is logged and left out, never patched. + */ +export async function indexSource( + root: string, + options: SourceIndexOptions, +): Promise { + const entries = await walk(options.filesystem, root, ""); + const unflushed = new Set(); + for (const entry of entries) { + if (entry.kind === "file" && entry.path.endsWith(WAL_SUFFIX)) { + if (!(await flush(entry.path.slice(0, -WAL_SUFFIX.length), options))) { + unflushed.add(entry.path); + } + } + } + const indexed: SourceFile[] = []; + for (const entry of entries) { + if (unflushed.has(entry.path)) continue; + const bytes = + entry.kind === "link" + ? Buffer.from(await options.filesystem.readlink(entry.path)) + : await options.filesystem.readBytes(entry.path); + if (namesAnyFolder(bytes, options.folders)) { + indexed.push({ kind: entry.kind, relativePath: entry.relativePath }); + } + } + return indexed; +} + +async function walk(filesystem: Filesystem, directory: string, relative: string): Promise { + const entries: Entry[] = []; + let names: string[]; + try { + names = await filesystem.readdir(directory); + } catch (error: unknown) { + // why: a folder that is not there holds nothing to record; `simctl` made it, and its size is counted the same way. + if ((error as NodeJS.ErrnoException).code === "ENOENT") return entries; + throw error; + } + for (const name of names) { + const path = join(directory, name); + const relativePath = relative === "" ? name : `${relative}/${name}`; + const { kind } = await filesystem.lstat(path); + if (kind === "directory") entries.push(...(await walk(filesystem, path, relativePath))); + else if (kind === "file" || kind === "symlink") { + entries.push({ kind: kind === "file" ? "file" : "link", path, relativePath }); + } + } + return entries; +} + +async function flush(database: string, options: SourceIndexOptions): Promise { + try { + if (options.sqlite === undefined) throw new Error("no SQLite is available"); + if (!(await options.filesystem.exists(database))) throw new Error("its database is gone"); + const opened = await options.sqlite.open(database, { readOnly: false }); + try { + opened.exec(CHECKPOINT); + } finally { + opened.close(); + } + return true; + } catch (error: unknown) { + options.logger?.warn( + `could not flush ${database}: ${error instanceof Error ? error.message : String(error)}`, + ); + return false; + } +} From 058196c60e142e4f5685a9823dd6312ff6984164 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:39:30 +0100 Subject: [PATCH 11/48] test: a copy whose swap fails is deleted and the provision fails (#1570) --- src/drivers/ios/index.ts | 2 +- src/drivers/ios/source-isolation.test.ts | 48 ++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index af762b456..28529f59c 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -989,7 +989,7 @@ export class IosSimctlDriver implements Driver { await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); await this.#swapForeignFolders(seed.sourceId, udid); } catch (error: unknown) { - // why: a clone whose mark or database drop failed is not a device Simlock hands out, and + // why: a clone whose mark, database drop or folder swap failed is not a device Simlock hands out, and // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. await this.#invokeSimctl(["delete", udid], COMMAND_TIMEOUT_MS).catch(() => undefined); throw error; diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts index d51359211..3a5b57596 100644 --- a/src/drivers/ios/source-isolation.test.ts +++ b/src/drivers/ios/source-isolation.test.ts @@ -29,6 +29,13 @@ class SlowReadFilesystem extends MemoryFilesystem { super(); } + failWrites = false; + + override async overwriteSameLength(path: string, bytes: Uint8Array): Promise { + if (this.failWrites) throw new Error("disk full"); + await super.overwriteSameLength(path, bytes); + } + override async readBytes(path: string): Promise { this.clock.advance(7); return super.readBytes(path); @@ -50,7 +57,8 @@ class LineLogger implements Logger { } interface Rig { - readonly filesystem: MemoryFilesystem; + readonly filesystem: SlowReadFilesystem; + readonly simctlCalls: () => readonly string[]; readonly logger: LineLogger; provision(): Promise; } @@ -58,7 +66,7 @@ interface Rig { async function rigWith( root: string, files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], - plant: (filesystem: MemoryFilesystem, copy: string) => Promise, + plant: (filesystem: SlowReadFilesystem, copy: string) => Promise, ): Promise { const clock = new FakeClock(); const filesystem = new SlowReadFilesystem(clock); @@ -105,7 +113,12 @@ async function rigWith( simlockHome: "/home/.simlock", }); await driver.resolveSpec(spec); - return { filesystem, logger, provision: () => driver.provision(spec) }; + return { + filesystem, + logger, + provision: () => driver.provision(spec), + simctlCalls: () => runner.calls.map((call) => call.args.slice(3).join(" ")), + }; } describe("a lease's copy before its first boot", () => { @@ -176,6 +189,35 @@ describe("a lease's copy before its first boot", () => { }); }); +describe("a copy whose swap fails", () => { + it.each([ + { + name: "a recorded file is gone", + plantFile: false, + failWrites: false, + message: "data/prefs.bin", + }, + { name: "a write fails", plantFile: true, failWrites: true, message: "disk full" }, + ])("is deleted and fails the provision: $name", async ({ plantFile, failWrites, message }) => { + const copy = `${deviceRoot}/${copyUdid}`; + const rig = await rigWith( + deviceRoot, + [{ kind: "file", relativePath: "data/prefs.bin" }], + async (fs) => { + await fs.mkdirp(`${copy}/data`); + if (plantFile) { + await fs.writeFileAtomic(`${copy}/data/prefs.bin`, `${sourcesRoot}/${seedUdid}`); + } + fs.failWrites = failWrites; + }, + ); + + await expect(rig.provision()).rejects.toThrow(message); + + expect(rig.simctlCalls()).toContain(`delete ${copyUdid}`); + }); +}); + describe("a source made", () => { const cloneUdid = "00000000-0000-0000-0000-0000000000CC"; const sourceFolder = (udid: string): string => `${sourcesRoot}/${udid}`; From 85d30e0f2244b247aa793572eb58f534d45af5d9 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:40:05 +0100 Subject: [PATCH 12/48] test: a seed not indexed is made again and a baseline not indexed is stale (#1570) --- src/drivers/ios/index.test.ts | 19 ++++++++++++++++--- src/drivers/ios/sources-folder.test.ts | 15 ++++++++++++--- src/drivers/ios/testing.ts | 5 +++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/drivers/ios/index.test.ts b/src/drivers/ios/index.test.ts index 39e3c0309..6a0d8f6ff 100644 --- a/src/drivers/ios/index.test.ts +++ b/src/drivers/ios/index.test.ts @@ -230,7 +230,12 @@ function sourcesSimctl(...args: readonly string[]): { /** why: A database that already holds the seed of the kind `spec` names, so a test of the copy step starts at the clone. */ async function seededDatabase( - seed: { readonly runtimeBuild?: string; readonly setPath?: string; readonly udid?: string } = {}, + seed: { + readonly indexed?: false; + readonly runtimeBuild?: string; + readonly setPath?: string; + readonly udid?: string; + } = {}, ): Promise { return plantReadySeed(await memorySourceDatabase(), { deviceTypeId: driverData.deviceTypeId, @@ -1583,9 +1588,16 @@ describe("IosSimctlDriver", () => { remade: true, setPath: "/home/.simlock/devices/ios-sources", }, - ])( + { + name: "it is not indexed", + seedBuild: "23F79", + runtimeBuild: "23F79", + remade: true, + indexed: false, + }, + ] as const)( "makes the seed again before copying it only when it is stale: $name -> remade $remade", - async ({ seedBuild, runtimeBuild, remade, setPath = sourcesRoot }) => { + async ({ seedBuild, runtimeBuild, remade, setPath = sourcesRoot, ...planting }) => { const staleUdid = "00000000-0000-0000-0000-0000000000bb"; const catalog = JSON.parse(listFixture) as { runtimes: { buildversion?: string; identifier: string }[]; @@ -1632,6 +1644,7 @@ describe("IosSimctlDriver", () => { await seededDatabase({ setPath, udid: planted, + ...("indexed" in planting ? { indexed: planting.indexed } : {}), ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), }), ); diff --git a/src/drivers/ios/sources-folder.test.ts b/src/drivers/ios/sources-folder.test.ts index af96b393f..332a6ae87 100644 --- a/src/drivers/ios/sources-folder.test.ts +++ b/src/drivers/ios/sources-folder.test.ts @@ -200,16 +200,24 @@ describe("IosSimctlDriver sources folder", () => { } }); - it("lists a baseline saved in another set as stale with the reason to save it again, one in the sources folder as fresh, and destroys the old one in its own set", async () => { - const [old, fresh] = [ + it("lists a baseline saved in another set, or not indexed, as stale with the reason to save it again, one in the sources folder as fresh, and destroys the old one in its own set", async () => { + const [old, fresh, unindexed] = [ "00000000-0000-0000-0000-0000000002a0", "00000000-0000-0000-0000-0000000002b0", - ] as [string, string]; + "00000000-0000-0000-0000-0000000002c0", + ] as [string, string, string]; const oldSet = "/home/.simlock/devices/ios-sources"; const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; const database = await memorySourceDatabase(); plantSource(database, { ...kind, kind: "baseline", setPath: oldSet, udid: old }); plantSource(database, { ...kind, kind: "baseline", setPath: sourcesRoot, udid: fresh }); + plantSource(database, { + ...kind, + indexed: false, + kind: "baseline", + setPath: sourcesRoot, + udid: unindexed, + }); const runner = new ScriptedProcessRunner([ { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, { match: simctlIn(oldSet, "shutdown", old) }, @@ -231,6 +239,7 @@ describe("IosSimctlDriver sources folder", () => { expect(listed.map((source) => source.staleReason)).toEqual([ `saved in ${oldSet}, which copies cannot be fixed from; save it again`, undefined, + `saved in ${sourcesRoot}, which copies cannot be fixed from; save it again`, ]); const oldId = listed[0]?.id ?? ""; await driver.destroySource(oldId); diff --git a/src/drivers/ios/testing.ts b/src/drivers/ios/testing.ts index d966d5261..ad3f556ee 100644 --- a/src/drivers/ios/testing.ts +++ b/src/drivers/ios/testing.ts @@ -15,6 +15,8 @@ export async function memorySourceDatabase(): Promise { export interface PlantedSource { readonly deviceTypeId: string; readonly files?: readonly SourceFile[]; + /** why: a source made before ADR 0076's index, which only a row of an earlier release has; absent means indexed. */ + readonly indexed?: false; readonly kind?: "seed" | "baseline"; readonly maker?: string; readonly model?: string; @@ -46,6 +48,9 @@ export function plantSource(database: SqliteDatabase, source: PlantedSource): Sq }); if (source.udid !== undefined) table.setUdid(sourceId, source.udid); if (source.state !== "making") table.markReady(sourceId, source.sizeBytes, source.files ?? []); + if (source.indexed === false) { + database.prepare("UPDATE ios_source_data SET indexed = 0 WHERE source_id = ?").run(sourceId); + } return database; } From c20da619a58a2eaadb7a9771719c49fe42e6a3be Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:40:29 +0100 Subject: [PATCH 13/48] feat(ios): a seed not indexed is made again and a baseline not indexed is stale (#1570) --- src/drivers/ios/index.ts | 8 +++++++- src/drivers/ios/source-data.ts | 8 ++++++-- src/drivers/ios/source-staleness.ts | 6 +++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 28529f59c..e7925f160 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -1102,7 +1102,13 @@ export class IosSimctlDriver implements Driver { throw error; } this.#sources.markReady(sourceId, await this.#sizeOf(udid), files); - return { sourceId, setPath: this.#sourcesSet, udid, runtimeBuild: resolved.runtime.build }; + return { + indexed: true, + runtimeBuild: resolved.runtime.build, + setPath: this.#sourcesSet, + sourceId, + udid, + }; } catch (error: unknown) { this.#sources.deleteRow(sourceId); throw error; diff --git a/src/drivers/ios/source-data.ts b/src/drivers/ios/source-data.ts index c6c183f25..fa1e6175c 100644 --- a/src/drivers/ios/source-data.ts +++ b/src/drivers/ios/source-data.ts @@ -74,9 +74,11 @@ export interface SourceRow { readonly state: "making" | "ready"; readonly maker: string; readonly sizeBytes: number | undefined; + readonly indexed: boolean; } export interface ReadySeed { + readonly indexed: boolean; readonly sourceId: string; readonly setPath: string; readonly udid: string; @@ -156,11 +158,12 @@ export class IosSourceTable { findReadySeed(specKey: string): ReadySeed | undefined { const row = this.#database .prepare( - "SELECT source_id, set_path, udid, runtime_build FROM ios_source_data WHERE kind = 'seed' AND state = 'ready' AND spec_key = ?", + "SELECT source_id, set_path, udid, runtime_build, indexed FROM ios_source_data WHERE kind = 'seed' AND state = 'ready' AND spec_key = ?", ) .get(specKey); if (row === undefined) return undefined; return { + indexed: row["indexed"] === 1, sourceId: row["source_id"] as string, setPath: row["set_path"] as string, udid: row["udid"] as string, @@ -203,7 +206,7 @@ export class IosSourceTable { return this.#database .prepare( `SELECT source_id, set_path, kind, udid, device_type_id, runtime_id, runtime_build, model, os_version, - state, maker, size_bytes FROM ios_source_data ${where}`, + state, maker, size_bytes, indexed FROM ios_source_data ${where}`, ) .all(...parameters) .map((row) => ({ @@ -219,6 +222,7 @@ export class IosSourceTable { state: row["state"] as "making" | "ready", maker: row["maker"] as string, sizeBytes: (row["size_bytes"] as number | null) ?? undefined, + indexed: row["indexed"] === 1, })); } } diff --git a/src/drivers/ios/source-staleness.ts b/src/drivers/ios/source-staleness.ts index 1a8a3151d..538ae64cb 100644 --- a/src/drivers/ios/source-staleness.ts +++ b/src/drivers/ios/source-staleness.ts @@ -1,13 +1,13 @@ import type { SimctlCatalog } from "./simctl-parse.js"; import type { ReadySeed, SourceRow } from "./source-data.js"; -/** why: a seed in another set than today's sources folder was made before ADR 0076, and a copy of it cannot be fixed from; a seed built on a runtime whose build changed no longer matches it; with no build on either side only the runtime's presence is checked, and a provision always has the runtime. */ +/** why: a seed in another set than today's sources folder, or without an index of its files, was made before ADR 0076, and a copy of it cannot be fixed from; a seed built on a runtime whose build changed no longer matches it; with no build on either side only the runtime's presence is checked, and a provision always has the runtime. */ export function seedIsStale( seed: ReadySeed, currentBuild: string | undefined, sourcesSet: string, ): boolean { - if (seed.setPath !== sourcesSet) return true; + if (seed.setPath !== sourcesSet || !seed.indexed) return true; return ( seed.runtimeBuild !== undefined && currentBuild !== undefined && @@ -21,7 +21,7 @@ export function sourceStaleReason( catalog: SimctlCatalog, sourcesSet: string, ): string | undefined { - if (row.setPath !== sourcesSet) { + if (row.setPath !== sourcesSet || !row.indexed) { return `saved in ${row.setPath}, which copies cannot be fixed from; save it again`; } const runtime = catalog.runtimes.find((entry) => entry.identifier === row.runtimeId); From aae54f036e82cb7cdd00ce3ca44d25b3e4c9012d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:42:10 +0100 Subject: [PATCH 14/48] feat(e2e): the lane keeps a seed's file rows with its row (#1570) --- e2e/helpers/lane-home.test.ts | 47 +++++++++++++++++++++++++++++++++-- e2e/helpers/lane-home.ts | 39 ++++++++++++++++++++++------- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/e2e/helpers/lane-home.test.ts b/e2e/helpers/lane-home.test.ts index 26914aa2e..7eb6f1ea3 100644 --- a/e2e/helpers/lane-home.test.ts +++ b/e2e/helpers/lane-home.test.ts @@ -1,9 +1,11 @@ -import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { describe, expect, it } from "vitest"; -import { clearAllButState } from "./lane-home.js"; +import { iosSourceDataOwner } from "../../src/drivers/ios/index.js"; +import { clearAllButState, releaseLaneHome, restoreKeptSeeds } from "./lane-home.js"; // why: lane-home is a helper file of the e2e lane, outside any module surface; the daemon start and stop that puts a kept seed's rows back runs only on the slow lane, where a server left behind fails a flow far from its cause. describe("clearAllButState", () => { @@ -28,3 +30,44 @@ describe("clearAllButState", () => { } }); }); + +describe("a seed the lane keeps between flows", () => { + it("keeps its file rows and indexed mark, so the next flow copies it instead of making it again", async () => { + const home = await mkdtemp(join(tmpdir(), "simlock-lane-home-test-")); + const udid = "00000000-0000-0000-0000-0000000000AA"; + const startDaemon = (): DatabaseSync => { + const database = new DatabaseSync(join(home, "state.db")); + for (const migration of iosSourceDataOwner.migrations) database.exec(migration); + return database; + }; + try { + await mkdir(join(home, "seeds", udid), { recursive: true }); + const first = startDaemon(); + first.exec( + `INSERT INTO ios_source_data (source_id, kind, spec_key, udid, set_path, device_type_id, runtime_id, + model, os_version, state, maker, created_at, indexed) + VALUES ('seed-1', 'seed', 'kind', '${udid}', '${join(home, "seeds")}', 'type', 'runtime', + 'iPhone 17', '27.0', 'ready', 'daemon', 0, 1); + INSERT INTO ios_source_files (source_id, relative_path, kind) + VALUES ('seed-1', 'data/Library/Logs', 'link')`, + ); + first.close(); + + await releaseLaneHome(home); + const second = startDaemon(); + second.close(); + await restoreKeptSeeds(home); + + const restored = new DatabaseSync(join(home, "state.db"), { readOnly: true }); + expect(restored.prepare("SELECT indexed FROM ios_source_data").all()).toEqual([ + { indexed: 1 }, + ]); + expect( + restored.prepare("SELECT source_id, relative_path, kind FROM ios_source_files").all(), + ).toEqual([{ kind: "link", relative_path: "data/Library/Logs", source_id: "seed-1" }]); + restored.close(); + } finally { + await rm(home, { force: true, recursive: true }); + } + }); +}); diff --git a/e2e/helpers/lane-home.ts b/e2e/helpers/lane-home.ts index fd98ae1c0..189b866a7 100644 --- a/e2e/helpers/lane-home.ts +++ b/e2e/helpers/lane-home.ts @@ -19,16 +19,21 @@ type Row = Record; interface KeptSeed { readonly table: string; readonly row: Row; + /** why: the rows other tables hold for the seed (ADR 0076: the files a copy swaps), kept and put back with it. */ + readonly children?: readonly { readonly table: string; readonly rows: Row[] }[]; } interface SeedTable { readonly table: string; + /** why: a table whose rows belong to a seed, and the column that holds the seed's `source_id`. */ + readonly children?: readonly string[]; folderOf(row: Row): string; } // why: a seed is a row and a folder (ADR 0061), and each driver names the folder in its own columns. const SEED_TABLES: readonly SeedTable[] = [ { + children: ["ios_source_files"], folderOf: (row) => join(String(row["set_path"]), String(row["udid"])), table: "ios_source_data", }, @@ -83,19 +88,26 @@ export async function restoreKeptSeeds(home: string): Promise { const seeds = JSON.parse(await readFile(join(home, KEPT_SEEDS_FILE), "utf8")) as KeptSeed[]; const database = new DatabaseSync(join(home, "state.db")); try { - for (const { table, row } of seeds) { - const columns = Object.keys(row); - database - .prepare( - `INSERT INTO ${table} (${columns.join(", ")}) VALUES (${columns.map(() => "?").join(", ")})`, - ) - .run(...columns.map((column) => row[column] ?? null)); + for (const { table, row, children } of seeds) { + insertRow(database, table, row); + for (const child of children ?? []) { + for (const childRow of child.rows) insertRow(database, child.table, childRow); + } } } finally { database.close(); } } +function insertRow(database: DatabaseSync, table: string, row: Row): void { + const columns = Object.keys(row); + database + .prepare( + `INSERT INTO ${table} (${columns.join(", ")}) VALUES (${columns.map(() => "?").join(", ")})`, + ) + .run(...columns.map((column) => row[column] ?? null)); +} + /** why: the driver's record of its adb server is the only handle on a server that outlives its daemon (it stays while an emulator is attached); removed with the rest, the flow's own daemon finds the port listening with no record and refuses Android as `occupied`. */ const ADB_SERVER_RECORD = "adb-server.json"; @@ -148,8 +160,17 @@ async function readySeeds(home: string): Promise { } const database = new DatabaseSync(path, { readOnly: true }); try { - return SEED_TABLES.flatMap(({ table }) => - readyRows(database, table).map((row) => ({ row, table })), + return SEED_TABLES.flatMap(({ table, children }) => + readyRows(database, table).map((row) => ({ + children: (children ?? []).map((child) => ({ + rows: database + .prepare(`SELECT * FROM ${child} WHERE source_id = ?`) + .all(String(row["source_id"])) as Row[], + table: child, + })), + row, + table, + })), ).filter((seed) => existsSync(folderOf(seed))); } finally { database.close(); From f27ce71c0748545fa9e466b2c138483ace6bcf30 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:43:49 +0100 Subject: [PATCH 15/48] test(e2e): the slow lane proves a lease writes only into its own folders, and its kept seed is copied (#1570) --- e2e/slow-agent-device.test.ts | 21 ++++ e2e/slow-ios-source-isolation.test.ts | 158 ++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 e2e/slow-ios-source-isolation.test.ts diff --git a/e2e/slow-agent-device.test.ts b/e2e/slow-agent-device.test.ts index ea1eaeda4..b5f6e1890 100644 --- a/e2e/slow-agent-device.test.ts +++ b/e2e/slow-agent-device.test.ts @@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto"; import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterAll, afterEach, @@ -144,6 +145,21 @@ function forLease(leaseId: string | undefined): (event: RecordedEvent) => boolea return (event) => (event.payload as { leaseId?: string }).leaseId === leaseId; } +/** why: the ready, indexed iOS seeds the flow's daemon holds; the lane put the earlier flows' back before the daemon served. */ +function keptIosSeeds(home: string): string[] { + const database = new DatabaseSync(join(home, "state.db"), { readOnly: true }); + try { + return database + .prepare( + "SELECT source_id FROM ios_source_data WHERE kind = 'seed' AND state = 'ready' AND indexed = 1", + ) + .all() + .map((row) => String(row["source_id"])); + } finally { + database.close(); + } +} + const sharedEnvs = new Set(); afterAll(async () => { await Promise.allSettled([...sharedEnvs].map((env) => env.teardown())); @@ -191,7 +207,12 @@ function defineSuite(target: Target): void { }); sharedEnvs.add(env); await awaitAgentDevice(); + const kept = keptIosSeeds(env.home); await warmUp(env, target, 1); + // why: a seed an earlier `keepSeeds` flow left, with its file rows, is copied by this flow; a seed made again has another id. + expect(keptIosSeeds(env.home), "the seeds the lane kept are copied, not made again").toEqual( + expect.arrayContaining(kept), + ); }, 420_000); afterAll(async () => { diff --git a/e2e/slow-ios-source-isolation.test.ts b/e2e/slow-ios-source-isolation.test.ts new file mode 100644 index 000000000..9bf60fa0f --- /dev/null +++ b/e2e/slow-ios-source-isolation.test.ts @@ -0,0 +1,158 @@ +import { existsSync } from "node:fs"; +import { lstat, readdir, readFile, readlink, stat } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join, relative } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { IOS } from "./helpers/agent-device-targets.js"; +import { withDaemon } from "./helpers/index.js"; +import type { TestEnv } from "./helpers/env.js"; +import { iosSourcesFolder, setDevices, sweepStaleDeviceSets } from "./helpers/ios-device-set.js"; +import { DEVICE_READY_MS } from "./helpers/wait.js"; + +// why: a copy writes its first minutes of state (home-screen snapshots, ManagedSettings records) into whatever folder its stored paths name, so each copy is used and left running this long before the sources are read. +const USE_MS = 45_000; +const DIAGNOSTICS = join("data", "var", "db", "diagnostics"); +const SWAP_MS_LIMIT = 1000; + +interface Lease { + readonly leaseId: string; + readonly udid: string; + readonly deviceSet: string; +} + +async function lease(env: TestEnv, extra: readonly string[]): Promise { + const granted = await env.cli( + ["lease", "--platform", "ios", "--agent-id", "isolation", "--detach", ...extra], + { timeout: DEVICE_READY_MS }, + ); + expect(granted.code, `lease failed: ${granted.stderr}`).toBe(0); + const grant = granted.json as { + device: { driverDeviceId: string }; + environment: Record; + lease: { id: string }; + }; + return { + deviceSet: grant.environment.SIMLOCK_IOS_DEVICE_SET as string, + leaseId: grant.lease.id, + udid: grant.device.driverDeviceId, + }; +} + +/** why: the copy runs an installed app and idles, as a lease does, so what it writes is what a lease's simulator writes. */ +async function use(env: TestEnv, held: Lease): Promise { + const installed = await env.cli(["simctl", "install", held.udid, IOS.app]); + expect(installed.code, `install failed: ${installed.stderr}`).toBe(0); + const started = await env.cli(["simctl", "launch", held.udid, IOS.bundleId]); + expect(started.code, `launch failed: ${started.stderr}`).toBe(0); + await new Promise((resolve) => setTimeout(resolve, USE_MS)); +} + +async function release(env: TestEnv, held: Lease): Promise { + const released = await env.cli(["release", held.leaseId]); + expect(released.code, `release failed: ${released.stderr}`).toBe(0); +} + +async function filesNewerThan(folder: string, since: number): Promise { + const found: string[] = []; + const walk = async (directory: string): Promise => { + for (const name of await readdir(directory)) { + const path = join(directory, name); + const details = await lstat(path); + if (relative(folder, path) === DIAGNOSTICS) continue; + if (details.isDirectory()) await walk(path); + else if (details.mtimeMs > since) found.push(relative(folder, path)); + } + }; + if (existsSync(folder)) await walk(folder); + return found; +} + +async function expectOwnLogFolder(held: Lease): Promise { + const target = await readlink(join(held.deviceSet, held.udid, "data", "Library", "Logs")); + expect(target, `${held.udid}'s Logs link`).toBe( + join(homedir(), "Library", "Logs", "CoreSimulator", held.udid), + ); +} + +async function swapMilliseconds(env: TestEnv, udid: string): Promise { + const log = await readFile(join(env.home, "daemon.log"), "utf8"); + const match = log + .split("\n") + .map((entry) => /swapped \d+ files and \d+ links in (\d+) ms for (\S+?)["\s]/.exec(entry)) + .find((found) => found?.[2] === udid); + expect(match, `the swap line of ${udid} in daemon.log`).toBeDefined(); + return Number(match?.[1]); +} + +// why: this lane needs the real simctl toolchain, hence darwin-only; it never installs a runtime. +describe.skipIf(process.platform !== "darwin")( + "iOS source isolation (real simctl)", + { tags: ["slow", "ios"] }, + () => { + it( + "three leases of one kind and two of a baseline saved from a used lease leave no file newer than its making in the seed, the baseline's source or the saved-from lease's folder, each simulator's Logs link names its own log folder, and each baseline copy's swap is logged under 1000 ms", + { timeout: 1_800_000 }, + async () => { + await sweepStaleDeviceSets(); + const env = await withDaemon({ + configOverrides: { log: { level: "debug" } }, + driver: "real", + }); + const sources = iosSourcesFolder(env.home); + + const first = await lease(env, []); + const seed = (await setDevices(sources)).find((device) => + device.name.startsWith("simlock-seed-"), + ); + expect(seed, "the seed made for the first lease").toBeDefined(); + const seedMadeAt = (await stat(join(sources, seed?.udid ?? "", "device.plist"))).mtimeMs; + await use(env, first); + await expectOwnLogFolder(first); + for (let index = 0; index < 2; index += 1) { + const copy = await lease(env, []); + await use(env, copy); + await expectOwnLogFolder(copy); + await release(env, copy); + } + + const saved = await env.cli(["baseline", "save", "isolation", "--lease", first.leaseId], { + timeout: 600_000, + }); + expect(saved.code, `baseline save failed: ${saved.stderr}`).toBe(0); + const savedAt = Date.now(); + const source = (await setDevices(sources)).find((device) => + device.name.startsWith("simlock-baseline-"), + ); + expect(source, "the baseline's source").toBeDefined(); + + const copies: Lease[] = []; + for (let index = 0; index < 2; index += 1) { + const copy = await lease(env, ["--baseline", "isolation"]); + copies.push(copy); + await use(env, copy); + await expectOwnLogFolder(copy); + await release(env, copy); + } + + expect( + await filesNewerThan(join(sources, seed?.udid ?? ""), seedMadeAt), + "the seed", + ).toEqual([]); + expect( + await filesNewerThan(join(sources, source?.udid ?? ""), savedAt), + "the baseline's source", + ).toEqual([]); + expect( + await filesNewerThan(join(first.deviceSet, first.udid), savedAt), + "the folder of the lease the baseline was saved from", + ).toEqual([]); + for (const copy of copies) { + expect(await swapMilliseconds(env, copy.udid), `${copy.udid}'s swap`).toBeLessThan( + SWAP_MS_LIMIT, + ); + } + }, + ); + }, +); From 5cfade049d25035540b0279a8f6585e41676e436 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:44:06 +0100 Subject: [PATCH 16/48] docs(ios): the manual says a source is flushed and indexed and a copy's folders are swapped (#1570) --- src/drivers/ios/AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 8f6a7c6a2..dc008ebe5 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -28,6 +28,8 @@ Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources folder (ADR 0076): `$SIMLOCK_HOME/devices/src` followed by `_` until its real path is as long as the device set's, an owned root under both settings. When none can fit, iOS is refused `sources-path-unfit`. A row naming another set is a seed made again, or a baseline stale (`BASELINE_STALE`); its simulator is deleted there only when the set is an earlier sources folder (`/sources`, `ios-sources`), else left and logged. It is in no `listManaged` or `simlock simctl list`. - Every clone (a seed's copy, `saveSource`'s own clone, a `provision` with `source`) has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app, system apps included, under the original device's folder. A clone whose drop fails is deleted. +- A source is flushed and indexed when it is made (ADR 0076 §2, `source-index.ts`): every SQLite WAL in it is checkpointed (one that cannot be is logged and left out), then each file and link that names any device or log folder, the source's own included, is recorded in `ios_source_files` and the row marked `indexed`, in the `markReady` step. A seed row without the mark is made again; a baseline without it is stale. +- Each copy of a source has those recorded files and links swapped before its first boot (`foreign-folders.ts`): every device or log folder of another simulator becomes the copy's own, byte for byte in each spelling (`/private` or not, `/` or `\/`), the length never changing. A result of another length, a missing file or a failed write deletes the copy, as a failed drop does. One debug line per copy says how long it took. A format that compresses or checksums a path (`.tracev3`) is not fixed. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. From ed86d6a8e0ab09c4161a097347af768d0cd89bdb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:46:10 +0100 Subject: [PATCH 17/48] refactor(ios): the copy steps live in source-copy.ts so index.ts stays within its listed length (#1570) --- src/daemon/state-owners.test.ts | 4 +- src/drivers/ios/index.ts | 126 ++++++-------------------------- src/drivers/ios/source-copy.ts | 90 +++++++++++++++++++++++ 3 files changed, 115 insertions(+), 105 deletions(-) create mode 100644 src/drivers/ios/source-copy.ts diff --git a/src/daemon/state-owners.test.ts b/src/daemon/state-owners.test.ts index c6c385faa..2a8411b5e 100644 --- a/src/daemon/state-owners.test.ts +++ b/src/daemon/state-owners.test.ts @@ -125,7 +125,7 @@ describe("the seed tables' one-ready-seed-per-kind index (#1367)", () => { ); it.each(seedTables)( - "$table: a state.db at version 1 moves to version 2 at open, keeping the newest ready seed of each kind (a tie kept by the larger source id), demoting the others to making, leaving baselines and making rows out of the comparison, and refusing a second ready seed afterwards", + "$table: a state.db at version 1 moves to the current version at open, keeping the newest ready seed of each kind (a tie kept by the larger source id), demoting the others to making, leaving baselines and making rows out of the comparison, and refusing a second ready seed afterwards", async ({ owner, table, insert }) => { const directory = await newDirectory(); const versionOne: SqliteSchemaOwner = { @@ -163,7 +163,7 @@ describe("the seed tables' one-ready-seed-per-kind index (#1367)", () => { state.database .prepare("SELECT version FROM schema_versions WHERE owner = ?") .get(owner.name), - ).toEqual({ version: 2 }); + ).toEqual({ version: owner.version }); expect( state.database .prepare(`SELECT source_id, kind, state FROM ${table} ORDER BY source_id`) diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index e7925f160..025e743c9 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -86,7 +86,13 @@ import { IosPhysicalHandler } from "./physical.js"; import { createIosDeviceInput } from "./input/index.js"; import { iosRuntimeVersionFromId } from "./runtime-version.js"; import { IosSourceTable, type ReadySeed, type SourceFile, type SourceRow } from "./source-data.js"; -import { swapForeignFolders } from "./foreign-folders.js"; +import { + copyOverrides, + dropLaunchServicesDatabase, + OVERRIDES_FILE_NAME, + swapCopyFolders, + type CopyContext, +} from "./source-copy.js"; import { indexSource } from "./source-index.js"; import { seedIsStale, sourceStaleReason } from "./source-staleness.js"; import { fitSourcesFolders, sourcesFolderOf, sourcesSetCheck } from "./sources-folder.js"; @@ -211,8 +217,6 @@ const RECLAIM_INSTEAD = // why: ADR 0060 §2: simslim's limit for a simulator just after its first boot. const READBACK_TIMEOUT_MS = 120_000; -const LAUNCH_SERVICES_DATABASE = /^com\.apple\.LaunchServices-.*\.csstore$/; -const OVERRIDES_FILE_NAME = "disabled.plist"; const DEFAULT_OVERRIDES_ROOT = "/private/var/tmp"; const OVERRIDES_FOLDER_PREFIX = "com.apple.CoreSimulator.SimDevice."; @@ -341,8 +345,7 @@ export class IosSimctlDriver implements Driver { readonly #sources: IosSourceTable; readonly #sourcesSet: string; readonly #ownsSourcesSet: (path: string) => boolean; - readonly #logsRoot: string | undefined; - readonly #sqlite: SqliteOpener | undefined; + readonly #copy: CopyContext; #sourceMaker: string | undefined; #swept = false; readonly #seedMakes = new Map>(); @@ -383,8 +386,13 @@ export class IosSimctlDriver implements Driver { this.#slimByDefault = options.slimByDefault === true; this.#sources = new IosSourceTable(options.database); this.#sourcesSet = sourcesSet; - this.#logsRoot = options.logsRoot; - this.#sqlite = options.sqlite; + this.#copy = { + clock: options.clock, + filesystem: options.filesystem, + folders: { deviceSet: deviceRoot, logsRoot: options.logsRoot, sourcesSet }, + logger: options.logger, + sqlite: options.sqlite, + }; this.#ownsSourcesSet = sourcesSetCheck( sourcesSet, options.simlockHome, @@ -896,9 +904,9 @@ export class IosSimctlDriver implements Driver { cloneUdid = cloned.stdout.trim(); if (cloneUdid === "") throw new DriverCrashError("simctl clone returned no device UDID"); this.#sources.setUdid(sourceId, cloneUdid); - await this.#copyOverrides(data.udid, cloneUdid); - await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); - const files = await this.#indexSource(cloneUdid); + await copyOverrides(this.#copy, (udid) => this.#overridesFolder(udid), data.udid, cloneUdid); + await dropLaunchServicesDatabase(this.#filesystem, this.#sourcesSet, cloneUdid); + const files = await indexSource(join(this.#sourcesSet, cloneUdid), this.#copy); const sizeBytes = await this.#sizeOf(cloneUdid); this.#sources.markReady(sourceId, sizeBytes, files); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; @@ -986,15 +994,15 @@ export class IosSimctlDriver implements Driver { try { await this.#writeMark(udid); - await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); - await this.#swapForeignFolders(seed.sourceId, udid); + await dropLaunchServicesDatabase(this.#filesystem, this.#deviceRoot, udid); + await swapCopyFolders(this.#copy, this.#sources.filesOf(seed.sourceId), udid); } catch (error: unknown) { // why: a clone whose mark, database drop or folder swap failed is not a device Simlock hands out, and // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. await this.#invokeSimctl(["delete", udid], COMMAND_TIMEOUT_MS).catch(() => undefined); throw error; } - await this.#copyOverrides(seed.udid, udid); + await copyOverrides(this.#copy, (folder) => this.#overridesFolder(folder), seed.udid, udid); return { address: udid, @@ -1088,15 +1096,10 @@ export class IosSimctlDriver implements Driver { if (error instanceof BootTimeoutError) throw error; throw new DriverCrashError(`seed ${udid} did not boot: ${errorMessage(error)}`); } - try { - await this.#shutdown(udid, this.#sourcesSet); - } catch (error: unknown) { - await this.#deleteSeedSimulator(udid); - throw error; - } let files: readonly SourceFile[]; try { - files = await this.#indexSource(udid); + await this.#shutdown(udid, this.#sourcesSet); + files = await indexSource(join(this.#sourcesSet, udid), this.#copy); } catch (error: unknown) { await this.#deleteSeedSimulator(udid); throw error; @@ -1144,89 +1147,6 @@ export class IosSimctlDriver implements Driver { return this.#sourceMaker; } - /** why: ADR 0060 §1: the slim file is in place before the copy's first boot, so the copy boots with what the seed was given. */ - async #copyOverrides(seedUdid: string, copyUdid: string): Promise { - const from = join(this.#overridesFolder(seedUdid), OVERRIDES_FILE_NAME); - try { - if (!(await this.#filesystem.exists(from))) return; - await this.#filesystem.mkdirp(this.#overridesFolder(copyUdid), { mode: 0o700 }); - await this.#filesystem.copyFile( - from, - join(this.#overridesFolder(copyUdid), OVERRIDES_FILE_NAME), - ); - } catch (error: unknown) { - this.#logger?.warn( - `could not copy the launchd overrides of seed ${seedUdid} to ${copyUdid}: ${errorMessage(error)}`, - ); - } - } - - /** - * why: `simctl clone` keeps the LaunchServices database, which records each app (system apps - * included) by its absolute path under the original device's folder; a copy then uses the - * original's data, or fails once that folder is gone. Dropped from every clone before its first - * boot, LaunchServices rebuilds it from the clone's own container folders (ADR 0061 §3). - */ - async #dropLaunchServicesDatabase(setPath: string, udid: string): Promise { - const folder = join(setPath, udid, "data", "var", "db", "lsd"); - if (!(await this.#filesystem.exists(folder))) return; - for (const entry of await this.#filesystem.readdir(folder)) { - if (LAUNCH_SERVICES_DATABASE.test(entry)) await this.#filesystem.rm(join(folder, entry)); - } - } - - #indexSource(udid: string): Promise { - return indexSource(join(this.#sourcesSet, udid), { - filesystem: this.#filesystem, - folders: { - deviceSet: this.#deviceRoot, - logsRoot: this.#logsRoot, - sourcesSet: this.#sourcesSet, - }, - logger: this.#logger, - sqlite: this.#sqlite, - }); - } - - /** - * why: ADR 0076 §3: a copy's files and links still name its source's device and log folders, and - * the copy would write into them; before its first boot each recorded one names the copy's own - * (a swap of the same length, so no format is parsed). A copy whose swap fails is deleted by the caller. - */ - async #swapForeignFolders(sourceId: string, copyUdid: string): Promise { - const folders = { - deviceSet: this.#deviceRoot, - logsRoot: this.#logsRoot, - ownUdid: copyUdid, - sourcesSet: this.#sourcesSet, - }; - const startedAt = this.#clock.now(); - let files = 0; - let links = 0; - for (const file of this.#sources.filesOf(sourceId)) { - const path = join(this.#deviceRoot, copyUdid, file.relativePath); - const before = - file.kind === "link" - ? Buffer.from(await this.#filesystem.readlink(path)) - : await this.#filesystem.readBytes(path); - const after = swapForeignFolders(before, folders); - if (after.length !== before.length) { - throw new DriverCrashError(`swapping the folders in ${path} would change its length`); - } - if (Buffer.compare(before, after) === 0) continue; - if (file.kind === "link") { - await this.#filesystem.replaceSymlink(path, Buffer.from(after).toString()); - links += 1; - } else { - await this.#filesystem.overwriteSameLength(path, after); - files += 1; - } - } - this.#logger?.debug( - `swapped ${String(files)} files and ${String(links)} links in ${String(this.#clock.now() - startedAt)} ms for ${copyUdid}`, - ); - } - async #destroySeed(seed: ReadySeed): Promise { await this.#deleteSeedInTurn(seed.udid, seed.setPath); this.#sources.deleteRow(seed.sourceId); diff --git a/src/drivers/ios/source-copy.ts b/src/drivers/ios/source-copy.ts new file mode 100644 index 000000000..37ee93bc7 --- /dev/null +++ b/src/drivers/ios/source-copy.ts @@ -0,0 +1,90 @@ +import { join } from "node:path"; + +import { DriverCrashError } from "../../core/index.js"; +import type { Clock, Filesystem } from "../../ports/index.js"; +import { swapForeignFolders } from "./foreign-folders.js"; +import type { SourceFile } from "./source-data.js"; +import type { SourceIndexOptions } from "./source-index.js"; + +export const OVERRIDES_FILE_NAME = "disabled.plist"; +const LAUNCH_SERVICES_DATABASE = /^com\.apple\.LaunchServices-.*\.csstore$/; + +export interface CopyContext extends SourceIndexOptions { + readonly clock: Clock; +} + +/** + * why: `simctl clone` keeps the LaunchServices database, which records each app (system apps + * included) by its absolute path under the original device's folder; a copy then uses the + * original's data, or fails once that folder is gone. Dropped from every clone before its first + * boot, LaunchServices rebuilds it from the clone's own container folders (ADR 0061 §3). + */ +export async function dropLaunchServicesDatabase( + filesystem: Filesystem, + setPath: string, + udid: string, +): Promise { + const folder = join(setPath, udid, "data", "var", "db", "lsd"); + if (!(await filesystem.exists(folder))) return; + for (const entry of await filesystem.readdir(folder)) { + if (LAUNCH_SERVICES_DATABASE.test(entry)) await filesystem.rm(join(folder, entry)); + } +} + +/** + * why: ADR 0076 §3: a copy's files and links still name its source's device and log folders, and + * the copy would write into them; before its first boot each recorded one names the copy's own + * (a swap of the same length, so no format is parsed). A copy whose swap fails is deleted by the caller. + */ +export async function swapCopyFolders( + context: CopyContext, + recorded: readonly SourceFile[], + copyUdid: string, +): Promise { + const { filesystem, folders } = context; + const startedAt = context.clock.now(); + let files = 0; + let links = 0; + for (const file of recorded) { + const path = join(folders.deviceSet, copyUdid, file.relativePath); + const before = + file.kind === "link" + ? Buffer.from(await filesystem.readlink(path)) + : await filesystem.readBytes(path); + const after = swapForeignFolders(before, { ...folders, ownUdid: copyUdid }); + if (after.length !== before.length) { + throw new DriverCrashError(`swapping the folders in ${path} would change its length`); + } + if (Buffer.compare(before, after) === 0) continue; + if (file.kind === "link") { + await filesystem.replaceSymlink(path, Buffer.from(after).toString()); + links += 1; + } else { + await filesystem.overwriteSameLength(path, after); + files += 1; + } + } + context.logger?.debug( + `swapped ${String(files)} files and ${String(links)} links in ${String(context.clock.now() - startedAt)} ms for ${copyUdid}`, + ); +} + +/** why: ADR 0060 §1: the slim file is in place before the copy's first boot, so the copy boots with what the seed was given. */ +export async function copyOverrides( + context: Pick, + folderOf: (udid: string) => string, + seedUdid: string, + copyUdid: string, +): Promise { + const { filesystem } = context; + const from = join(folderOf(seedUdid), OVERRIDES_FILE_NAME); + try { + if (!(await filesystem.exists(from))) return; + await filesystem.mkdirp(folderOf(copyUdid), { mode: 0o700 }); + await filesystem.copyFile(from, join(folderOf(copyUdid), OVERRIDES_FILE_NAME)); + } catch (error: unknown) { + context.logger?.warn( + `could not copy the launchd overrides of seed ${seedUdid} to ${copyUdid}: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} From 36822e6d2df86ff0e2b3709a340f0efeb5cb5b69 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 21:51:30 +0100 Subject: [PATCH 18/48] test(ios): a copy's unchanged recorded file is not counted, and a made-again seed's file rows are deleted (#1570) --- src/drivers/ios/index.test.ts | 27 ++++++++++++++---------- src/drivers/ios/source-isolation.test.ts | 7 +++++- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/drivers/ios/index.test.ts b/src/drivers/ios/index.test.ts index 6a0d8f6ff..b80b7148e 100644 --- a/src/drivers/ios/index.test.ts +++ b/src/drivers/ios/index.test.ts @@ -231,6 +231,7 @@ function sourcesSimctl(...args: readonly string[]): { /** why: A database that already holds the seed of the kind `spec` names, so a test of the copy step starts at the clone. */ async function seededDatabase( seed: { + readonly files?: readonly { readonly kind: "link" | "file"; readonly relativePath: string }[]; readonly indexed?: false; readonly runtimeBuild?: string; readonly setPath?: string; @@ -1637,17 +1638,15 @@ describe("IosSimctlDriver", () => { ]); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); - const driver = await createDriver( - runner, - new FakeClock(), - filesystem, - await seededDatabase({ - setPath, - udid: planted, - ...("indexed" in planting ? { indexed: planting.indexed } : {}), - ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), - }), - ); + const database = await seededDatabase({ + // why: a seed made again takes its file rows with it; a kept one's would be swapped in the copy, which holds no such file here. + ...(remade ? { files: [{ kind: "file", relativePath: "data/prefs.bin" }] as const } : {}), + setPath, + udid: planted, + ...("indexed" in planting ? { indexed: planting.indexed } : {}), + ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), + }); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); await driver.resolveSpec({ model: "iPhone 16", osVersion: "26.5", platform: "ios" }); await driver.provision(spec); @@ -1657,6 +1656,12 @@ describe("IosSimctlDriver", () => { ? ["shutdown", "delete", "create", "boot", "bootstatus", "listapps", "shutdown", "clone"] : ["clone"], ); + expect( + database + .prepare("SELECT source_id FROM ios_source_files WHERE relative_path = 'data/prefs.bin'") + .all(), + "the made-again seed's old file rows", + ).toEqual([]); }, ); diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts index 3a5b57596..82686c537 100644 --- a/src/drivers/ios/source-isolation.test.ts +++ b/src/drivers/ios/source-isolation.test.ts @@ -130,6 +130,7 @@ describe("a lease's copy before its first boot", () => { { kind: "link", relativePath: "data/Library/Logs" }, { kind: "file", relativePath: "data/prefs.bin" }, { kind: "file", relativePath: "data/Media/state.json" }, + { kind: "file", relativePath: "data/own.txt" }, ], async (fs) => { await fs.mkdirp(`${copy}/data/Library`); @@ -143,6 +144,7 @@ describe("a lease's copy before its first boot", () => { `${copy}/data/Media/state.json`, `{"from":"${deviceRoot}/${usedUdid}/data"}`, ); + await fs.writeFileAtomic(`${copy}/data/own.txt`, `${deviceRoot}/${copyUdid}/data`); await fs.writeFileAtomic(`${copy}/data/unrecorded.txt`, `${sourcesRoot}/${seedUdid}`); }, ); @@ -159,11 +161,14 @@ describe("a lease's copy before its first boot", () => { await rig.filesystem.readFile(`${copy}/data/Media/state.json`), "a saved-from folder", ).toBe(`{"from":"${deviceRoot}/${copyUdid}/data"}`); + expect(await rig.filesystem.readFile(`${copy}/data/own.txt`), "a path naming the copy").toBe( + `${deviceRoot}/${copyUdid}/data`, + ); expect( await rig.filesystem.readFile(`${copy}/data/unrecorded.txt`), "only recorded files", ).toBe(`${sourcesRoot}/${seedUdid}`); - expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 14 ms for ${copyUdid}`); + expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 21 ms for ${copyUdid}`); }); it("refuses a file whose swap would change its length", async () => { From 4deefd75e5fa571af6267878c6ac927c4539c732 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 22:08:55 +0100 Subject: [PATCH 19/48] fix(ios): a WAL's flush is listed again, the swap names the real device set, and the manual and messages say what is true (#1570) --- src/core/AGENTS.md | 2 +- src/drivers/ios/AGENTS.md | 9 ++-- src/drivers/ios/foreign-folders.test.ts | 2 +- src/drivers/ios/index.ts | 18 ++++---- src/drivers/ios/source-index.ts | 6 +-- src/drivers/ios/source-isolation.test.ts | 53 ++++++++++++++++++++++-- src/drivers/ios/source-staleness.ts | 5 ++- src/drivers/ios/sources-folder.test.ts | 2 +- src/drivers/ios/sources-folder.ts | 15 +++++++ src/drivers/ios/testing.ts | 9 ++-- 10 files changed, 93 insertions(+), 28 deletions(-) diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index c2db1d6d5..b1155460d 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -58,7 +58,7 @@ Baselines (ADR 0061 §4): - One save or remove at a time. A save: leasing's `releaseToSave` ends the lease, then `ReclaimCoordinator.retireSaved` calls `driver.saveSource` under the `reclaim` claim. A driver failure quarantines the device and answers `BASELINE_SAVE_FAILED`. - A replace keeps the leased devices of older versions. An older version is dropped only when no device record and no provision (`hold`) has it; that is checked after a save, on every `device.deleted` and at startup. - `remove` refuses a baseline a lease uses, and destroys each source before its row. `removeAll` (nuke) deletes rows first. -- `checkStale` runs once at startup (after the sources are listed, before the warm pool starts), before `list` and `doctor`, and after `component.removed`. +- `checkStale` runs once at startup, before `list` and `doctor`, and after `component.removed`. ## Known gaps diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index dc008ebe5..4eae19b81 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -9,7 +9,7 @@ The device set: - Under `drivers.ios.deviceSet: simlock`, never `simctl` without `--set`, except three unscoped calls: `findLegacy`, `destroyLegacy`, and the read-only `simctl list -j devices` of `listComponents`. - Under `default` (ADR 0048) the set is Xcode's own: never `--set`, no owned root, and never act on a device whose durable mark does not name this instance. `listManaged` lists only those, and `findLegacy` answers `undefined`. -- `ios.simulator.headless: false` needs `default`: the `simlock` set has no DeviceHub, so iOS is refused (`invalid-device-set`). Simulators left in the set being given up, or in either setting's sources folder, refuse iOS too (`device-set-changed`); that check reads files only, never `simctl`. +- `ios.simulator.headless: false` needs `default`: the `simlock` set has no DeviceHub, so iOS is refused (`invalid-device-set`). Simulators left in the set being given up or in a sources folder refuse iOS too (`device-set-changed`); that check reads files only, never `simctl`. - The erasable mark is written before the durable one, never both at once. Under `default` a failed mark write deletes the new simulator before the error is rethrown. Resolve and boot: @@ -26,10 +26,9 @@ Resolve and boot: Seeds (ADR 0061): -- A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources folder (ADR 0076): `$SIMLOCK_HOME/devices/src` followed by `_` until its real path is as long as the device set's, an owned root under both settings. When none can fit, iOS is refused `sources-path-unfit`. A row naming another set is a seed made again, or a baseline stale (`BASELINE_STALE`); its simulator is deleted there only when the set is an earlier sources folder (`/sources`, `ios-sources`), else left and logged. It is in no `listManaged` or `simlock simctl list`. -- Every clone (a seed's copy, `saveSource`'s own clone, a `provision` with `source`) has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app, system apps included, under the original device's folder. A clone whose drop fails is deleted. -- A source is flushed and indexed when it is made (ADR 0076 §2, `source-index.ts`): every SQLite WAL in it is checkpointed (one that cannot be is logged and left out), then each file and link that names any device or log folder, the source's own included, is recorded in `ios_source_files` and the row marked `indexed`, in the `markReady` step. A seed row without the mark is made again; a baseline without it is stale. -- Each copy of a source has those recorded files and links swapped before its first boot (`foreign-folders.ts`): every device or log folder of another simulator becomes the copy's own, byte for byte in each spelling (`/private` or not, `/` or `\/`), the length never changing. A result of another length, a missing file or a failed write deletes the copy, as a failed drop does. One debug line per copy says how long it took. A format that compresses or checksums a path (`.tracev3`) is not fixed. +- A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources folder (ADR 0076): `$SIMLOCK_HOME/devices/src` padded with `_` to the device set's real-path length, an owned root under both settings. A row naming another set is a seed made again, or a baseline stale (`BASELINE_STALE`), deleted there only in an earlier sources folder. It is in no `listManaged` or `simlock simctl list`. +- Every clone has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app, system apps included, under the original device's folder. A clone whose drop fails is deleted. +- Each copy has the files and links its source's index records swapped before its first boot (ADR 0076): another simulator's device or log folder becomes the copy's own, same spelling and length; a failed swap deletes the copy. An unindexed seed is made again, a baseline stale. `.tracev3` is not fixed. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. diff --git a/src/drivers/ios/foreign-folders.test.ts b/src/drivers/ios/foreign-folders.test.ts index 1d200c4e6..390bcb6fc 100644 --- a/src/drivers/ios/foreign-folders.test.ts +++ b/src/drivers/ios/foreign-folders.test.ts @@ -12,7 +12,7 @@ const seed = "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA"; const usedCopy = "BBBBBBBB-BBBB-BBBB-BBBB-BBBBBBBBBBBB"; const own = folders.ownUdid; -// why: the swap is a pure function with many byte-level cases (spellings, separators, binary neighbours) that every driver verb leans on; the driver's own test proves the files and links it is applied to. +// why: the swap is a pure function with many byte-level cases (spellings, separators, binary neighbours) that a `provision` leans on; the driver's own test proves the files and links it is applied to. describe("swapForeignFolders", () => { it.each([ { diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 025e743c9..4f70e26ab 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -95,7 +95,7 @@ import { } from "./source-copy.js"; import { indexSource } from "./source-index.js"; import { seedIsStale, sourceStaleReason } from "./source-staleness.js"; -import { fitSourcesFolders, sourcesFolderOf, sourcesSetCheck } from "./sources-folder.js"; +import { fitSourcesFolders, sourcesLayoutOf, sourcesSetCheck } from "./sources-folder.js"; import { createIosScreenCapture } from "./screen-capture.js"; import { defaultIosDeviceRoot, type IosDriverConfig } from "./definition.js"; import { @@ -355,7 +355,7 @@ export class IosSimctlDriver implements Driver { options: IosSimctlDriverOptions, deviceRoot: string, rootOptions: EnsureOwnedRootOptions | undefined, - sourcesSet: string, + sourceFolders: CopyContext["folders"], ) { this.rebootReadyBeforeGrant = rootOptions === undefined; this.#clock = options.clock; @@ -385,16 +385,16 @@ export class IosSimctlDriver implements Driver { this.#afterBoot = this.#afterBootFor(options); this.#slimByDefault = options.slimByDefault === true; this.#sources = new IosSourceTable(options.database); - this.#sourcesSet = sourcesSet; + this.#sourcesSet = sourceFolders.sourcesSet; this.#copy = { clock: options.clock, filesystem: options.filesystem, - folders: { deviceSet: deviceRoot, logsRoot: options.logsRoot, sourcesSet }, + folders: { ...sourceFolders, logsRoot: options.logsRoot }, logger: options.logger, sqlite: options.sqlite, }; this.#ownsSourcesSet = sourcesSetCheck( - sourcesSet, + sourceFolders.sourcesSet, options.simlockHome, rootOptions === undefined ? undefined : deviceRoot, ); @@ -443,8 +443,8 @@ export class IosSimctlDriver implements Driver { ...(await fitSourcesFolders(options, ownRoot, () => xcodeDeviceSetPath(options))), ]); const devices = xcodeDeviceSetPath(options); - const folder = await sourcesFolderOf(options, devices); - return new IosSimctlDriver(options, devices, undefined, folder); + const layout = await sourcesLayoutOf(options, devices); + return new IosSimctlDriver(options, devices, undefined, layout); } await refuseDefaultSourcesLeftovers(options.filesystem, [ join(options.simlockHome, "devices", "ios-sources"), @@ -472,8 +472,8 @@ export class IosSimctlDriver implements Driver { }; const deviceRoot = await ensureOwnedRoot(rootOptions); - const folder = await sourcesFolderOf(options, deviceRoot, true); - return new IosSimctlDriver(options, deviceRoot, rootOptions, folder); + const layout = await sourcesLayoutOf(options, deviceRoot, true); + return new IosSimctlDriver(options, deviceRoot, rootOptions, layout); } get deviceRoot(): string { diff --git a/src/drivers/ios/source-index.ts b/src/drivers/ios/source-index.ts index a4320b0e4..321286467 100644 --- a/src/drivers/ios/source-index.ts +++ b/src/drivers/ios/source-index.ts @@ -27,17 +27,17 @@ export async function indexSource( root: string, options: SourceIndexOptions, ): Promise { - const entries = await walk(options.filesystem, root, ""); const unflushed = new Set(); - for (const entry of entries) { + for (const entry of await walk(options.filesystem, root, "")) { if (entry.kind === "file" && entry.path.endsWith(WAL_SUFFIX)) { if (!(await flush(entry.path.slice(0, -WAL_SUFFIX.length), options))) { unflushed.add(entry.path); } } } + // why: closing a flushed database deletes its `-wal` and `-shm`, so the files are listed again after the flush. const indexed: SourceFile[] = []; - for (const entry of entries) { + for (const entry of await walk(options.filesystem, root, "")) { if (unflushed.has(entry.path)) continue; const bytes = entry.kind === "link" diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts index 82686c537..738f88bd6 100644 --- a/src/drivers/ios/source-isolation.test.ts +++ b/src/drivers/ios/source-isolation.test.ts @@ -31,6 +31,17 @@ class SlowReadFilesystem extends MemoryFilesystem { failWrites = false; + /** why: the memory filesystem follows no link above a path's last part, so a home reached through a link is a prefix this maps to its real path. */ + alias: { readonly from: string; readonly to: string } | undefined; + + override async realpath(path: string): Promise { + const alias = this.alias; + if (alias !== undefined && path.startsWith(`${alias.from}/`)) { + return `${alias.to}${path.slice(alias.from.length)}`; + } + return super.realpath(path); + } + override async overwriteSameLength(path: string, bytes: Uint8Array): Promise { if (this.failWrites) throw new Error("disk full"); await super.overwriteSameLength(path, bytes); @@ -67,20 +78,33 @@ async function rigWith( root: string, files: readonly { readonly relativePath: string; readonly kind: "link" | "file" }[], plant: (filesystem: SlowReadFilesystem, copy: string) => Promise, + layout: { + readonly home: string; + readonly sources: string; + readonly alias?: { readonly from: string; readonly to: string }; + readonly real?: string; + } = { + home: "/home/.simlock", + sources: sourcesRoot, + }, ): Promise { const clock = new FakeClock(); const filesystem = new SlowReadFilesystem(clock); const logger = new LineLogger(); - await filesystem.mkdirp(`${root}/${copyUdid}`); + const planted = layout.real ?? root; + filesystem.alias = layout.alias; + await filesystem.mkdirp(`${planted}/${copyUdid}`); + await filesystem.mkdirp(`${root}/${copyUdid}/data`); await filesystem.writeFileAtomic( `${root}/${OWNED_ROOT_MARKER_FILE}`, JSON.stringify({ instanceId, owner: "simlock", platform: "ios", schemaVersion: 1 }), ); - await plant(filesystem, `${root}/${copyUdid}`); + await plant(filesystem, `${planted}/${copyUdid}`); const database = plantReadySeed(await memorySourceDatabase(), { deviceTypeId: driverData.deviceTypeId, files, runtimeId: driverData.runtimeId, + setPath: layout.sources, udid: seedUdid, }); const runner = new ScriptedProcessRunner([ @@ -93,7 +117,7 @@ async function rigWith( }, { match: { - args: ["simctl", "--set", sourcesRoot, "clone", seedUdid, driverData.name, root], + args: ["simctl", "--set", layout.sources, "clone", seedUdid, driverData.name, root], command: "xcrun", }, result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, @@ -110,7 +134,7 @@ async function rigWith( logger, logsRoot, processRunner: runner, - simlockHome: "/home/.simlock", + simlockHome: layout.home, }); await driver.resolveSpec(spec); return { @@ -171,6 +195,27 @@ describe("a lease's copy before its first boot", () => { expect(rig.logger.lines).toContain(`swapped 2 files and 1 links in 21 ms for ${copyUdid}`); }); + it("swaps by real paths when SIMLOCK_HOME is reached through a link", async () => { + const root = "/tmp/h/devices/dev-ios"; + const real = "/private/tmp/h/devices/dev-ios"; + const realSources = "/private/tmp/h/devices/src____"; + const rig = await rigWith( + root, + [{ kind: "file", relativePath: "data/prefs.bin" }], + async (fs, copy) => { + await fs.mkdirp(`${copy}/data`); + await fs.writeFileAtomic(`${copy}/data/prefs.bin`, `${realSources}/${seedUdid}/data`); + }, + { alias: { from: "/tmp", to: "/private/tmp" }, home: "/tmp/h", real, sources: realSources }, + ); + + await rig.provision(); + + expect(await rig.filesystem.readFile(`${real}/${copyUdid}/data/prefs.bin`)).toBe( + `${real}/${copyUdid}/data`, + ); + }); + it("refuses a file whose swap would change its length", async () => { const copy = `${deepRoot}/${copyUdid}`; const escaped = (root: string, udid: string): string => diff --git a/src/drivers/ios/source-staleness.ts b/src/drivers/ios/source-staleness.ts index 538ae64cb..d69a5d719 100644 --- a/src/drivers/ios/source-staleness.ts +++ b/src/drivers/ios/source-staleness.ts @@ -21,9 +21,12 @@ export function sourceStaleReason( catalog: SimctlCatalog, sourcesSet: string, ): string | undefined { - if (row.setPath !== sourcesSet || !row.indexed) { + if (row.setPath !== sourcesSet) { return `saved in ${row.setPath}, which copies cannot be fixed from; save it again`; } + if (!row.indexed) { + return "saved without an index of its files, which copies cannot be fixed from; save it again"; + } const runtime = catalog.runtimes.find((entry) => entry.identifier === row.runtimeId); if (runtime === undefined) return `runtime ${row.runtimeId} is not installed`; if ( diff --git a/src/drivers/ios/sources-folder.test.ts b/src/drivers/ios/sources-folder.test.ts index 332a6ae87..c77000ca3 100644 --- a/src/drivers/ios/sources-folder.test.ts +++ b/src/drivers/ios/sources-folder.test.ts @@ -239,7 +239,7 @@ describe("IosSimctlDriver sources folder", () => { expect(listed.map((source) => source.staleReason)).toEqual([ `saved in ${oldSet}, which copies cannot be fixed from; save it again`, undefined, - `saved in ${sourcesRoot}, which copies cannot be fixed from; save it again`, + "saved without an index of its files, which copies cannot be fixed from; save it again", ]); const oldId = listed[0]?.id ?? ""; await driver.destroySource(oldId); diff --git a/src/drivers/ios/sources-folder.ts b/src/drivers/ios/sources-folder.ts index 9951852b3..c2c839a25 100644 --- a/src/drivers/ios/sources-folder.ts +++ b/src/drivers/ios/sources-folder.ts @@ -2,6 +2,7 @@ import { basename, dirname, join } from "node:path"; import { OwnedRootError } from "../../core/index.js"; import type { Filesystem } from "../../ports/index.js"; +import type { ForeignFolders } from "./foreign-folders.js"; const SOURCES_FOLDER_NAME = "src"; @@ -108,6 +109,20 @@ export function sourcesFolderOf( }); } +export type SourcesLayout = Pick; + +/** why: a file stores the real path of the device set (ADR 0076 §1), so the swap names a copy's folder by it, never by the path the set was configured or found under. */ +export async function sourcesLayoutOf( + options: SourcesFolderContext, + deviceSet: string, + configured = false, +): Promise { + return { + deviceSet: await realPathOf(options.filesystem, deviceSet), + sourcesSet: await sourcesFolderOf(options, deviceSet, configured), + }; +} + /** * why: the other setting's sources folder only matters when it can exist: where its length is unfit, nothing was ever made there, and it is read to nothing. * A set of `todaySet` whose folder is the same one (a device set of exactly the home's length plus 12) is today's own, never a leftover. diff --git a/src/drivers/ios/testing.ts b/src/drivers/ios/testing.ts index ad3f556ee..40c52a5d7 100644 --- a/src/drivers/ios/testing.ts +++ b/src/drivers/ios/testing.ts @@ -61,7 +61,7 @@ export function plantReadySeed( return plantSource(database, seed); } -/** why: stands in for the opener that flushes a source's databases: a checkpoint empties the database's `-wal` file in the filesystem, as SQLite's does, and a path in `failing` cannot be opened. */ +/** why: stands in for the opener that flushes a source's databases: closing a checkpointed database deletes its `-wal` and `-shm` files, as SQLite does, and a path in `failing` cannot be opened. */ export class FakeSqliteOpener implements SqliteOpener { readonly flushed: string[] = []; readonly failing = new Set(); @@ -75,11 +75,14 @@ export class FakeSqliteOpener implements SqliteOpener { if (options.readOnly) throw new Error(`${path} is opened read-write to flush it`); if (this.failing.has(path)) throw new Error("database is locked"); return { - close: () => undefined, + close: () => { + if (this.flushed.at(-1) !== path) return; + void this.#filesystem.rm(`${path}-wal`); + void this.#filesystem.rm(`${path}-shm`); + }, exec: (sql) => { if (sql !== "PRAGMA wal_checkpoint(TRUNCATE)") throw new Error(`unexpected ${sql}`); this.flushed.push(path); - void this.#filesystem.writeFileAtomic(`${path}-wal`, ""); }, prepare: () => { throw new Error("a flush runs one pragma and nothing else"); From bce83b1886ded32c1c1fc387be1120bc7a1101c6 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Sun, 15 Nov 2026 22:09:16 +0100 Subject: [PATCH 20/48] chore(ios): sourcesFolderOf is private to sources-folder.ts (#1570) --- src/drivers/ios/sources-folder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/ios/sources-folder.ts b/src/drivers/ios/sources-folder.ts index c2c839a25..ff0372a77 100644 --- a/src/drivers/ios/sources-folder.ts +++ b/src/drivers/ios/sources-folder.ts @@ -95,7 +95,7 @@ export interface SourcesFolderContext { } /** why: `configured` is true only for `simlock`, where a `deviceRoot` the operator set is what the refusal names. */ -export function sourcesFolderOf( +function sourcesFolderOf( options: SourcesFolderContext, deviceSet: string, configured = false, From bfb9b90b521eef3c21edc312f5b2ba11f9e2241f Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 00:14:57 +0100 Subject: [PATCH 21/48] fix(ios): a source's index skips unreadable and huge files and the clone-dropped database, and a swap matches whole paths only (#1570) The slow lane showed four faults: macOS refuses open on locationd's files (EPERM), a used simulator holds files past one string, the index recorded a file every clone deletes, and a stray backslash before a plain path made a match of mixed separators whose swap changed a length. The lane's test now uses one agent id per lease and reads the seed's making as its newest file after the first lease. --- e2e/slow-ios-source-isolation.test.ts | 80 ++++++++++++------- src/drivers/ios/AGENTS.md | 2 +- src/drivers/ios/foreign-folders.test.ts | 5 ++ src/drivers/ios/foreign-folders.ts | 12 ++- src/drivers/ios/source-copy.ts | 7 +- src/drivers/ios/source-index.ts | 46 +++++++++-- src/drivers/ios/source-isolation.test.ts | 98 ++++++++++++++++++++++-- src/ports/filesystem.test.ts | 4 + src/ports/filesystem.ts | 10 ++- 9 files changed, 217 insertions(+), 47 deletions(-) diff --git a/e2e/slow-ios-source-isolation.test.ts b/e2e/slow-ios-source-isolation.test.ts index 9bf60fa0f..7a2634ab4 100644 --- a/e2e/slow-ios-source-isolation.test.ts +++ b/e2e/slow-ios-source-isolation.test.ts @@ -1,5 +1,5 @@ import { existsSync } from "node:fs"; -import { lstat, readdir, readFile, readlink, stat } from "node:fs/promises"; +import { lstat, readdir, readFile, readlink } from "node:fs/promises"; import { homedir } from "node:os"; import { join, relative } from "node:path"; import { describe, expect, it } from "vitest"; @@ -21,9 +21,21 @@ interface Lease { readonly deviceSet: string; } +// why: a requester holds one lease at a time, and the first lease stays held while the others are made. +let leasesMade = 0; + async function lease(env: TestEnv, extra: readonly string[]): Promise { + leasesMade += 1; const granted = await env.cli( - ["lease", "--platform", "ios", "--agent-id", "isolation", "--detach", ...extra], + [ + "lease", + "--platform", + "ios", + "--agent-id", + `isolation-${String(leasesMade)}`, + "--detach", + ...extra, + ], { timeout: DEVICE_READY_MS }, ); expect(granted.code, `lease failed: ${granted.stderr}`).toBe(0); @@ -53,21 +65,32 @@ async function release(env: TestEnv, held: Lease): Promise { expect(released.code, `release failed: ${released.stderr}`).toBe(0); } -async function filesNewerThan(folder: string, since: number): Promise { - const found: string[] = []; +async function modifiedFiles(folder: string): Promise> { + const found = new Map(); const walk = async (directory: string): Promise => { for (const name of await readdir(directory)) { const path = join(directory, name); const details = await lstat(path); if (relative(folder, path) === DIAGNOSTICS) continue; if (details.isDirectory()) await walk(path); - else if (details.mtimeMs > since) found.push(relative(folder, path)); + else found.set(relative(folder, path), details.mtimeMs); } }; if (existsSync(folder)) await walk(folder); return found; } +async function filesNewerThan(folder: string, since: number): Promise { + return [...(await modifiedFiles(folder))] + .filter(([, time]) => time > since) + .map(([path]) => path); +} + +/** why: a seed is made when its WALs are flushed and its files indexed, which is after `simctl` wrote its last file; the newest file once the first lease is granted is that moment. */ +async function newestModification(folder: string): Promise { + return Math.max(0, ...(await modifiedFiles(folder)).values()); +} + async function expectOwnLogFolder(held: Lease): Promise { const target = await readlink(join(held.deviceSet, held.udid, "data", "Library", "Logs")); expect(target, `${held.udid}'s Logs link`).toBe( @@ -75,6 +98,19 @@ async function expectOwnLogFolder(held: Lease): Promise { ); } +/** why: a lease used and released, as an agent's would be; the returned lease is the one released. */ +async function useAndRelease(env: TestEnv, extra: readonly string[]): Promise { + const copy = await lease(env, extra); + await use(env, copy); + await expectOwnLogFolder(copy); + await release(env, copy); + return copy; +} + +async function expectNothingNewer(folder: string, since: number, what: string): Promise { + expect(await filesNewerThan(folder, since), what).toEqual([]); +} + async function swapMilliseconds(env: TestEnv, udid: string): Promise { const log = await readFile(join(env.home, "daemon.log"), "utf8"); const match = log @@ -106,15 +142,10 @@ describe.skipIf(process.platform !== "darwin")( device.name.startsWith("simlock-seed-"), ); expect(seed, "the seed made for the first lease").toBeDefined(); - const seedMadeAt = (await stat(join(sources, seed?.udid ?? "", "device.plist"))).mtimeMs; + const seedMadeAt = await newestModification(join(sources, seed?.udid ?? "")); await use(env, first); await expectOwnLogFolder(first); - for (let index = 0; index < 2; index += 1) { - const copy = await lease(env, []); - await use(env, copy); - await expectOwnLogFolder(copy); - await release(env, copy); - } + for (let index = 0; index < 2; index += 1) await useAndRelease(env, []); const saved = await env.cli(["baseline", "save", "isolation", "--lease", first.leaseId], { timeout: 600_000, @@ -128,25 +159,20 @@ describe.skipIf(process.platform !== "darwin")( const copies: Lease[] = []; for (let index = 0; index < 2; index += 1) { - const copy = await lease(env, ["--baseline", "isolation"]); - copies.push(copy); - await use(env, copy); - await expectOwnLogFolder(copy); - await release(env, copy); + copies.push(await useAndRelease(env, ["--baseline", "isolation"])); } - expect( - await filesNewerThan(join(sources, seed?.udid ?? ""), seedMadeAt), - "the seed", - ).toEqual([]); - expect( - await filesNewerThan(join(sources, source?.udid ?? ""), savedAt), + await expectNothingNewer(join(sources, seed?.udid ?? ""), seedMadeAt, "the seed"); + await expectNothingNewer( + join(sources, source?.udid ?? ""), + savedAt, "the baseline's source", - ).toEqual([]); - expect( - await filesNewerThan(join(first.deviceSet, first.udid), savedAt), + ); + await expectNothingNewer( + join(first.deviceSet, first.udid), + savedAt, "the folder of the lease the baseline was saved from", - ).toEqual([]); + ); for (const copy of copies) { expect(await swapMilliseconds(env, copy.udid), `${copy.udid}'s swap`).toBeLessThan( SWAP_MS_LIMIT, diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 4eae19b81..c85d2a9d2 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -28,7 +28,7 @@ Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources folder (ADR 0076): `$SIMLOCK_HOME/devices/src` padded with `_` to the device set's real-path length, an owned root under both settings. A row naming another set is a seed made again, or a baseline stale (`BASELINE_STALE`), deleted there only in an earlier sources folder. It is in no `listManaged` or `simlock simctl list`. - Every clone has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app, system apps included, under the original device's folder. A clone whose drop fails is deleted. -- Each copy has the files and links its source's index records swapped before its first boot (ADR 0076): another simulator's device or log folder becomes the copy's own, same spelling and length; a failed swap deletes the copy. An unindexed seed is made again, a baseline stale. `.tracev3` is not fixed. +- Each copy has the files and links its source's index records swapped before its first boot (ADR 0076): another simulator's device or log folder becomes the copy's own, same spelling and length; a failed swap deletes the copy. An unindexed seed is made again, a baseline stale. `.tracev3` is not fixed. A file the system refuses to read (`EPERM`, as under `locationd`) or one past `MAX_INDEXED_BYTES` is left out of the index (the larger one logged), so it is never swapped. The LaunchServices database a clone drops is never indexed. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. diff --git a/src/drivers/ios/foreign-folders.test.ts b/src/drivers/ios/foreign-folders.test.ts index 390bcb6fc..bc2b72978 100644 --- a/src/drivers/ios/foreign-folders.test.ts +++ b/src/drivers/ios/foreign-folders.test.ts @@ -40,6 +40,11 @@ describe("swapForeignFolders", () => { before: `{"p":"\\/private\\/tmp\\/h\\/devices\\/src\\/${seed}\\/x"}`, after: `{"p":"\\/private\\/tmp\\/h\\/devices\\/ios\\/${own}\\/x"}`, }, + { + name: "a backslash before a plain path is no escape and is kept", + before: `\\${folders.deviceSet}/${usedCopy}/x`, + after: `\\${folders.deviceSet}/${own}/x`, + }, { name: "a path already naming the copy is unchanged", before: `${folders.deviceSet}/${own}/data ${folders.logsRoot}/${own}`, diff --git a/src/drivers/ios/foreign-folders.ts b/src/drivers/ios/foreign-folders.ts index d01cc1598..6b7ae989e 100644 --- a/src/drivers/ios/foreign-folders.ts +++ b/src/drivers/ios/foreign-folders.ts @@ -8,19 +8,25 @@ export interface ForeignFolders { const UDID = "[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}"; const PRIVATE = "/private"; -const SEPARATOR = "(?:\\\\/|/)"; +// why: the regular expression for one separator: a plain `/`, or a backslash and a `/`. +const SEPARATORS = ["/", "\\\\/"] as const; /** why: iOS writes a folder under `/private` in two spellings (`/private/tmp/x` and `/tmp/x`), and each is matched and kept. */ function spellings(root: string): readonly string[] { return root.startsWith(`${PRIVATE}/`) ? [root, root.slice(PRIVATE.length)] : [root]; } +/** why: a path is written with plain separators or with all of them escaped, never a mix; a stray backslash before a plain `/` in binary data is not part of a path, and swapping it as one would change a length. */ function pattern(roots: readonly string[]): RegExp { const alternatives = roots .flatMap(spellings) .sort((a, b) => b.length - a.length) - .map((root) => root.split("/").map(escapeRegExp).join(SEPARATOR)); - return new RegExp(`(?:${alternatives.join("|")})${SEPARATOR}(${UDID})`, "g"); + .flatMap((root) => + SEPARATORS.map( + (separator) => `${root.split("/").map(escapeRegExp).join(separator)}${separator}`, + ), + ); + return new RegExp(`(?:${alternatives.join("|")})(${UDID})`, "g"); } function escapeRegExp(text: string): string { diff --git a/src/drivers/ios/source-copy.ts b/src/drivers/ios/source-copy.ts index 37ee93bc7..7a0805897 100644 --- a/src/drivers/ios/source-copy.ts +++ b/src/drivers/ios/source-copy.ts @@ -4,10 +4,9 @@ import { DriverCrashError } from "../../core/index.js"; import type { Clock, Filesystem } from "../../ports/index.js"; import { swapForeignFolders } from "./foreign-folders.js"; import type { SourceFile } from "./source-data.js"; -import type { SourceIndexOptions } from "./source-index.js"; +import { LAUNCH_SERVICES_DATABASE, type SourceIndexOptions } from "./source-index.js"; export const OVERRIDES_FILE_NAME = "disabled.plist"; -const LAUNCH_SERVICES_DATABASE = /^com\.apple\.LaunchServices-.*\.csstore$/; export interface CopyContext extends SourceIndexOptions { readonly clock: Clock; @@ -27,7 +26,9 @@ export async function dropLaunchServicesDatabase( const folder = join(setPath, udid, "data", "var", "db", "lsd"); if (!(await filesystem.exists(folder))) return; for (const entry of await filesystem.readdir(folder)) { - if (LAUNCH_SERVICES_DATABASE.test(entry)) await filesystem.rm(join(folder, entry)); + if (LAUNCH_SERVICES_DATABASE.test(`data/var/db/lsd/${entry}`)) { + await filesystem.rm(join(folder, entry)); + } } } diff --git a/src/drivers/ios/source-index.ts b/src/drivers/ios/source-index.ts index 321286467..b14deeb5f 100644 --- a/src/drivers/ios/source-index.ts +++ b/src/drivers/ios/source-index.ts @@ -5,6 +5,9 @@ import { namesAnyFolder, type ForeignFolders } from "./foreign-folders.js"; import type { SourceFile } from "./source-data.js"; const WAL_SUFFIX = "-wal"; +/** why: a clone drops this file before its swap (`dropLaunchServicesDatabase`), so it is never one the swap can patch. */ +export const LAUNCH_SERVICES_DATABASE = + /^data\/var\/db\/lsd\/com\.apple\.LaunchServices-.*\.csstore$/; const CHECKPOINT = "PRAGMA wal_checkpoint(TRUNCATE)"; export interface SourceIndexOptions { @@ -16,8 +19,16 @@ export interface SourceIndexOptions { interface Entry extends SourceFile { readonly path: string; + readonly size: number; } +/** + * why: a file is read whole to find the folders it names and again to swap them, and a used simulator + * holds files of hundreds of megabytes (past the limit of one string); none of them is a settings file, so + * a larger one is left out and logged. + */ +export const MAX_INDEXED_BYTES = 64 * 1024 * 1024; + /** * why: ADR 0076 §2: a WAL frame carries a checksum a swap would break, so every WAL of a made source is * flushed into its database first; then the files and links that name a device or log folder are found @@ -38,11 +49,15 @@ export async function indexSource( // why: closing a flushed database deletes its `-wal` and `-shm`, so the files are listed again after the flush. const indexed: SourceFile[] = []; for (const entry of await walk(options.filesystem, root, "")) { - if (unflushed.has(entry.path)) continue; - const bytes = - entry.kind === "link" - ? Buffer.from(await options.filesystem.readlink(entry.path)) - : await options.filesystem.readBytes(entry.path); + if (unflushed.has(entry.path) || LAUNCH_SERVICES_DATABASE.test(entry.relativePath)) continue; + if (entry.kind === "file" && entry.size > MAX_INDEXED_BYTES) { + options.logger?.warn( + `not searching ${entry.path} for folders to swap: its ${String(entry.size)} bytes are past ${String(MAX_INDEXED_BYTES)}`, + ); + continue; + } + const bytes = await readEntry(entry, options.filesystem); + if (bytes === undefined) continue; if (namesAnyFolder(bytes, options.folders)) { indexed.push({ kind: entry.kind, relativePath: entry.relativePath }); } @@ -50,6 +65,23 @@ export async function indexSource( return indexed; } +/** + * why: macOS refuses `open` on some files of a simulator's own folder (`locationd`'s) even to the + * owner, as `EPERM`; the simulator's own daemon writes them, so a file we cannot read is one no + * swap could patch either, and it is left out rather than failing the make. + */ +async function readEntry(entry: Entry, filesystem: Filesystem): Promise { + try { + return entry.kind === "link" + ? Buffer.from(await filesystem.readlink(entry.path)) + : await filesystem.readBytes(entry.path); + } catch (error: unknown) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "EPERM" || code === "EACCES") return undefined; + throw error; + } +} + async function walk(filesystem: Filesystem, directory: string, relative: string): Promise { const entries: Entry[] = []; let names: string[]; @@ -63,10 +95,10 @@ async function walk(filesystem: Filesystem, directory: string, relative: string) for (const name of names) { const path = join(directory, name); const relativePath = relative === "" ? name : `${relative}/${name}`; - const { kind } = await filesystem.lstat(path); + const { kind, size } = await filesystem.lstat(path); if (kind === "directory") entries.push(...(await walk(filesystem, path, relativePath))); else if (kind === "file" || kind === "symlink") { - entries.push({ kind: kind === "file" ? "file" : "link", path, relativePath }); + entries.push({ kind: kind === "file" ? "file" : "link", path, relativePath, size }); } } return entries; diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts index 738f88bd6..acc6ed21a 100644 --- a/src/drivers/ios/source-isolation.test.ts +++ b/src/drivers/ios/source-isolation.test.ts @@ -19,6 +19,8 @@ const logsRoot = "/home/Library/Logs/CoreSimulator"; const seedUdid = "00000000-0000-0000-0000-0000000000AA"; const usedUdid = "00000000-0000-0000-0000-0000000000BB"; const copyUdid = driverData.udid; +/** why: well past the 64 MiB a file may be to be searched for folders. */ +const HUGE_BYTES = 1024 * 1024 * 1024; /** why: a device set of another depth than the sources folder, as long as it, so an escaped path swapped into it changes length. */ const deepRoot = "/home/.simlock/a/b/c/devices-d"; @@ -53,6 +55,28 @@ class SlowReadFilesystem extends MemoryFilesystem { } } +/** why: macOS refuses `open` on some files of a simulator's own folder (`locationd`'s) to its owner, as `EPERM`. */ +class DeniedReadFilesystem extends MemoryFilesystem { + readonly denied = new Set(); + + /** why: a file of another size than its bytes, so a huge one needs no memory. */ + readonly sizes = new Map(); + + override async lstat(path: string): ReturnType { + const details = await super.lstat(path); + return { ...details, size: this.sizes.get(path) ?? details.size }; + } + + override async readBytes(path: string): Promise { + if (this.denied.has(path)) { + throw Object.assign(new Error(`EPERM: operation not permitted, open '${path}'`), { + code: "EPERM", + }); + } + return super.readBytes(path); + } +} + class LineLogger implements Logger { readonly lines: string[] = []; @@ -306,6 +330,8 @@ describe("a source made", () => { ], flushed: ["data/a.sqlite", "data/b.sqlite"], warned: [], + denied: [], + huge: [], }, { name: "a baseline saved from a used copy names that copy's folder", @@ -316,6 +342,8 @@ describe("a source made", () => { recorded: [["data/Media/state.plist", "file"]], flushed: [], warned: [], + denied: [], + huge: [], }, { name: "a WAL that cannot be flushed is logged and left out, and the source is still made", @@ -329,6 +357,8 @@ describe("a source made", () => { recorded: [], flushed: [], warned: ["data/b.sqlite"], + denied: [], + huge: [], }, { name: "a source that names no folder has no rows and is indexed", @@ -339,12 +369,60 @@ describe("a source made", () => { recorded: [], flushed: [], warned: [], + denied: [], + huge: [], + }, + { + name: "the LaunchServices database every clone drops is not recorded", + make: "seed", + failing: [], + tree: { + "data/var/db/lsd/com.apple.LaunchServices-1-v2.csstore": (udid: string) => + `${sourceFolder(udid)}/data`, + "data/Media/state.plist": (udid: string) => `${sourceFolder(udid)}/data`, + }, + link: false, + recorded: [["data/Media/state.plist", "file"]], + flushed: [], + warned: [], + denied: [], + huge: [], + }, + { + name: "a file past the limit is not searched, and is logged", + make: "baseline", + failing: [], + tree: { + "data/Media/huge.db": (udid: string) => `${sourceFolder(udid)}/data`, + "data/Media/state.plist": (udid: string) => `${sourceFolder(udid)}/data`, + }, + link: false, + recorded: [["data/Media/state.plist", "file"]], + flushed: [], + warned: [], + denied: [], + huge: ["data/Media/huge.db"], + }, + { + name: "a file the system refuses to read is left out, and the source is still made", + make: "baseline", + failing: [], + tree: { + "data/Library/Caches/locationd/locScoreInfo/secondary-1": () => "private", + "data/Media/state.plist": (udid: string) => `${sourceFolder(udid)}/data`, + }, + link: false, + recorded: [["data/Media/state.plist", "file"]], + flushed: [], + warned: [], + denied: ["data/Library/Caches/locationd/locScoreInfo/secondary-1"], + huge: [], }, ] as const)( "flushes its WALs and records the files and links that name a folder: $name", - async ({ make, failing, tree, link, recorded, flushed, warned }) => { + async ({ make, failing, tree, link, recorded, flushed, warned, denied, huge }) => { const udid = make === "seed" ? seedUdid : cloneUdid; - const filesystem = new MemoryFilesystem(); + const filesystem = new DeniedReadFilesystem(); const sqlite = new FakeSqliteOpener(filesystem); const warnings: string[] = []; const logger = Object.assign(new LineLogger(), { @@ -371,6 +449,10 @@ describe("a source made", () => { filesystem.defineSymlink(`${folder}/data/Library/Logs`, `${logsRoot}/${udid}`); } } + for (const path of huge) { + filesystem.sizes.set(`${sourceFolder(udid)}/${path}`, HUGE_BYTES); + } + for (const path of denied) filesystem.denied.add(`${sourceFolder(udid)}/${path}`); for (const path of failing) sqlite.failing.add(`${sourceFolder(udid)}/${path}`); const database = await memorySourceDatabase(); const runner = new ScriptedProcessRunner([ @@ -435,9 +517,15 @@ describe("a source made", () => { expect(sqlite.flushed, "the databases whose WAL was flushed").toEqual( flushed.map((path) => `${sourceFolder(udid)}/${path}`), ); - expect(warnings).toEqual( - warned.map((path) => `could not flush ${sourceFolder(udid)}/${path}: database is locked`), - ); + expect(warnings).toEqual([ + ...warned.map( + (path) => `could not flush ${sourceFolder(udid)}/${path}: database is locked`, + ), + ...huge.map( + (path) => + `not searching ${sourceFolder(udid)}/${path} for folders to swap: its ${String(HUGE_BYTES)} bytes are past 67108864`, + ), + ]); }, ); }); diff --git a/src/ports/filesystem.test.ts b/src/ports/filesystem.test.ts index 77d1570b5..8196b117d 100644 --- a/src/ports/filesystem.test.ts +++ b/src/ports/filesystem.test.ts @@ -324,8 +324,11 @@ describe.each(implementations)("Filesystem ownership contract: $name", ({ create await expect(filesystem.lstat(root)).resolves.toEqual({ kind: "directory", mode: 0o700, + size: 0, uid: process.getuid?.() ?? 0, }); + await filesystem.writeFileAtomic(`${root}/note.txt`, "four"); + expect((await filesystem.lstat(`${root}/note.txt`)).size, "a file's length").toBe(4); }); it("reports the permission bits a path was last given, not the ones it was created with", async () => { @@ -422,6 +425,7 @@ describe("MemoryFilesystem", () => { await expect(filesystem.lstat("/devices/ios")).resolves.toEqual({ kind: "directory", mode: 0o755, + size: 0, uid: 4242, }); }); diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index 840fc91dd..9489ea51f 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -55,6 +55,8 @@ export interface PathDetails { readonly kind: "file" | "directory" | "symlink" | "other"; readonly uid: number; readonly mode: number; + /** why: a file's length in bytes, so a caller can refuse one too large to read whole; 0 for anything that is not a file. */ + readonly size: number; } /** @@ -356,6 +358,7 @@ export class NodeFilesystem implements Filesystem { kind: nodeKind(details), uid: details.uid, mode: details.mode & 0o777, + size: details.isFile() ? details.size : 0, }; } @@ -714,7 +717,12 @@ export class MemoryFilesystem implements Filesystem { this.#failIfDefined(path); const entry = this.#rawEntryAt(path); - return { kind: entry.kind, uid: entry.uid, mode: entry.mode }; + return { + kind: entry.kind, + uid: entry.uid, + mode: entry.mode, + size: entry.kind === "file" ? fileBytes(entry).byteLength : 0, + }; } async mkdir(path: string, options: { readonly mode?: number } = {}): Promise { From 3bdeb467d930c756e1de64abed645a539b8c2c33 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 00:15:25 +0100 Subject: [PATCH 22/48] refactor(ios): the index's skips live in one reader, and the lane test names its sources through one helper (#1570) --- e2e/slow-ios-source-isolation.test.ts | 29 ++++++++++++++------------- src/drivers/ios/source-index.ts | 25 +++++++++++++---------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/e2e/slow-ios-source-isolation.test.ts b/e2e/slow-ios-source-isolation.test.ts index 7a2634ab4..49d631782 100644 --- a/e2e/slow-ios-source-isolation.test.ts +++ b/e2e/slow-ios-source-isolation.test.ts @@ -91,6 +91,13 @@ async function newestModification(folder: string): Promise { return Math.max(0, ...(await modifiedFiles(folder)).values()); } +/** why: the UDID of the one device in the sources folder whose name starts so. */ +async function sourceUdid(sources: string, prefix: string, what: string): Promise { + const found = (await setDevices(sources)).find((device) => device.name.startsWith(prefix)); + expect(found, what).toBeDefined(); + return join(sources, found?.udid ?? ""); +} + async function expectOwnLogFolder(held: Lease): Promise { const target = await readlink(join(held.deviceSet, held.udid, "data", "Library", "Logs")); expect(target, `${held.udid}'s Logs link`).toBe( @@ -138,11 +145,12 @@ describe.skipIf(process.platform !== "darwin")( const sources = iosSourcesFolder(env.home); const first = await lease(env, []); - const seed = (await setDevices(sources)).find((device) => - device.name.startsWith("simlock-seed-"), + const seed = await sourceUdid( + sources, + "simlock-seed-", + "the seed made for the first lease", ); - expect(seed, "the seed made for the first lease").toBeDefined(); - const seedMadeAt = await newestModification(join(sources, seed?.udid ?? "")); + const seedMadeAt = await newestModification(seed); await use(env, first); await expectOwnLogFolder(first); for (let index = 0; index < 2; index += 1) await useAndRelease(env, []); @@ -152,22 +160,15 @@ describe.skipIf(process.platform !== "darwin")( }); expect(saved.code, `baseline save failed: ${saved.stderr}`).toBe(0); const savedAt = Date.now(); - const source = (await setDevices(sources)).find((device) => - device.name.startsWith("simlock-baseline-"), - ); - expect(source, "the baseline's source").toBeDefined(); + const source = await sourceUdid(sources, "simlock-baseline-", "the baseline's source"); const copies: Lease[] = []; for (let index = 0; index < 2; index += 1) { copies.push(await useAndRelease(env, ["--baseline", "isolation"])); } - await expectNothingNewer(join(sources, seed?.udid ?? ""), seedMadeAt, "the seed"); - await expectNothingNewer( - join(sources, source?.udid ?? ""), - savedAt, - "the baseline's source", - ); + await expectNothingNewer(seed, seedMadeAt, "the seed"); + await expectNothingNewer(source, savedAt, "the baseline's source"); await expectNothingNewer( join(first.deviceSet, first.udid), savedAt, diff --git a/src/drivers/ios/source-index.ts b/src/drivers/ios/source-index.ts index b14deeb5f..6ca1ca93f 100644 --- a/src/drivers/ios/source-index.ts +++ b/src/drivers/ios/source-index.ts @@ -27,7 +27,7 @@ interface Entry extends SourceFile { * holds files of hundreds of megabytes (past the limit of one string); none of them is a settings file, so * a larger one is left out and logged. */ -export const MAX_INDEXED_BYTES = 64 * 1024 * 1024; +const MAX_INDEXED_BYTES = 64 * 1024 * 1024; /** * why: ADR 0076 §2: a WAL frame carries a checksum a swap would break, so every WAL of a made source is @@ -50,13 +50,7 @@ export async function indexSource( const indexed: SourceFile[] = []; for (const entry of await walk(options.filesystem, root, "")) { if (unflushed.has(entry.path) || LAUNCH_SERVICES_DATABASE.test(entry.relativePath)) continue; - if (entry.kind === "file" && entry.size > MAX_INDEXED_BYTES) { - options.logger?.warn( - `not searching ${entry.path} for folders to swap: its ${String(entry.size)} bytes are past ${String(MAX_INDEXED_BYTES)}`, - ); - continue; - } - const bytes = await readEntry(entry, options.filesystem); + const bytes = await readEntry(entry, options); if (bytes === undefined) continue; if (namesAnyFolder(bytes, options.folders)) { indexed.push({ kind: entry.kind, relativePath: entry.relativePath }); @@ -70,11 +64,20 @@ export async function indexSource( * owner, as `EPERM`; the simulator's own daemon writes them, so a file we cannot read is one no * swap could patch either, and it is left out rather than failing the make. */ -async function readEntry(entry: Entry, filesystem: Filesystem): Promise { +async function readEntry( + entry: Entry, + options: SourceIndexOptions, +): Promise { + if (entry.kind === "file" && entry.size > MAX_INDEXED_BYTES) { + options.logger?.warn( + `not searching ${entry.path} for folders to swap: its ${String(entry.size)} bytes are past ${String(MAX_INDEXED_BYTES)}`, + ); + return undefined; + } try { return entry.kind === "link" - ? Buffer.from(await filesystem.readlink(entry.path)) - : await filesystem.readBytes(entry.path); + ? Buffer.from(await options.filesystem.readlink(entry.path)) + : await options.filesystem.readBytes(entry.path); } catch (error: unknown) { const code = (error as NodeJS.ErrnoException).code; if (code === "EPERM" || code === "EACCES") return undefined; From 252cba6e6f8d45b8581c60d2c37790206260d3bc Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 00:15:47 +0100 Subject: [PATCH 23/48] test(ios): the memory filesystem's faults are planted in one call (#1570) --- src/drivers/ios/source-isolation.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/drivers/ios/source-isolation.test.ts b/src/drivers/ios/source-isolation.test.ts index acc6ed21a..312de2f81 100644 --- a/src/drivers/ios/source-isolation.test.ts +++ b/src/drivers/ios/source-isolation.test.ts @@ -62,6 +62,14 @@ class DeniedReadFilesystem extends MemoryFilesystem { /** why: a file of another size than its bytes, so a huge one needs no memory. */ readonly sizes = new Map(); + plantFaults( + folder: string, + faults: { readonly denied: readonly string[]; readonly huge: readonly string[] }, + ): void { + for (const path of faults.denied) this.denied.add(`${folder}/${path}`); + for (const path of faults.huge) this.sizes.set(`${folder}/${path}`, HUGE_BYTES); + } + override async lstat(path: string): ReturnType { const details = await super.lstat(path); return { ...details, size: this.sizes.get(path) ?? details.size }; @@ -449,10 +457,7 @@ describe("a source made", () => { filesystem.defineSymlink(`${folder}/data/Library/Logs`, `${logsRoot}/${udid}`); } } - for (const path of huge) { - filesystem.sizes.set(`${sourceFolder(udid)}/${path}`, HUGE_BYTES); - } - for (const path of denied) filesystem.denied.add(`${sourceFolder(udid)}/${path}`); + filesystem.plantFaults(sourceFolder(udid), { denied, huge }); for (const path of failing) sqlite.failing.add(`${sourceFolder(udid)}/${path}`); const database = await memorySourceDatabase(); const runner = new ScriptedProcessRunner([ From 2c01e726561c42aca3a42d5a0c59d72919bb654e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 01:26:20 +0100 Subject: [PATCH 24/48] docs(ios): the iOS manual fits its 8000 characters again (#1570) --- src/drivers/ios/AGENTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index c85d2a9d2..28bb9552b 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -17,9 +17,9 @@ Resolve and boot: - A slim spec is returned only for a slim request the driver will slim. The registry is the only record of a device's mode. - `RuntimeMissingError` is only for what a download can fix: an OS outside the device type's range, an installed runtime not paired with the type, and anything below iOS 16 are refused instead. - A boot is `simctl boot`, `bootstatus -b`, then `listapps` until it succeeds, on one deadline: bootstatus can finish before the installer accepts requests. -- Slim (ADR 0060), on `prepare` of a slim spec only: the managed labels are merged into the simulator's `disabled.plist` before the boot (other entries kept; a failed write is logged and the boot goes on), and `launchctl print-disabled system` is read back after it. A label not as written returns `full`; a readback that fails returns `slim`; both call `onSlimSkipped`, logged only. Every label as written and the write ok: `slim`, and `onSlimmed` becomes `device.slimmed`. +- Slim (ADR 0060), on `prepare` of a slim spec only: the managed labels are merged into the simulator's `disabled.plist` before the boot, and `launchctl print-disabled system` is read back after it. A label not as written returns `full`; a readback that fails returns `slim`; both call `onSlimSkipped`, logged only. Every label as written and the write ok: `slim`, and `onSlimmed` becomes `device.slimmed`. - Slim `except` (ADR 0066, `slim-except.ts`): an excepted category's labels are written `false`; an unknown name is `BAD_REQUEST`; a kept label not read back throws `KeptCategoriesError`. -- A recovery boot writes no slim overrides and reads nothing back. DeviceHub clipboard isolation is configured before preparation and recovery boots; ready default-set simulators reboot under the lease claim before a grant. Active leases at upgrade are not interrupted and may retain clipboard sync until a later boot or lease end. +- A recovery boot writes no slim overrides and reads nothing back. DeviceHub clipboard isolation is configured before preparation and recovery boots; ready default-set simulators reboot under the lease claim before a grant. - After every boot and before the readback, `runAfterBoot` (`boot-courtesies.ts`) runs the volume step, then the window step. Each only reports its failure (`onSilenceFailed`, `onWindowFailed`), and the device stays in use. - Volume (`ios.simulator.audio: false`): Apps keep playing at volume 0. A baseline source is not silenced. - Window (`ios.simulator.headless: false`): DeviceHub is opened in the background, so the front app stays in front, and is told not to shut down started devices when it quits. Only a non-zero exit of that open means DeviceHub is missing: Simulator.app is then opened instead, detached, so quitting it leaves the simulator running. Any other failure opens nothing. @@ -28,7 +28,7 @@ Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources folder (ADR 0076): `$SIMLOCK_HOME/devices/src` padded with `_` to the device set's real-path length, an owned root under both settings. A row naming another set is a seed made again, or a baseline stale (`BASELINE_STALE`), deleted there only in an earlier sources folder. It is in no `listManaged` or `simlock simctl list`. - Every clone has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app, system apps included, under the original device's folder. A clone whose drop fails is deleted. -- Each copy has the files and links its source's index records swapped before its first boot (ADR 0076): another simulator's device or log folder becomes the copy's own, same spelling and length; a failed swap deletes the copy. An unindexed seed is made again, a baseline stale. `.tracev3` is not fixed. A file the system refuses to read (`EPERM`, as under `locationd`) or one past `MAX_INDEXED_BYTES` is left out of the index (the larger one logged), so it is never swapped. The LaunchServices database a clone drops is never indexed. +- Each copy has the files and links its source's index records swapped before its first boot (ADR 0076): another simulator's device or log folder becomes the copy's own, same spelling and length; a failed swap deletes the copy. An unindexed seed is made again, a baseline stale. A file the system refuses to read (`EPERM`) or one past `MAX_INDEXED_BYTES` is left out of the index, so it is never swapped. The LaunchServices database a clone drops is never indexed. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. From 06829b2b1b5c65228bc1ca9da14aea603cdfab73 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:22:15 +0100 Subject: [PATCH 25/48] test: a copy swaps its lineage's UDIDs for its own before its first boot (#1570) --- src/drivers/ios/device-data.ts | 27 ++++++--- src/drivers/ios/index-seed-swap.test.ts | 49 ++++++++++++++++ src/drivers/ios/seeds.ts | 9 ++- src/drivers/ios/simulator-data.ts | 6 ++ src/drivers/ios/source-data.ts | 76 ++++++++++++++++++++++--- src/drivers/ios/test-index.ts | 4 +- src/drivers/ios/testing.ts | 48 +++++++++++++++- 7 files changed, 194 insertions(+), 25 deletions(-) create mode 100644 src/drivers/ios/index-seed-swap.test.ts diff --git a/src/drivers/ios/device-data.ts b/src/drivers/ios/device-data.ts index a65f897b4..94134f279 100644 --- a/src/drivers/ios/device-data.ts +++ b/src/drivers/ios/device-data.ts @@ -1,4 +1,5 @@ import type { DeviceDataTable } from "../../core/index.js"; +import { UDID_PATTERN } from "./device-set-leftovers.js"; interface IosDeviceDataRow { readonly udid: string; @@ -7,6 +8,7 @@ interface IosDeviceDataRow { readonly runtime_id: string; readonly slim_signature: string | null; readonly slim_mark_token: string | null; + readonly lineage: string | null; } interface IosDriverData { @@ -16,19 +18,23 @@ interface IosDriverData { readonly slimMarkToken?: string; readonly slimSignature?: string; readonly udid: string; + readonly lineage?: readonly string[]; } const REQUIRED_KEYS = ["deviceTypeId", "name", "runtimeId", "udid"] as const; const OPTIONAL_KEYS = ["slimMarkToken", "slimSignature"] as const; +const KNOWN_KEYS = new Set([...REQUIRED_KEYS, ...OPTIONAL_KEYS, "lineage"]); function isIosDriverData(value: unknown): value is IosDriverData { if (typeof value !== "object" || value === null) return false; const record = value as Record; - const known = new Set([...REQUIRED_KEYS, ...OPTIONAL_KEYS]); return ( - Object.keys(record).every((key) => known.has(key)) && + Object.keys(record).every((key) => KNOWN_KEYS.has(key)) && REQUIRED_KEYS.every((key) => typeof record[key] === "string") && - OPTIONAL_KEYS.every((key) => record[key] === undefined || typeof record[key] === "string") + OPTIONAL_KEYS.every((key) => record[key] === undefined || typeof record[key] === "string") && + (record["lineage"] === undefined || + (Array.isArray(record["lineage"]) && + record["lineage"].every((udid) => typeof udid === "string" && UDID_PATTERN.test(udid)))) ); } @@ -43,12 +49,14 @@ export const iosDeviceDataTable: DeviceDataTable = { slim_signature TEXT, slim_mark_token TEXT ) STRICT`, + // why: ADR 0076: the lineage of the source this device was copied from, kept so a save knows it without reading the copy; null on a device made before. + "ALTER TABLE ios_device_data ADD COLUMN lineage TEXT", ], name: "ios-device-data", read(database, deviceId) { const row = database .prepare( - "SELECT udid, name, device_type_id, runtime_id, slim_signature, slim_mark_token FROM ios_device_data WHERE device_id = ?", + "SELECT udid, name, device_type_id, runtime_id, slim_signature, slim_mark_token, lineage FROM ios_device_data WHERE device_id = ?", ) .get(deviceId) as IosDeviceDataRow | undefined; if (row === undefined) throw new Error(`No ios driver data for device ${deviceId}`); @@ -58,10 +66,11 @@ export const iosDeviceDataTable: DeviceDataTable = { runtimeId: row.runtime_id, ...(row.slim_mark_token === null ? {} : { slimMarkToken: row.slim_mark_token }), ...(row.slim_signature === null ? {} : { slimSignature: row.slim_signature }), + ...(row.lineage === null ? {} : { lineage: JSON.parse(row.lineage) as string[] }), udid: row.udid, }; }, - version: 1, + version: 2, write(database, deviceId, driverData) { if (!isIosDriverData(driverData)) { throw new Error(`Invalid ios driver data for device ${deviceId}`); @@ -69,11 +78,12 @@ export const iosDeviceDataTable: DeviceDataTable = { const data = driverData; database .prepare( - `INSERT INTO ios_device_data (device_id, udid, name, device_type_id, runtime_id, slim_signature, slim_mark_token) - VALUES (?, ?, ?, ?, ?, ?, ?) + `INSERT INTO ios_device_data (device_id, udid, name, device_type_id, runtime_id, slim_signature, slim_mark_token, lineage) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (device_id) DO UPDATE SET udid = excluded.udid, name = excluded.name, device_type_id = excluded.device_type_id, runtime_id = excluded.runtime_id, - slim_signature = excluded.slim_signature, slim_mark_token = excluded.slim_mark_token`, + slim_signature = excluded.slim_signature, slim_mark_token = excluded.slim_mark_token, + lineage = excluded.lineage`, ) .run( deviceId, @@ -83,6 +93,7 @@ export const iosDeviceDataTable: DeviceDataTable = { data.runtimeId, data.slimSignature ?? null, data.slimMarkToken ?? null, + data.lineage === undefined ? null : JSON.stringify(data.lineage), ); }, }; diff --git a/src/drivers/ios/index-seed-swap.test.ts b/src/drivers/ios/index-seed-swap.test.ts new file mode 100644 index 000000000..f460ce7b8 --- /dev/null +++ b/src/drivers/ios/index-seed-swap.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; +import { plantBytes, readBytes } from "./testing.js"; +import { deviceRoot, driverData, plantManagedDevice } from "./test-slim.js"; +import { + createDriver, + listFixture, + listInvocation, + scopedRunners, + seedUdid, + seededDatabase, + sourcesSimctl, + spec, +} from "./test-index.js"; + +afterEach(() => { + scopedRunners.length = 0; +}); + +const lineageUdid = "AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"; +const copyUdid = "CCCCCCCC-4444-5555-6666-DDDDDDDDDDDD"; + +describe("IosSimctlDriver copy swap", () => { + it("a copy keeps its source's lineage and swaps the lineage's UDIDs in every recorded file for its own before its first boot", async () => { + const database = await seededDatabase({ + files: [{ kind: "file", relativePath: "data/Library/note.plist" }], + lineage: [lineageUdid], + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + const file = `${deviceRoot}/${copyUdid}/data/Library/note.plist`; + await plantBytes(filesystem, file, Buffer.from(`path /Devices/${lineageUdid}/data end`)); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + const device = await driver.provision(spec); + + expect((await readBytes(filesystem, file)).toString("latin1")).toBe( + `path /Devices/${copyUdid}/data end`, + ); + expect(device.driverData).toMatchObject({ lineage: [lineageUdid] }); + }); +}); diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index 579f2814c..4517a87ae 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -161,7 +161,7 @@ export class Seeds { await this.#slimBoot.copyOverrides(data.udid, cloneUdid); await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes); + this.#sources.markReady(sourceId, sizeBytes, { files: [], lineage: [] }); // why: ADR 0076: the lease is destroyed next, and `simctl delete` leaves its log folder behind for the baseline's copies to inherit. await this.#deleteLogFolder(data.udid); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; @@ -369,8 +369,11 @@ export class Seeds { keepRow = true; await this.#deleteThrowaway(done, false); keepRow = false; - this.#sources.markReady(sourceId, await this.#sizeOf(seedClone)); - return { sourceId, udid: seedClone, runtimeBuild: resolved.runtime.build }; + this.#sources.markReady(sourceId, await this.#sizeOf(seedClone), { + files: [], + lineage: [], + }); + return { sourceId, udid: seedClone, runtimeBuild: resolved.runtime.build, lineage: [] }; } catch (error: unknown) { if (seedClone !== undefined) await this.#deleteSeedSimulator(seedClone).catch(() => undefined); diff --git a/src/drivers/ios/simulator-data.ts b/src/drivers/ios/simulator-data.ts index 89acb6fe3..58b9de717 100644 --- a/src/drivers/ios/simulator-data.ts +++ b/src/drivers/ios/simulator-data.ts @@ -6,6 +6,7 @@ export interface IosDriverData { readonly name: string; readonly runtimeId: string; readonly udid: string; + readonly lineage?: readonly string[]; } export function iosDriverData(device: DriverDevice): IosDriverData { @@ -29,6 +30,7 @@ export function iosDriverData(device: DriverDevice): IosDriverData { // an older daemon stamped here is not read: the spec's mode is the only record of it (ADR 0007 §7). ...(typeof value.slimSignature === "string" ? { slimSignature: value.slimSignature } : {}), ...(typeof value.slimMarkToken === "string" ? { slimMarkToken: value.slimMarkToken } : {}), + ...(isUdidList(value.lineage) ? { lineage: value.lineage } : {}), }; } @@ -37,3 +39,7 @@ export function requireIosPlatform(platform: string): void { throw new DriverCrashError(`iOS simctl driver cannot handle ${platform} device requests`); } } + +function isUdidList(value: unknown): value is readonly string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string"); +} diff --git a/src/drivers/ios/source-data.ts b/src/drivers/ios/source-data.ts index 674ed8170..c49efebe9 100644 --- a/src/drivers/ios/source-data.ts +++ b/src/drivers/ios/source-data.ts @@ -1,5 +1,5 @@ import { join } from "node:path"; -import type { SqliteDatabase, SqliteSchemaOwner } from "../../ports/index.js"; +import { inTransaction, type SqliteDatabase, type SqliteSchemaOwner } from "../../ports/index.js"; export const iosSourceDataOwner: SqliteSchemaOwner = { migrations: [ @@ -31,9 +31,18 @@ export const iosSourceDataOwner: SqliteSchemaOwner = { WHERE kind = 'seed' AND state = 'ready';`, // why: ADR 0076: the throwaway a seed was copied from, kept so a restart mid-make can delete it through the row. "ALTER TABLE ios_source_data ADD COLUMN throwaway_udid TEXT", + // why: ADR 0076: the UDIDs a source was copied from (null: made before lineages were kept), the files that name them, and whether a baseline was saved from a device that had no lineage. + `ALTER TABLE ios_source_data ADD COLUMN lineage TEXT; + ALTER TABLE ios_source_data ADD COLUMN saved_without_lineage INTEGER NOT NULL DEFAULT 0; + CREATE TABLE ios_source_files ( + source_id TEXT NOT NULL, + relative_path TEXT NOT NULL CHECK (length(CAST(relative_path AS BLOB)) <= 4096), + kind TEXT NOT NULL CHECK (kind IN ('link', 'file', 'unreadable')), + PRIMARY KEY (source_id, relative_path) + ) STRICT;`, ], name: "ios-source-data", - version: 3, + version: 4, }; export interface NewSourceRow { @@ -63,12 +72,27 @@ export interface SourceRow { readonly state: "making" | "ready"; readonly maker: string; readonly sizeBytes: number | undefined; + readonly lineage: readonly string[] | undefined; + readonly savedWithoutLineage: boolean; } export interface ReadySeed { readonly sourceId: string; readonly udid: string; readonly runtimeBuild: string | undefined; + readonly lineage: readonly string[] | undefined; +} + +export interface SourceFile { + readonly relativePath: string; + readonly kind: "link" | "file" | "unreadable"; +} + +/** why: what a source records when it is made: the UDIDs it came from and the files that name them. */ +export interface SourceRecord { + readonly lineage: readonly string[] | undefined; + readonly files: readonly SourceFile[]; + readonly savedWithoutLineage?: boolean; } /** why: the one place that reads or writes `ios_source_data`; nothing outside the iOS driver does (ADR 0036 §2). */ @@ -113,20 +137,47 @@ export class IosSourceTable { .run(udid, sourceId); } - markReady(sourceId: string, sizeBytes: number | undefined): void { - this.#database - .prepare("UPDATE ios_source_data SET state = 'ready', size_bytes = ? WHERE source_id = ?") - .run(sizeBytes ?? null, sourceId); + markReady(sourceId: string, sizeBytes: number | undefined, record: SourceRecord): void { + inTransaction(this.#database, () => { + this.#database + .prepare( + `UPDATE ios_source_data SET state = 'ready', size_bytes = ?, lineage = ?, saved_without_lineage = ? + WHERE source_id = ?`, + ) + .run( + sizeBytes ?? null, + record.lineage === undefined ? null : JSON.stringify(record.lineage), + record.savedWithoutLineage === true ? 1 : 0, + sourceId, + ); + const insert = this.#database.prepare( + "INSERT INTO ios_source_files (source_id, relative_path, kind) VALUES (?, ?, ?)", + ); + for (const file of record.files) insert.run(sourceId, file.relativePath, file.kind); + }); + } + + filesOf(sourceId: string): readonly SourceFile[] { + return this.#database + .prepare("SELECT relative_path, kind FROM ios_source_files WHERE source_id = ?") + .all(sourceId) + .map((row) => ({ + relativePath: row["relative_path"] as string, + kind: row["kind"] as SourceFile["kind"], + })); } deleteRow(sourceId: string): void { - this.#database.prepare("DELETE FROM ios_source_data WHERE source_id = ?").run(sourceId); + inTransaction(this.#database, () => { + this.#database.prepare("DELETE FROM ios_source_files WHERE source_id = ?").run(sourceId); + this.#database.prepare("DELETE FROM ios_source_data WHERE source_id = ?").run(sourceId); + }); } findReadySeed(specKey: string): ReadySeed | undefined { const row = this.#database .prepare( - "SELECT source_id, udid, runtime_build FROM ios_source_data WHERE kind = 'seed' AND state = 'ready' AND spec_key = ?", + "SELECT source_id, udid, runtime_build, lineage FROM ios_source_data WHERE kind = 'seed' AND state = 'ready' AND spec_key = ?", ) .get(specKey); if (row === undefined) return undefined; @@ -134,6 +185,7 @@ export class IosSourceTable { sourceId: row["source_id"] as string, udid: row["udid"] as string, runtimeBuild: (row["runtime_build"] as string | null) ?? undefined, + lineage: parseLineage(row["lineage"]), }; } @@ -181,7 +233,7 @@ export class IosSourceTable { return this.#database .prepare( `SELECT source_id, kind, udid, throwaway_udid, device_type_id, runtime_id, runtime_build, model, os_version, - state, maker, size_bytes FROM ios_source_data ${where}`, + state, maker, size_bytes, lineage, saved_without_lineage FROM ios_source_data ${where}`, ) .all(...parameters) .map((row) => ({ @@ -197,10 +249,16 @@ export class IosSourceTable { state: row["state"] as "making" | "ready", maker: row["maker"] as string, sizeBytes: (row["size_bytes"] as number | null) ?? undefined, + lineage: parseLineage(row["lineage"]), + savedWithoutLineage: row["saved_without_lineage"] === 1, })); } } +function parseLineage(value: unknown): readonly string[] | undefined { + return typeof value === "string" ? (JSON.parse(value) as string[]) : undefined; +} + export function sourcesSetFor(simlockHome: string, deviceRoot: string, owned: boolean): string { return owned ? join(deviceRoot, "sources") : join(simlockHome, "devices", "ios-sources"); } diff --git a/src/drivers/ios/test-index.ts b/src/drivers/ios/test-index.ts index 0eacab7df..ce47934de 100644 --- a/src/drivers/ios/test-index.ts +++ b/src/drivers/ios/test-index.ts @@ -9,7 +9,7 @@ import { type Logger, } from "../../ports/index.js"; import { IosSimctlDriver, SLIM_CATEGORIES } from "./index.js"; -import { plantReadySeed, memorySourceDatabase } from "./testing.js"; +import { plantReadySeed, memorySourceDatabase, type PlantedSource } from "./testing.js"; import { simctl, deviceRoot, @@ -94,7 +94,7 @@ export function sourcesSimctl(...args: readonly string[]): { /** why: A database that already holds the seed of the kind `spec` names, so a test of the copy step starts at the clone. */ export async function seededDatabase( - seed: { readonly runtimeBuild?: string; readonly udid?: string } = {}, + seed: Partial> = {}, ): Promise { return plantReadySeed(await memorySourceDatabase(), { deviceTypeId: driverData.deviceTypeId, diff --git a/src/drivers/ios/testing.ts b/src/drivers/ios/testing.ts index 6e074517e..522df47ac 100644 --- a/src/drivers/ios/testing.ts +++ b/src/drivers/ios/testing.ts @@ -1,5 +1,5 @@ -import { NodeSqliteOpener, type SqliteDatabase } from "../../ports/index.js"; -import { iosSourceDataOwner, IosSourceTable } from "./source-data.js"; +import { NodeSqliteOpener, type MemoryFilesystem, type SqliteDatabase } from "../../ports/index.js"; +import { iosSourceDataOwner, IosSourceTable, type SourceFile } from "./source-data.js"; export async function memorySourceDatabase(): Promise { const database = await new NodeSqliteOpener().open(":memory:", { readOnly: false }); @@ -9,6 +9,11 @@ export async function memorySourceDatabase(): Promise { export interface PlantedSource { readonly deviceTypeId: string; + /** why: the files that name the lineage, as a source's make recorded them. */ + readonly files?: readonly SourceFile[]; + /** why: `null` is a source made before lineages were kept; absent is a lineage of no simulator. */ + readonly lineage?: readonly string[] | null; + readonly savedWithoutLineage?: boolean; readonly kind?: "seed" | "baseline"; readonly maker?: string; readonly model?: string; @@ -40,7 +45,13 @@ export function plantSource(database: SqliteDatabase, source: PlantedSource): Sq }); if (source.udid !== undefined) table.setUdid(sourceId, source.udid); if (source.throwawayUdid !== undefined) table.setThrowaway(sourceId, source.throwawayUdid); - if (source.state !== "making") table.markReady(sourceId, source.sizeBytes); + if (source.state !== "making") { + table.markReady(sourceId, source.sizeBytes, { + files: source.files ?? [], + lineage: source.lineage === null ? undefined : (source.lineage ?? []), + savedWithoutLineage: source.savedWithoutLineage ?? false, + }); + } return database; } @@ -50,3 +61,34 @@ export function plantReadySeed( ): SqliteDatabase { return plantSource(database, seed); } + +/** why: a file with the exact bytes a test names, which `writeFileAtomic` cannot hold for a UTF-16 or binary one. */ +export async function plantBytes( + filesystem: MemoryFilesystem, + path: string, + bytes: Uint8Array, +): Promise { + await filesystem.mkdirp(path.slice(0, path.lastIndexOf("/"))); + await filesystem.writeFileStream( + path, + new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }), + ); +} + +export async function readBytes(filesystem: MemoryFilesystem, path: string): Promise { + const file = await filesystem.openFile(path); + const chunks: Uint8Array[] = []; + const reader = file.read().getReader(); + for (;;) { + const piece = await reader.read(); + if (piece.done) break; + chunks.push(piece.value); + } + await file.close(); + return Buffer.concat(chunks); +} From ed4ea7092814e4216bd9b761450f373dea0caa22 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:23:35 +0100 Subject: [PATCH 26/48] feat(ios): a copy swaps a recorded file's lineage UDIDs for its own before its first boot (#1570) --- src/auth/admin-secret.test.ts | 6 +++ .../ios/index-save-source-lifecycle.test.ts | 2 +- src/drivers/ios/index-seed-provision.test.ts | 2 +- src/drivers/ios/lineage-files.ts | 49 +++++++++++++++++++ src/drivers/ios/seeds.ts | 25 ++++++++-- src/drivers/ios/udid-swap.ts | 20 ++++++++ src/ports/filesystem.ts | 25 ++++++++++ 7 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/drivers/ios/lineage-files.ts create mode 100644 src/drivers/ios/udid-swap.ts diff --git a/src/auth/admin-secret.test.ts b/src/auth/admin-secret.test.ts index 85dd1effc..984f264d6 100644 --- a/src/auth/admin-secret.test.ts +++ b/src/auth/admin-secret.test.ts @@ -102,6 +102,9 @@ describe("AdminSecretManager", () => { directorySize: () => { throw new Error("verify() must not touch the filesystem"); }, + writeAt: () => { + throw new Error("verify() must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("verify() must not touch the filesystem"); }, @@ -189,6 +192,9 @@ describe("AdminSecretManager", () => { directorySize: () => { throw new Error("must not touch the filesystem"); }, + writeAt: () => { + throw new Error("must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("must not touch the filesystem"); }, diff --git a/src/drivers/ios/index-save-source-lifecycle.test.ts b/src/drivers/ios/index-save-source-lifecycle.test.ts index 1beb0023e..bc13dc694 100644 --- a/src/drivers/ios/index-save-source-lifecycle.test.ts +++ b/src/drivers/ios/index-save-source-lifecycle.test.ts @@ -274,7 +274,7 @@ describe("IosSimctlDriver", () => { await expect(driver.provision({ ...spec, mode: "slim" })).resolves.toEqual({ address: driverData.udid, deviceId: driverData.udid, - driverData, + driverData: { ...driverData, lineage: [] }, }); }); diff --git a/src/drivers/ios/index-seed-provision.test.ts b/src/drivers/ios/index-seed-provision.test.ts index dd74be977..27abdc7b3 100644 --- a/src/drivers/ios/index-seed-provision.test.ts +++ b/src/drivers/ios/index-seed-provision.test.ts @@ -121,7 +121,7 @@ describe("IosSimctlDriver", () => { await expect(driver.provision(spec)).resolves.toEqual({ address: driverData.udid, deviceId: driverData.udid, - driverData, + driverData: { ...driverData, lineage: [throwawayUdid.toUpperCase()] }, }); expect( rowsWhenCreated, diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts new file mode 100644 index 000000000..715c0aa36 --- /dev/null +++ b/src/drivers/ios/lineage-files.ts @@ -0,0 +1,49 @@ +import { DriverCrashError } from "../../core/index.js"; +import type { Filesystem } from "../../ports/index.js"; +import type { SourceFile } from "./source-data.js"; +import { SWAP_OVERLAP_BYTES, swapLineage, type LineageSwap } from "./udid-swap.js"; + +/** why: a window is read whole, so this bounds memory at any file size (ADR 0076 §3). */ +const WINDOW_BYTES = 1_048_576; + +/** why: ADR 0076 §4: before its first boot a copy's recorded files name its own UDID in place of the lineage's, each file keeping its length. */ +export async function swapCopyFiles( + filesystem: Filesystem, + folder: string, + files: readonly SourceFile[], + swap: LineageSwap, +): Promise { + for (const file of files) { + if (file.kind === "file") await swapFile(filesystem, `${folder}/${file.relativePath}`, swap); + } +} + +async function swapFile(filesystem: Filesystem, path: string, swap: LineageSwap): Promise { + const { size } = await filesystem.stat(path); + for (let start = 0; start < size; start += WINDOW_BYTES) { + const end = Math.min(start + WINDOW_BYTES + SWAP_OVERLAP_BYTES, size) - 1; + const window = await readRange(filesystem, path, start, end); + const swapped = swapLineage(window, swap); + if (swapped.length !== window.length) { + throw new DriverCrashError(`swapping UDIDs in ${path} changed its length`); + } + if (!Buffer.from(swapped).equals(window)) await filesystem.writeAt(path, start, swapped); + } +} + +async function readRange( + filesystem: Filesystem, + path: string, + start: number, + end: number, +): Promise { + const file = await filesystem.openFile(path); + const chunks: Uint8Array[] = []; + const reader = file.read({ end, start }).getReader(); + for (;;) { + const piece = await reader.read(); + if (piece.done) break; + chunks.push(piece.value); + } + return Buffer.concat(chunks); +} diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index 4517a87ae..b06f78eb8 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -22,6 +22,8 @@ import { UDID_PATTERN } from "./device-set-leftovers.js"; import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; +import { swapCopyFiles } from "./lineage-files.js"; + import { type SimctlCatalog } from "./simctl-parse.js"; import { type IosDriverData, iosDriverData } from "./simulator-data.js"; @@ -219,6 +221,14 @@ export class Seeds { try { await this.#marks.writeMark(udid); await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); + if (seed.lineage !== undefined) { + await swapCopyFiles( + this.#filesystem, + join(this.#deviceRoot, udid), + this.#sources.filesOf(seed.sourceId), + { lineage: seed.lineage, own: udid }, + ); + } } catch (error: unknown) { // why: a clone whose mark or database drop failed is not a device Simlock hands out, and // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. @@ -235,6 +245,7 @@ export class Seeds { name, runtimeId: resolved.runtime.identifier, udid, + ...(seed.lineage === undefined ? {} : { lineage: seed.lineage }), } satisfies IosDriverData, }; } @@ -268,14 +279,19 @@ export class Seeds { } } - async #namedSource(sourceId: string): Promise<{ readonly udid: string }> { + async #namedSource(sourceId: string): Promise { const row = this.#sources.findRow(sourceId); if (row?.state !== "ready" || row.udid === undefined) { throw new DriverCrashError(`iOS keeps no ready source ${sourceId}`); } const reason = sourceStaleReason(row, await this.#simctl.loadCatalog()); if (reason !== undefined) throw new DriverCrashError(`source ${sourceId} is stale: ${reason}`); - return { udid: row.udid }; + return { + lineage: row.lineage, + runtimeBuild: row.runtimeBuild, + sourceId: row.sourceId, + udid: row.udid, + }; } /** why: two provisions of one kind at once make one seed, the second waiting for the first's make. */ @@ -365,15 +381,16 @@ export class Seeds { seedClone = cloned.stdout.trim(); this.#sources.setUdid(sourceId, seedClone); const done = throwaway; + const lineage = [done.toUpperCase()]; throwaway = undefined; keepRow = true; await this.#deleteThrowaway(done, false); keepRow = false; this.#sources.markReady(sourceId, await this.#sizeOf(seedClone), { files: [], - lineage: [], + lineage, }); - return { sourceId, udid: seedClone, runtimeBuild: resolved.runtime.build, lineage: [] }; + return { sourceId, udid: seedClone, runtimeBuild: resolved.runtime.build, lineage }; } catch (error: unknown) { if (seedClone !== undefined) await this.#deleteSeedSimulator(seedClone).catch(() => undefined); diff --git a/src/drivers/ios/udid-swap.ts b/src/drivers/ios/udid-swap.ts new file mode 100644 index 000000000..267adc15f --- /dev/null +++ b/src/drivers/ios/udid-swap.ts @@ -0,0 +1,20 @@ +/** why: a UDID is 36 characters, so its UTF-16 form is 72 bytes; a window overlaps the next by one byte less, and a match straddling two windows lies whole in one of them (ADR 0076 §4). */ +export const SWAP_OVERLAP_BYTES = 71; + +export interface LineageSwap { + readonly lineage: readonly string[]; + readonly own: string; +} + +/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own, byte for byte, so the output has the input's length. */ +export function swapLineage(chunk: Uint8Array, swap: LineageSwap): Uint8Array { + const out = Buffer.from(chunk); + const own = Buffer.from(swap.own, "latin1"); + for (const udid of swap.lineage) { + const needle = Buffer.from(udid, "latin1"); + for (let at = out.indexOf(needle); at !== -1; at = out.indexOf(needle, at + needle.length)) { + own.copy(out, at); + } + } + return out; +} diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index 835a32b7d..cd2a27677 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -121,6 +121,8 @@ export interface Filesystem { diskFree(path: string): Promise; diskSpace(path: string): Promise<{ readonly freeBytes: number; readonly totalBytes: number }>; directorySize(path: string): Promise; + /** why: `bytes` written into an existing file at `offset`, with no truncation, keeping its mode and modification time. */ + writeAt(path: string, offset: number, bytes: Uint8Array): Promise; } // why: only the directories that are absent get the mode, so one that is already there keeps its own. @@ -406,6 +408,17 @@ export class NodeFilesystem implements Filesystem { } } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { + const handle = await open(path, "r+"); + try { + const { atime, mtime } = await handle.stat(); + await handle.write(bytes, 0, bytes.length, offset); + await handle.utimes(atime, mtime); + } finally { + await handle.close(); + } + } + async directorySize(path: string): Promise { let total = 0; for (const entry of await readdir(path, { withFileTypes: true })) { @@ -766,6 +779,18 @@ export class MemoryFilesystem implements Filesystem { return total; } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { + this.#failIfDefined(path); + const entry = this.#entryAt(path); + if (entry.kind !== "file") throw errnoError("EISDIR", `Is a directory: ${path}`); + const old = fileBytes(entry); + const next = new Uint8Array(Math.max(old.byteLength, offset + bytes.byteLength)); + next.set(old); + next.set(bytes, offset); + entry.bytes = next; + entry.contents = Buffer.from(next).toString("utf8"); + } + /** * why: Test-only: makes every operation on `path` fail with `code`, which is the only way to * reach the branches that turn an unexpected filesystem failure into a typed rejection. From 9447c02ed3ec6e66ea59d18c81e48ee3935c25c0 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:26:11 +0100 Subject: [PATCH 27/48] test: a source made is flushed, its lineage stored and its files naming the lineage recorded (#1570) --- src/drivers/ios/driver-options.ts | 3 + src/drivers/ios/index-seed-record.test.ts | 302 ++++++++++++++++++++++ src/drivers/ios/test-index.ts | 4 +- src/ports/index.ts | 1 + src/ports/sqlite.ts | 43 +++ 5 files changed, 352 insertions(+), 1 deletion(-) create mode 100644 src/drivers/ios/index-seed-record.test.ts diff --git a/src/drivers/ios/driver-options.ts b/src/drivers/ios/driver-options.ts index 10da15860..9c969ad8b 100644 --- a/src/drivers/ios/driver-options.ts +++ b/src/drivers/ios/driver-options.ts @@ -8,6 +8,7 @@ import type { Logger, ProcessRunner, SqliteDatabase, + SqliteOpener, TcpListenerFactory, TcpProbe, TcpStreamConnector, @@ -35,6 +36,8 @@ export interface IosSimctlDriverOptions { readonly coreSimulatorRoot?: string; // why: where CoreSimulator keeps each simulator's log folder, which `simctl delete` leaves behind; with none given the driver deletes no log folder. readonly logsRoot?: string; + // why: how a source's WALs are flushed when it is made; with none given the driver flushes nothing. + readonly sqliteOpener?: SqliteOpener; // why: where launchd_sim keeps each simulator's overrides folder; a test points it elsewhere. readonly launchdOverridesRoot?: string; readonly onSlimmed?: (fact: SlimmedFact) => void; diff --git a/src/drivers/ios/index-seed-record.test.ts b/src/drivers/ios/index-seed-record.test.ts new file mode 100644 index 000000000..3e44aa9bc --- /dev/null +++ b/src/drivers/ios/index-seed-record.test.ts @@ -0,0 +1,302 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + FakeClock, + FakeSqliteOpener, + MemoryFilesystem, + ScriptedProcessRunner, + type SqliteDatabase, +} from "../../ports/index.js"; +import { memorySourceDatabase, plantBytes } from "./testing.js"; +import { deviceRoot, driverData, plantManagedDevice, simctl } from "./test-slim.js"; +import { + RecordingLogger, + createDriver, + listFixture, + listInvocation, + logsRoot, + scopedRunners, + seedUdid, + sourcesSimctl, + spec, + throwawaySeedSteps, + throwawayUdid, +} from "./test-index.js"; + +afterEach(() => { + scopedRunners.length = 0; +}); + +const copyUdid = "CCCCCCCC-4444-5555-6666-DDDDDDDDDDDD"; +const lineageUdid = throwawayUdid.toUpperCase(); +const sourcesSet = `${deviceRoot}/sources`; +const seedFolder = `${sourcesSet}/${seedUdid}`; +const SQLITE_HEADER = Buffer.from("SQLite format 3\u0000"); +const MEBIBYTE = 1_048_576; + +function utf16be(text: string): Buffer { + return Buffer.from(text, "utf16le").swap16(); +} + +class UnreadableFiles extends MemoryFilesystem { + readonly unreadable = new Set(); + + override async openFile(path: string): ReturnType { + if (this.unreadable.has(path)) { + throw Object.assign(new Error(`EACCES: permission denied, open '${path}'`), { + code: "EACCES", + }); + } + return super.openFile(path); + } +} + +interface Made { + readonly database: SqliteDatabase; + readonly error: unknown; + readonly filesystem: UnreadableFiles; + readonly logger: RecordingLogger; + readonly opener: FakeSqliteOpener; + readonly runner: ScriptedProcessRunner; +} + +/** why: the first provision of a kind, with `plant` writing what the seed's clone holds and `fail` naming what the fake filesystem and opener refuse. */ +async function provisionNewSeed(options: { + readonly plant: (filesystem: UnreadableFiles) => Promise; + readonly failingPaths?: readonly string[]; + readonly unreadablePaths?: readonly string[]; + readonly failingDatabases?: readonly string[]; + readonly keepingFrames?: readonly string[]; +}): Promise { + const database = await memorySourceDatabase(); + const filesystem = new UnreadableFiles(); + await plantManagedDevice(filesystem); + await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); + await filesystem.mkdirp(`${logsRoot}/${throwawayUdid}`); + await options.plant(filesystem); + for (const path of options.failingPaths ?? []) filesystem.defineFailure(path, "EACCES"); + for (const path of options.unreadablePaths ?? []) filesystem.unreadable.add(path); + const opener = new FakeSqliteOpener(filesystem, { + failing: options.failingDatabases ?? [], + keepingFrames: options.keepingFrames ?? [], + }); + const logger = new RecordingLogger(); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + ...throwawaySeedSteps(), + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + ...[sourcesSimctl("shutdown", seedUdid), sourcesSimctl("delete", seedUdid)].map((match) => ({ + match, + })), + ]); + const driver = await createDriver( + runner, + new FakeClock(), + filesystem, + database, + logger, + undefined, + { + sqliteOpener: opener, + }, + ); + const error = await driver.provision(spec).then( + () => undefined, + (caught: unknown) => caught, + ); + return { database, error, filesystem, logger, opener, runner }; +} + +const baselineUdid = "EEEEEEEE-7777-8888-9999-FFFFFFFFFFFF"; + +/** why: a save of the lease `driverData` made, with the clone's files planted, so the baseline's record can be read back. */ +async function saveBaseline(options: { + readonly lineage?: readonly string[]; + readonly plant: (filesystem: UnreadableFiles) => Promise; +}): Promise { + const database = await memorySourceDatabase(); + const filesystem = new UnreadableFiles(); + await plantManagedDevice(filesystem); + await options.plant(filesystem); + const opener = new FakeSqliteOpener(filesystem); + const logger = new RecordingLogger(); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { match: simctl("shutdown", driverData.udid) }, + { + match: simctl("clone", driverData.udid, "simlock-baseline-device-1", sourcesSet), + result: { code: 0, stderr: "", stdout: `${baselineUdid}\n` }, + }, + ]); + const driver = await createDriver( + runner, + new FakeClock(), + filesystem, + database, + logger, + undefined, + { + sqliteOpener: opener, + }, + ); + const error = await driver + .saveSource({ + address: driverData.udid, + deviceId: driverData.udid, + driverData: { + ...driverData, + ...(options.lineage === undefined ? {} : { lineage: options.lineage }), + }, + physical: false, + }) + .then( + () => undefined, + (caught: unknown) => caught, + ); + return { database, error, filesystem, logger, opener, runner }; +} + +function recorded(database: SqliteDatabase): unknown[] { + return database + .prepare("SELECT relative_path, kind FROM ios_source_files ORDER BY relative_path") + .all(); +} + +function storedLineage(database: SqliteDatabase): unknown { + return database.prepare("SELECT lineage, saved_without_lineage FROM ios_source_data").get(); +} + +describe("IosSimctlDriver source records", () => { + it("a source made is flushed, its lineage stored and every file or link naming the lineage recorded before it turns ready", async () => { + const naming = Buffer.from(`root /Devices/${lineageUdid}/data`); + const big = Buffer.alloc(64 * MEBIBYTE + 100); + naming.copy(big, big.length - naming.length); + + const seed = await provisionNewSeed({ + plant: async (filesystem) => { + await plantBytes(filesystem, `${seedFolder}/data/a.plist`, naming); + await plantBytes(filesystem, `${seedFolder}/data/big.bin`, big); + await plantBytes(filesystem, `${seedFolder}/data/wide.bin`, utf16be(lineageUdid)); + await plantBytes(filesystem, `${seedFolder}/data/none.txt`, Buffer.from("nothing")); + await plantBytes(filesystem, `${seedFolder}/data/one.db`, SQLITE_HEADER); + await plantBytes(filesystem, `${seedFolder}/data/one.db-wal`, naming); + await plantBytes(filesystem, `${seedFolder}/data/two.db`, SQLITE_HEADER); + await plantBytes(filesystem, `${seedFolder}/data/two.db-wal`, naming); + filesystem.defineSymlink(`${seedFolder}/data/Logs`, `${logsRoot}/${lineageUdid}`); + }, + }); + + expect(seed.error).toBeUndefined(); + expect(seed.opener.checkpoints.sort()).toEqual([ + `${seedFolder}/data/one.db`, + `${seedFolder}/data/two.db`, + ]); + expect(storedLineage(seed.database)).toEqual({ + lineage: JSON.stringify([lineageUdid]), + saved_without_lineage: 0, + }); + expect(recorded(seed.database)).toEqual([ + { kind: "link", relative_path: "data/Logs" }, + { kind: "file", relative_path: "data/a.plist" }, + { kind: "file", relative_path: "data/big.bin" }, + { kind: "file", relative_path: "data/wide.bin" }, + ]); + expect(seed.logger.warnings).toEqual([]); + + const unflushed = await provisionNewSeed({ + failingDatabases: [`${seedFolder}/data/one.db`], + keepingFrames: [`${seedFolder}/data/two.db`], + plant: async (filesystem) => { + await plantBytes(filesystem, `${seedFolder}/data/one.db`, SQLITE_HEADER); + await plantBytes(filesystem, `${seedFolder}/data/one.db-wal`, naming); + await plantBytes(filesystem, `${seedFolder}/data/two.db`, SQLITE_HEADER); + await plantBytes(filesystem, `${seedFolder}/data/two.db-wal`, naming); + }, + }); + expect(unflushed.error).toBeUndefined(); + expect(unflushed.logger.warnings).toEqual([ + `could not flush ${seedFolder}/data/one.db: unable to open database file ${seedFolder}/data/one.db`, + `WAL ${seedFolder}/data/two.db-wal still holds frames after the flush; left unpatched`, + ]); + expect(recorded(unflushed.database), "a WAL is never recorded").toEqual([]); + + const unreadable = await provisionNewSeed({ + plant: async (filesystem) => { + await plantBytes(filesystem, `${seedFolder}/data/secret.db`, naming); + await plantBytes(filesystem, `${seedFolder}/data/other.txt`, Buffer.from("nothing")); + }, + unreadablePaths: [`${seedFolder}/data/secret.db`], + }); + expect(unreadable.error).toBeUndefined(); + expect(recorded(unreadable.database)).toEqual([ + { kind: "unreadable", relative_path: "data/secret.db" }, + ]); + expect(unreadable.logger.warnings).toEqual([ + `could not read ${seedFolder}/data/secret.db in source ${seedUdid}: EACCES: permission denied, open '${seedFolder}/data/secret.db'; recorded as unreadable`, + ]); + + const namingNothing = await provisionNewSeed({ + plant: (filesystem) => + plantBytes(filesystem, `${seedFolder}/data/none.txt`, Buffer.from("nothing")), + }); + expect(namingNothing.error).toBeUndefined(); + expect(recorded(namingNothing.database)).toEqual([]); + expect(storedLineage(namingNothing.database)).toEqual({ + lineage: JSON.stringify([lineageUdid]), + saved_without_lineage: 0, + }); + + const walkFails = await provisionNewSeed({ + failingPaths: [`${seedFolder}/data/sealed`], + plant: async (filesystem) => { + await plantBytes(filesystem, `${seedFolder}/data/sealed/a.plist`, naming); + await plantBytes(filesystem, `${seedFolder}/data/b.plist`, naming); + }, + }); + expect(walkFails.error).toMatchObject({ code: "EACCES" }); + expect( + walkFails.runner.calls.some( + (call) => call.args.join(" ") === `simctl --set ${sourcesSet} delete ${seedUdid}`, + ), + "the seed's clone is deleted", + ).toBe(true); + expect(walkFails.database.prepare("SELECT source_id FROM ios_source_data").all()).toEqual([]); + expect(recorded(walkFails.database)).toEqual([]); + + const baselineFolder = `${sourcesSet}/${baselineUdid}`; + const lease = driverData.udid.toUpperCase(); + const fromUsedLease = await saveBaseline({ + lineage: [lineageUdid], + plant: async (filesystem) => { + await plantBytes(filesystem, `${baselineFolder}/data/lease.plist`, Buffer.from(lease)); + await plantBytes(filesystem, `${baselineFolder}/data/seed.plist`, naming); + await plantBytes(filesystem, `${baselineFolder}/data/none.txt`, Buffer.from("nothing")); + await plantBytes(filesystem, `${baselineFolder}/data/one.db`, SQLITE_HEADER); + await plantBytes(filesystem, `${baselineFolder}/data/one.db-wal`, naming); + }, + }); + expect(fromUsedLease.error).toBeUndefined(); + expect(fromUsedLease.opener.checkpoints).toEqual([`${baselineFolder}/data/one.db`]); + expect(storedLineage(fromUsedLease.database)).toEqual({ + lineage: JSON.stringify([lease, lineageUdid]), + saved_without_lineage: 0, + }); + expect(recorded(fromUsedLease.database)).toEqual([ + { kind: "file", relative_path: "data/lease.plist" }, + { kind: "file", relative_path: "data/seed.plist" }, + ]); + + const fromOldDevice = await saveBaseline({ + plant: (filesystem) => + plantBytes(filesystem, `${baselineFolder}/data/lease.plist`, Buffer.from(lease)), + }); + expect(fromOldDevice.error).toBeUndefined(); + expect(storedLineage(fromOldDevice.database)).toEqual({ + lineage: null, + saved_without_lineage: 1, + }); + expect(recorded(fromOldDevice.database)).toEqual([]); + }); +}); diff --git a/src/drivers/ios/test-index.ts b/src/drivers/ios/test-index.ts index ce47934de..5fbc0427a 100644 --- a/src/drivers/ios/test-index.ts +++ b/src/drivers/ios/test-index.ts @@ -8,7 +8,7 @@ import { MemoryFilesystem, type Logger, } from "../../ports/index.js"; -import { IosSimctlDriver, SLIM_CATEGORIES } from "./index.js"; +import { IosSimctlDriver, type IosSimctlDriverOptions, SLIM_CATEGORIES } from "./index.js"; import { plantReadySeed, memorySourceDatabase, type PlantedSource } from "./testing.js"; import { simctl, @@ -218,9 +218,11 @@ export async function createDriver( database?: SqliteDatabase, logger?: Logger, fullBootTimeoutMs?: number, + extra: Partial = {}, ): Promise { scopedRunners.push(runner); return IosSimctlDriver.create({ + ...extra, database: database ?? (await memorySourceDatabase()), clock, driverConfig: { deviceRoot }, diff --git a/src/ports/index.ts b/src/ports/index.ts index 869a4d0bf..821ff3ebf 100644 --- a/src/ports/index.ts +++ b/src/ports/index.ts @@ -161,6 +161,7 @@ export { WindowsHostLoadProbe, } from "./host-load.js"; export { + FakeSqliteOpener, filterSqliteExperimentalWarning, inTransaction, isLockError, diff --git a/src/ports/sqlite.ts b/src/ports/sqlite.ts index dabff6a28..bdfa62215 100644 --- a/src/ports/sqlite.ts +++ b/src/ports/sqlite.ts @@ -1,5 +1,7 @@ import { closeSync, openSync } from "node:fs"; +import type { MemoryFilesystem } from "./filesystem.js"; + export interface SqliteRunResult { readonly changes: number | bigint; readonly lastInsertRowid: number | bigint; @@ -46,6 +48,47 @@ export class NodeSqliteOpener implements SqliteOpener { } } +/** + * why: a fake for a caller that opens a database only to flush its WAL: it records each database + * a checkpoint ran on, can fail one, and empties the `-wal` file in a memory filesystem unless told + * to leave its frames. + */ +export class FakeSqliteOpener implements SqliteOpener { + readonly checkpoints: string[] = []; + readonly #filesystem: MemoryFilesystem; + readonly #failing: ReadonlySet; + readonly #keepingFrames: ReadonlySet; + + constructor( + filesystem: MemoryFilesystem, + options: { + readonly failing?: readonly string[]; + readonly keepingFrames?: readonly string[]; + } = {}, + ) { + this.#filesystem = filesystem; + this.#failing = new Set(options.failing); + this.#keepingFrames = new Set(options.keepingFrames); + } + + async open(path: string): Promise { + if (this.#failing.has(path)) throw new Error(`unable to open database file ${path}`); + const checkpoint = (): void => { + this.checkpoints.push(path); + if (!this.#keepingFrames.has(path)) void this.#filesystem.rm(`${path}-wal`); + }; + return { + close: () => undefined, + exec: (sql) => { + if (sql === "PRAGMA wal_checkpoint(TRUNCATE)") checkpoint(); + }, + prepare: () => { + throw new Error("FakeSqliteOpener runs no statements"); + }, + }; + } +} + const SQLITE_WARNING_PREFIX = "SQLite is an experimental feature"; export function filterSqliteExperimentalWarning(): void { From bdc70a3955e73fca08b1918c63a171e145a3183a Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:28:08 +0100 Subject: [PATCH 28/48] feat(ios): a source made is flushed, its lineage stored and its files naming the lineage recorded (#1570) --- src/auth/admin-secret.test.ts | 6 + src/drivers/ios/index-device-set.test.ts | 7 ++ src/drivers/ios/index-seed-provision.test.ts | 7 ++ src/drivers/ios/index-seed-record.test.ts | 4 +- src/drivers/ios/index-seed-sources.test.ts | 6 + src/drivers/ios/index-seed-throwaway.test.ts | 9 +- src/drivers/ios/index.ts | 1 + src/drivers/ios/lineage-files.ts | 121 +++++++++++++++++-- src/drivers/ios/seeds.ts | 34 ++++-- src/drivers/ios/test-index.ts | 9 ++ src/drivers/ios/udid-swap.ts | 15 +++ src/ports/filesystem.ts | 14 +++ 12 files changed, 214 insertions(+), 19 deletions(-) diff --git a/src/auth/admin-secret.test.ts b/src/auth/admin-secret.test.ts index 984f264d6..69b21d617 100644 --- a/src/auth/admin-secret.test.ts +++ b/src/auth/admin-secret.test.ts @@ -105,6 +105,9 @@ describe("AdminSecretManager", () => { writeAt: () => { throw new Error("verify() must not touch the filesystem"); }, + readlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("verify() must not touch the filesystem"); }, @@ -195,6 +198,9 @@ describe("AdminSecretManager", () => { writeAt: () => { throw new Error("must not touch the filesystem"); }, + readlink: () => { + throw new Error("must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("must not touch the filesystem"); }, diff --git a/src/drivers/ios/index-device-set.test.ts b/src/drivers/ios/index-device-set.test.ts index c0eea0a21..243f91a42 100644 --- a/src/drivers/ios/index-device-set.test.ts +++ b/src/drivers/ios/index-device-set.test.ts @@ -122,6 +122,13 @@ describe("IosSimctlDriver device set", () => { : []), { match: { args: argvOf(step), command: "xcrun" }, + ...(step.args[0] === "clone" && step.args[1] === throwawayUdid + ? { + onSpawn: () => { + void filesystem.mkdirp(`${sourcesSet}/${seedUdid}`); + }, + } + : {}), ...(step.stdout === undefined ? {} : { result: { code: 0, stderr: "", stdout: step.stdout } }), diff --git a/src/drivers/ios/index-seed-provision.test.ts b/src/drivers/ios/index-seed-provision.test.ts index 27abdc7b3..0d856ec74 100644 --- a/src/drivers/ios/index-seed-provision.test.ts +++ b/src/drivers/ios/index-seed-provision.test.ts @@ -26,6 +26,7 @@ import { listFixture, listInvocation, scopedRunners, + plantSeedClone, seedUdid, seededDatabase, sourcesSimctl, @@ -76,6 +77,7 @@ describe("IosSimctlDriver", () => { full, ); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await driver.provision(spec); @@ -116,6 +118,7 @@ describe("IosSimctlDriver", () => { const filesystem = new ModeRecordingFilesystem(); const driver = await createDriver(runner, new FakeClock(), filesystem, database); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); await expect(driver.provision(spec)).resolves.toEqual({ @@ -163,6 +166,7 @@ describe("IosSimctlDriver", () => { ]); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); const driver = await createDriver(runner, new FakeClock(), filesystem, database); const error = await driver.provision(spec).catch((caught: unknown) => caught); @@ -189,6 +193,7 @@ describe("IosSimctlDriver", () => { const clock = new FakeClock(); const driver = await createDriver(runner, clock, filesystem); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); const clones = (): number => runner.calls.filter((call) => call.args[3] === "clone" && call.args[4] === seedUdid).length; @@ -266,6 +271,7 @@ describe("IosSimctlDriver", () => { scopedRunners.push(runner); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); const clock = new FakeClock(); const config = { simulator: { audio: true, createTimeoutMs: 120_000, headless: true } }; const driver = await IosSimctlDriver.create({ @@ -393,6 +399,7 @@ describe("IosSimctlDriver", () => { }; const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); const clock = new FakeClock(); const driver = await IosSimctlDriver.create({ database: seeded ? await seededDatabase() : await memorySourceDatabase(), diff --git a/src/drivers/ios/index-seed-record.test.ts b/src/drivers/ios/index-seed-record.test.ts index 3e44aa9bc..fd5d9d4c6 100644 --- a/src/drivers/ios/index-seed-record.test.ts +++ b/src/drivers/ios/index-seed-record.test.ts @@ -70,9 +70,11 @@ async function provisionNewSeed(options: { const database = await memorySourceDatabase(); const filesystem = new UnreadableFiles(); await plantManagedDevice(filesystem); - await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); await filesystem.mkdirp(`${logsRoot}/${throwawayUdid}`); await options.plant(filesystem); + // why: `simctl clone` gives the copy what the seed holds, and the scripted runner cannot. + await filesystem.mkdirp(seedFolder); + await filesystem.copyTree(seedFolder, `${deviceRoot}/${copyUdid}`, { timeoutMs: 1 }); for (const path of options.failingPaths ?? []) filesystem.defineFailure(path, "EACCES"); for (const path of options.unreadablePaths ?? []) filesystem.unreadable.add(path); const opener = new FakeSqliteOpener(filesystem, { diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts index 145896e29..a1316bd29 100644 --- a/src/drivers/ios/index-seed-sources.test.ts +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -20,6 +20,7 @@ import { listInvocation, plantLaunchServices, scopedRunners, + plantSeedClone, seedUdid, seededDatabase, sourcesSimctl, @@ -122,6 +123,7 @@ describe("IosSimctlDriver", () => { it("provision deletes the clone and throws when the LaunchServices database cannot be dropped", async () => { const filesystem = new FaultyFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); filesystem.failing = "rm"; const runner = new ScriptedProcessRunner([ @@ -159,6 +161,7 @@ describe("IosSimctlDriver", () => { .run(oldUdid); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); const catalog = JSON.parse(listFixture) as { runtimes: { buildversion?: string; identifier: string }[]; @@ -225,6 +228,7 @@ describe("IosSimctlDriver", () => { ]); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); const driver = await createDriver(runner, new FakeClock(now), filesystem, database); @@ -418,6 +422,7 @@ describe("IosSimctlDriver", () => { ]); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); const driver = await createDriver( runner, new FakeClock(), @@ -567,6 +572,7 @@ describe("IosSimctlDriver", () => { const orphanUdid = "00000000-0000-0000-0000-0000000000ee"; const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await filesystem.mkdirp(`${deviceRoot}/sources/${seedUdid}`); await filesystem.mkdirp(`${deviceRoot}/sources/${orphanUdid}`); const database = await seededDatabase(); diff --git a/src/drivers/ios/index-seed-throwaway.test.ts b/src/drivers/ios/index-seed-throwaway.test.ts index bbaa78b37..7bd5d6ffb 100644 --- a/src/drivers/ios/index-seed-throwaway.test.ts +++ b/src/drivers/ios/index-seed-throwaway.test.ts @@ -16,6 +16,7 @@ import { logsRoot, scopedRunners, listInvocation, + plantSeedClone, seedUdid, sourcesSimctl, spec, @@ -107,7 +108,12 @@ describe("IosSimctlDriver seed throwaway", () => { step(inSet("bootstatus", throwawayUdid, "-b")), step(inSet("listapps", throwawayUdid)), step(inSet("shutdown", throwawayUdid)), - step(inSet("clone", throwawayUdid, name, world.sourcesSet), `${seedUdid}\n`), + { + ...step(inSet("clone", throwawayUdid, name, world.sourcesSet), `${seedUdid}\n`), + onSpawn: () => { + void filesystem.mkdirp(`${world.sourcesSet}/${seedUdid}`); + }, + }, step(inSet("delete", throwawayUdid)), ...hub(`Delete :DevicePreferences:${throwawayUdid}`), step( @@ -233,6 +239,7 @@ describe("IosSimctlDriver seed throwaway", () => { ]); scopedRunners.push(runner); await plantManagedDevice(filesystem); + await plantSeedClone(filesystem); await filesystem.mkdirp(`${deviceRoot}/${copyUdid}/data`); if (failure.absent !== true) await filesystem.mkdirp(`${logsRoot}/${throwawayUdid}`); const driver = await IosSimctlDriver.create({ diff --git a/src/drivers/ios/index.ts b/src/drivers/ios/index.ts index 5a5360730..e5f82bae2 100644 --- a/src/drivers/ios/index.ts +++ b/src/drivers/ios/index.ts @@ -225,6 +225,7 @@ export class IosSimctlDriver implements Driver { seedTurns: this.#seedTurns, simctl: this.#simctl, sourcesSet: sourcesSetFor(options.simlockHome, deviceRoot, rootOptions !== undefined), + sqliteOpener: options.sqliteOpener, }); this.screenCapture = createScreenCaptureFor( options, diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts index 715c0aa36..9db518ea7 100644 --- a/src/drivers/ios/lineage-files.ts +++ b/src/drivers/ios/lineage-files.ts @@ -1,11 +1,19 @@ import { DriverCrashError } from "../../core/index.js"; -import type { Filesystem } from "../../ports/index.js"; +import { + isMissingPathError, + type Filesystem, + type Logger, + type SqliteOpener, +} from "../../ports/index.js"; +import { errorMessage } from "./command-run.js"; import type { SourceFile } from "./source-data.js"; -import { SWAP_OVERLAP_BYTES, swapLineage, type LineageSwap } from "./udid-swap.js"; +import { SWAP_OVERLAP_BYTES, namesLineage, swapLineage, type LineageSwap } from "./udid-swap.js"; /** why: a window is read whole, so this bounds memory at any file size (ADR 0076 §3). */ const WINDOW_BYTES = 1_048_576; +const WAL_SUFFIX = "-wal"; + /** why: ADR 0076 §4: before its first boot a copy's recorded files name its own UDID in place of the lineage's, each file keeping its length. */ export async function swapCopyFiles( filesystem: Filesystem, @@ -14,21 +22,114 @@ export async function swapCopyFiles( swap: LineageSwap, ): Promise { for (const file of files) { - if (file.kind === "file") await swapFile(filesystem, `${folder}/${file.relativePath}`, swap); + if (file.kind !== "file") continue; + const path = `${folder}/${file.relativePath}`; + await eachWindow(filesystem, path, async (window, start) => { + const swapped = swapLineage(window, swap); + if (swapped.length !== window.length) { + throw new DriverCrashError(`swapping UDIDs in ${path} changed its length`); + } + if (!Buffer.from(swapped).equals(window)) await filesystem.writeAt(path, start, swapped); + return false; + }); + } +} + +export interface RecordOptions { + readonly filesystem: Filesystem; + /** why: the source's UDID folder; recorded paths are relative to it. */ + readonly folder: string; + readonly lineage: readonly string[]; + readonly logger: Logger | undefined; + readonly sqlite: SqliteOpener | undefined; + readonly udid: string; +} + +/** + * why: ADR 0076 §3: when a source is made every WAL in it is flushed into its database, and the files and links that name a lineage UDID are found once and recorded, so a copy patches only those. + */ +export async function recordLineageFiles(options: RecordOptions): Promise { + const files: SourceFile[] = []; + const visit = async (directory: string, relative: string): Promise => { + for (const name of (await options.filesystem.readdir(directory)).sort()) { + const path = `${directory}/${name}`; + const relativePath = relative === "" ? name : `${relative}/${name}`; + const details = await options.filesystem.lstat(path); + if (details.kind === "directory") { + await visit(path, relativePath); + } else if (details.kind === "symlink") { + const target = await options.filesystem.readlink(path); + if (namesLineage(Buffer.from(target), options.lineage)) { + files.push({ kind: "link", relativePath }); + } + } else if (details.kind === "file") { + if (name.endsWith(WAL_SUFFIX)) await flushWal(options, path); + else { + const kind = await classify(options, path); + if (kind !== undefined) files.push({ kind, relativePath }); + } + } + } + }; + await visit(options.folder, ""); + return files; +} + +async function flushWal(options: RecordOptions, wal: string): Promise { + if (options.sqlite === undefined) return; + const database = wal.slice(0, -WAL_SUFFIX.length); + try { + const opened = await options.sqlite.open(database, { readOnly: false }); + try { + opened.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + } finally { + opened.close(); + } + } catch (error: unknown) { + options.logger?.warn(`could not flush ${database}: ${errorMessage(error)}`); + return; + } + const left = await options.filesystem.stat(wal).then( + (found) => found.size, + (error: unknown) => { + if (isMissingPathError(error)) return 0; + throw error; + }, + ); + if (left > 0) { + options.logger?.warn(`WAL ${wal} still holds frames after the flush; left unpatched`); } } -async function swapFile(filesystem: Filesystem, path: string, swap: LineageSwap): Promise { +async function classify( + options: RecordOptions, + path: string, +): Promise<"file" | "unreadable" | undefined> { + try { + const names = await eachWindow(options.filesystem, path, (window) => + Promise.resolve(namesLineage(window, options.lineage)), + ); + return names ? "file" : undefined; + } catch (error: unknown) { + options.logger?.warn( + `could not read ${path} in source ${options.udid}: ${errorMessage(error)}; recorded as unreadable`, + ); + return "unreadable"; + } +} + +/** why: windows overlap by one UDID in UTF-16 less a byte, so a UDID spanning two reads is whole in one; `visit` answers true to stop. */ +async function eachWindow( + filesystem: Filesystem, + path: string, + visit: (window: Buffer, start: number) => Promise, +): Promise { const { size } = await filesystem.stat(path); for (let start = 0; start < size; start += WINDOW_BYTES) { const end = Math.min(start + WINDOW_BYTES + SWAP_OVERLAP_BYTES, size) - 1; - const window = await readRange(filesystem, path, start, end); - const swapped = swapLineage(window, swap); - if (swapped.length !== window.length) { - throw new DriverCrashError(`swapping UDIDs in ${path} changed its length`); - } - if (!Buffer.from(swapped).equals(window)) await filesystem.writeAt(path, start, swapped); + if (await visit(await readRange(filesystem, path, start, end), start)) return true; } + return false; } async function readRange( diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index b06f78eb8..7817af862 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -22,13 +22,13 @@ import { UDID_PATTERN } from "./device-set-leftovers.js"; import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; -import { swapCopyFiles } from "./lineage-files.js"; +import { recordLineageFiles, swapCopyFiles } from "./lineage-files.js"; import { type SimctlCatalog } from "./simctl-parse.js"; import { type IosDriverData, iosDriverData } from "./simulator-data.js"; -import type { SqliteDatabase } from "../../ports/index.js"; +import type { SqliteDatabase, SqliteOpener } from "../../ports/index.js"; /** why: ADR 0061 §1: a `making` row of this process older than this is a make that never ended. */ const STALE_MAKING_MS = 300_000; @@ -53,6 +53,7 @@ interface SeedsOptions { readonly seedTurns: SeedTurns; readonly simctl: Simctl; readonly sourcesSet: string; + readonly sqliteOpener: SqliteOpener | undefined; } /** why: the part of IosSimctlDriver (ADR 0063) that makes, keeps and deletes the seeds every provision copies (ADR 0061), and lists them as sources. */ @@ -75,6 +76,7 @@ export class Seeds { readonly #simctl: Simctl; readonly #sources: IosSourceTable; readonly #sourcesSet: string; + readonly #sqliteOpener: SqliteOpener | undefined; #sourceMaker: string | undefined; #swept = false; @@ -96,6 +98,7 @@ export class Seeds { this.#simctl = options.simctl; this.#sources = new IosSourceTable(options.database); this.#sourcesSet = options.sourcesSet; + this.#sqliteOpener = options.sqliteOpener; } async listSources(): Promise { @@ -162,8 +165,16 @@ export class Seeds { this.#sources.setUdid(sourceId, cloneUdid); await this.#slimBoot.copyOverrides(data.udid, cloneUdid); await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); + // why: ADR 0076 §2: the baseline's lineage is its lease's UDID and the lineage the lease was copied with; a lease made before lineages were kept has none, so the baseline is never copied. + const lineage = + data.lineage === undefined ? undefined : [data.udid.toUpperCase(), ...data.lineage]; + const files = lineage === undefined ? [] : await this.#recordFiles(cloneUdid, lineage); const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes, { files: [], lineage: [] }); + this.#sources.markReady(sourceId, sizeBytes, { + files, + lineage, + savedWithoutLineage: lineage === undefined, + }); // why: ADR 0076: the lease is destroyed next, and `simctl delete` leaves its log folder behind for the baseline's copies to inherit. await this.#deleteLogFolder(data.udid); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; @@ -386,10 +397,8 @@ export class Seeds { keepRow = true; await this.#deleteThrowaway(done, false); keepRow = false; - this.#sources.markReady(sourceId, await this.#sizeOf(seedClone), { - files: [], - lineage, - }); + const files = await this.#recordFiles(seedClone, lineage); + this.#sources.markReady(sourceId, await this.#sizeOf(seedClone), { files, lineage }); return { sourceId, udid: seedClone, runtimeBuild: resolved.runtime.build, lineage }; } catch (error: unknown) { if (seedClone !== undefined) @@ -485,6 +494,17 @@ export class Seeds { }); } + #recordFiles(udid: string, lineage: readonly string[]): ReturnType { + return recordLineageFiles({ + filesystem: this.#filesystem, + folder: join(this.#sourcesSet, udid), + lineage, + logger: this.#logger, + sqlite: this.#sqliteOpener, + udid, + }); + } + async #sizeOf(udid: string): Promise { try { return await this.#filesystem.directorySize(join(this.#sourcesSet, udid)); diff --git a/src/drivers/ios/test-index.ts b/src/drivers/ios/test-index.ts index 5fbc0427a..f4ca97fe3 100644 --- a/src/drivers/ios/test-index.ts +++ b/src/drivers/ios/test-index.ts @@ -397,3 +397,12 @@ export function throwawaySeedSteps( { match: simctl("delete", throwawayUdid) }, ]; } + +/** why: `simctl clone` made the seed's folder in the sources set, which the scripted runner cannot do, and a make walks it before the seed turns ready. */ +export async function plantSeedClone( + filesystem: Filesystem, + sourcesSet: string = `${deviceRoot}/sources`, + udid: string = seedUdid, +): Promise { + await filesystem.mkdirp(`${sourcesSet}/${udid}`); +} diff --git a/src/drivers/ios/udid-swap.ts b/src/drivers/ios/udid-swap.ts index 267adc15f..ebc44cba9 100644 --- a/src/drivers/ios/udid-swap.ts +++ b/src/drivers/ios/udid-swap.ts @@ -6,6 +6,21 @@ export interface LineageSwap { readonly own: string; } +/** why: ADR 0076 §4: a UDID is stored upper or lower case, as ASCII or as UTF-16 of either byte order, and no other way is looked for. */ +function forms(udid: string): readonly Buffer[] { + return [udid.toUpperCase(), udid.toLowerCase()].flatMap((text) => [ + Buffer.from(text, "latin1"), + Buffer.from(text, "utf16le"), + Buffer.from(text, "utf16le").swap16(), + ]); +} + +/** why: the finder of ADR 0076 §3: whether any lineage UDID is in the bytes in any of its forms. */ +export function namesLineage(chunk: Uint8Array, lineage: readonly string[]): boolean { + const bytes = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + return lineage.some((udid) => forms(udid).some((form) => bytes.includes(form))); +} + /** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own, byte for byte, so the output has the input's length. */ export function swapLineage(chunk: Uint8Array, swap: LineageSwap): Uint8Array { const out = Buffer.from(chunk); diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index cd2a27677..ae51e78b6 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -9,6 +9,7 @@ import { open, readdir, readFile, + readlink, realpath, rename, rm, @@ -121,6 +122,8 @@ export interface Filesystem { diskFree(path: string): Promise; diskSpace(path: string): Promise<{ readonly freeBytes: number; readonly totalBytes: number }>; directorySize(path: string): Promise; + /** why: what a symbolic link points at, without following it. */ + readlink(path: string): Promise; /** why: `bytes` written into an existing file at `offset`, with no truncation, keeping its mode and modification time. */ writeAt(path: string, offset: number, bytes: Uint8Array): Promise; } @@ -408,6 +411,10 @@ export class NodeFilesystem implements Filesystem { } } + async readlink(path: string): Promise { + return readlink(path); + } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { const handle = await open(path, "r+"); try { @@ -779,6 +786,13 @@ export class MemoryFilesystem implements Filesystem { return total; } + async readlink(path: string): Promise { + this.#failIfDefined(path); + const entry = this.#rawEntryAt(this.#underResolvedParent(path)); + if (entry.kind !== "symlink") throw errnoError("EINVAL", `Not a symbolic link: ${path}`); + return entry.target; + } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { this.#failIfDefined(path); const entry = this.#entryAt(path); From 249b697550a33829eb7643ba5c63f3de25234ff3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:28:48 +0100 Subject: [PATCH 29/48] test: a copy whose swap fails is deleted and the provision fails (#1570) --- src/drivers/ios/index-seed-swap.test.ts | 66 ++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/src/drivers/ios/index-seed-swap.test.ts b/src/drivers/ios/index-seed-swap.test.ts index f460ce7b8..770681028 100644 --- a/src/drivers/ios/index-seed-swap.test.ts +++ b/src/drivers/ios/index-seed-swap.test.ts @@ -1,7 +1,8 @@ import { afterEach, describe, expect, it } from "vitest"; +import { DriverCrashError } from "../../core/index.js"; import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/index.js"; import { plantBytes, readBytes } from "./testing.js"; -import { deviceRoot, driverData, plantManagedDevice } from "./test-slim.js"; +import { deviceRoot, driverData, plantManagedDevice, simctl } from "./test-slim.js"; import { createDriver, listFixture, @@ -46,4 +47,67 @@ describe("IosSimctlDriver copy swap", () => { ); expect(device.driverData).toMatchObject({ lineage: [lineageUdid] }); }); + + it("a copy whose swap fails is deleted and the provision fails", async () => { + class FailingWrites extends MemoryFilesystem { + override async writeAt(): Promise { + throw new Error("disk full"); + } + } + const file = `${deviceRoot}/${copyUdid}/data/Library/note.plist`; + const cases = [ + { label: "control: a swap that works", error: undefined }, + { label: "a recorded file gone", gone: true, error: /ENOENT/ }, + { + label: "a write that fails", + filesystem: new FailingWrites(), + error: /disk full/, + }, + { + label: "a result of another length", + lineage: ["AAAAAAAA-1111-2222-3333-BBBBBBBBBBB"], + crash: true, + error: new RegExp(`${file}.*length`), + }, + ]; + for (const attempt of cases) { + const database = await seededDatabase({ + files: [{ kind: "file", relativePath: "data/Library/note.plist" }], + lineage: attempt.lineage ?? [lineageUdid], + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + { match: simctl("delete", copyUdid) }, + ]); + const filesystem = attempt.filesystem ?? new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await plantBytes( + filesystem as MemoryFilesystem, + attempt.gone === true ? `${deviceRoot}/${copyUdid}/data/other` : file, + Buffer.from(`/Devices/${lineageUdid}/ AAAAAAAA-1111-2222-3333-BBBBBBBBBBB`), + ); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + const outcome = await driver.provision(spec).then( + () => undefined, + (error: unknown) => error as Error, + ); + + const deleted = runner.calls.some((call) => + call.args.join(" ").endsWith(`delete ${copyUdid}`), + ); + if (attempt.error === undefined) { + expect(outcome, attempt.label).toBeUndefined(); + expect(deleted, attempt.label).toBe(false); + } else { + expect(outcome?.message ?? "no error", attempt.label).toMatch(attempt.error); + expect(deleted, attempt.label).toBe(true); + if (attempt.crash === true) expect(outcome, attempt.label).toBeInstanceOf(DriverCrashError); + } + } + }); }); From 3e1c913973aeb010ba0f3996c0ced3715a27ae21 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:29:03 +0100 Subject: [PATCH 30/48] feat(ios): a copy whose swap fails is deleted and the provision fails (#1570) --- src/drivers/ios/udid-swap.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/drivers/ios/udid-swap.ts b/src/drivers/ios/udid-swap.ts index ebc44cba9..f9aadf950 100644 --- a/src/drivers/ios/udid-swap.ts +++ b/src/drivers/ios/udid-swap.ts @@ -21,15 +21,21 @@ export function namesLineage(chunk: Uint8Array, lineage: readonly string[]): boo return lineage.some((udid) => forms(udid).some((form) => bytes.includes(form))); } -/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own, byte for byte, so the output has the input's length. */ +/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own; a UDID has one length, so the output has the input's length, and a caller checks it. */ export function swapLineage(chunk: Uint8Array, swap: LineageSwap): Uint8Array { - const out = Buffer.from(chunk); + let out = Buffer.from(chunk); const own = Buffer.from(swap.own, "latin1"); for (const udid of swap.lineage) { const needle = Buffer.from(udid, "latin1"); - for (let at = out.indexOf(needle); at !== -1; at = out.indexOf(needle, at + needle.length)) { - own.copy(out, at); + if (needle.length === 0) continue; + const pieces: Buffer[] = []; + let from = 0; + for (let at = out.indexOf(needle); at !== -1; at = out.indexOf(needle, from)) { + pieces.push(out.subarray(from, at), own); + from = at + needle.length; } + pieces.push(out.subarray(from)); + out = Buffer.concat(pieces); } return out; } From 114bd4b6d675528babf8d6ce08e5db2e5811a3fb Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:29:47 +0100 Subject: [PATCH 31/48] test: a copy swaps every form of the lineage's UDIDs in its recorded files and links and logs the swap (#1570) --- src/drivers/ios/index-seed-swap.test.ts | 89 ++++++++++++++++++++++--- src/drivers/ios/test-index.ts | 5 +- 2 files changed, 85 insertions(+), 9 deletions(-) diff --git a/src/drivers/ios/index-seed-swap.test.ts b/src/drivers/ios/index-seed-swap.test.ts index 770681028..4d858a2cb 100644 --- a/src/drivers/ios/index-seed-swap.test.ts +++ b/src/drivers/ios/index-seed-swap.test.ts @@ -4,6 +4,7 @@ import { FakeClock, MemoryFilesystem, ScriptedProcessRunner } from "../../ports/ import { plantBytes, readBytes } from "./testing.js"; import { deviceRoot, driverData, plantManagedDevice, simctl } from "./test-slim.js"; import { + RecordingLogger, createDriver, listFixture, listInvocation, @@ -22,9 +23,70 @@ const lineageUdid = "AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"; const copyUdid = "CCCCCCCC-4444-5555-6666-DDDDDDDDDDDD"; describe("IosSimctlDriver copy swap", () => { - it("a copy keeps its source's lineage and swaps the lineage's UDIDs in every recorded file for its own before its first boot", async () => { + it("a copy keeps its source's lineage, swaps the lineage's UDIDs in every recorded file and link for its own before its first boot and logs the swap", async () => { + class TickingWrites extends MemoryFilesystem { + constructor(private readonly clock: FakeClock) { + super(); + } + + override async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { + this.clock.advance(5); + await super.writeAt(path, offset, bytes); + } + } + const utf16le = (text: string): Buffer => Buffer.from(text, "utf16le"); + const utf16be = (text: string): Buffer => utf16le(text).swap16(); + const ascii = (text: string): Buffer => Buffer.from(text, "latin1"); + const lower = lineageUdid.toLowerCase(); + const ownLower = copyUdid.toLowerCase(); + const window = 1_048_576; + const spanning = (udid: Buffer, at: number): Buffer => { + const bytes = Buffer.alloc(window + 200, 0x20); + udid.copy(bytes, at); + return bytes; + }; + const big = Buffer.alloc(64 * window + 100); + const bigTail = ascii(`/${lineageUdid}/`); + bigTail.copy(big, big.length - bigTail.length); + const bigSwapped = Buffer.from(big); + ascii(`/${copyUdid}/`).copy(bigSwapped, big.length - bigTail.length); + const files = [ + { + name: "ascii", + before: ascii(`path /Devices/${lineageUdid}/data end`), + after: ascii(`path /Devices/${copyUdid}/data end`), + }, + { name: "lower", before: ascii(`/logs/${lower}/x`), after: ascii(`/logs/${ownLower}/x`) }, + { name: "le", before: utf16le(`a ${lineageUdid} b`), after: utf16le(`a ${copyUdid} b`) }, + { name: "be", before: utf16be(`a ${lower} b`), after: utf16be(`a ${ownLower} b`) }, + { + name: "span-ascii", + before: spanning(ascii(lineageUdid), window - 18), + after: spanning(ascii(copyUdid), window - 18), + }, + { + name: "span-utf16", + before: spanning(utf16le(lineageUdid), window - 36), + after: spanning(utf16le(copyUdid), window - 36), + }, + { name: "big", before: big, after: bigSwapped }, + { name: "own", before: ascii(`own ${copyUdid}`), after: ascii(`own ${copyUdid}`) }, + { + name: "unreadable", + before: ascii(`x ${lineageUdid}`), + after: ascii(`x ${lineageUdid}`), + kind: "unreadable" as const, + }, + ]; + const linkTarget = (udid: string): string => `/Users/x/Library/Logs/CoreSimulator/${udid}`; const database = await seededDatabase({ - files: [{ kind: "file", relativePath: "data/Library/note.plist" }], + files: [ + ...files.map((file) => ({ + kind: file.kind ?? ("file" as const), + relativePath: `data/${file.name}`, + })), + { kind: "link", relativePath: "data/Logs" }, + ], lineage: [lineageUdid], }); const runner = new ScriptedProcessRunner([ @@ -34,18 +96,29 @@ describe("IosSimctlDriver copy swap", () => { result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, }, ]); - const filesystem = new MemoryFilesystem(); + const clock = new FakeClock(); + const filesystem = new TickingWrites(clock); + const logger = new RecordingLogger(); await plantManagedDevice(filesystem); - const file = `${deviceRoot}/${copyUdid}/data/Library/note.plist`; - await plantBytes(filesystem, file, Buffer.from(`path /Devices/${lineageUdid}/data end`)); - const driver = await createDriver(runner, new FakeClock(), filesystem, database); + for (const file of files) { + await plantBytes(filesystem, `${deviceRoot}/${copyUdid}/data/${file.name}`, file.before); + } + filesystem.defineSymlink(`${deviceRoot}/${copyUdid}/data/Logs`, linkTarget(lineageUdid)); + const driver = await createDriver(runner, clock, filesystem, database, logger); const device = await driver.provision(spec); - expect((await readBytes(filesystem, file)).toString("latin1")).toBe( - `path /Devices/${copyUdid}/data end`, + for (const file of files) { + const bytes = await readBytes(filesystem, `${deviceRoot}/${copyUdid}/data/${file.name}`); + expect(bytes.equals(file.after), file.name).toBe(true); + } + expect(await filesystem.readlink(`${deviceRoot}/${copyUdid}/data/Logs`)).toBe( + linkTarget(copyUdid), ); expect(device.driverData).toMatchObject({ lineage: [lineageUdid] }); + expect(logger.debugs).toEqual([ + `swapped 8 files and 1 links in 35 ms for ${copyUdid}; 1 unreadable`, + ]); }); it("a copy whose swap fails is deleted and the provision fails", async () => { diff --git a/src/drivers/ios/test-index.ts b/src/drivers/ios/test-index.ts index f4ca97fe3..bf4990a75 100644 --- a/src/drivers/ios/test-index.ts +++ b/src/drivers/ios/test-index.ts @@ -262,8 +262,11 @@ export async function launchServicesFiles( /** why: Records every `warn` message, so the destroy's log line is asserted on its text. */ export class RecordingLogger implements Logger { readonly warnings: string[] = []; + readonly debugs: string[] = []; - debug(): void {} + debug(message: string): void { + this.debugs.push(message); + } info(): void {} warn(message: string): void { this.warnings.push(message); From 9d4c807fcbe82dadb10fcda34f29e7083ed0e9ed Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:30:21 +0100 Subject: [PATCH 32/48] feat(ios): a copy swaps every form of the lineage's UDIDs in its recorded files and links and logs the swap (#1570) --- src/auth/admin-secret.test.ts | 6 +++++ src/drivers/ios/lineage-files.ts | 44 ++++++++++++++++++++++++-------- src/drivers/ios/seeds.ts | 22 ++++++++++------ src/drivers/ios/udid-swap.ts | 28 ++++++++++++-------- src/ports/filesystem.ts | 22 ++++++++++++++++ 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/src/auth/admin-secret.test.ts b/src/auth/admin-secret.test.ts index 69b21d617..84272bb3b 100644 --- a/src/auth/admin-secret.test.ts +++ b/src/auth/admin-secret.test.ts @@ -108,6 +108,9 @@ describe("AdminSecretManager", () => { readlink: () => { throw new Error("verify() must not touch the filesystem"); }, + replaceSymlink: () => { + throw new Error("verify() must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("verify() must not touch the filesystem"); }, @@ -201,6 +204,9 @@ describe("AdminSecretManager", () => { readlink: () => { throw new Error("must not touch the filesystem"); }, + replaceSymlink: () => { + throw new Error("must not touch the filesystem"); + }, writeFileExclusive: () => { throw new Error("must not touch the filesystem"); }, diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts index 9db518ea7..2ee6bf76a 100644 --- a/src/drivers/ios/lineage-files.ts +++ b/src/drivers/ios/lineage-files.ts @@ -14,24 +14,48 @@ const WINDOW_BYTES = 1_048_576; const WAL_SUFFIX = "-wal"; -/** why: ADR 0076 §4: before its first boot a copy's recorded files name its own UDID in place of the lineage's, each file keeping its length. */ +export interface SwapCounts { + readonly files: number; + readonly links: number; + readonly unreadable: number; +} + +/** why: ADR 0076 §4: before its first boot a copy's recorded files and links name its own UDID in place of the lineage's, each keeping its length; an unreadable file is left. */ export async function swapCopyFiles( filesystem: Filesystem, folder: string, files: readonly SourceFile[], swap: LineageSwap, -): Promise { +): Promise { + const counts = { files: 0, links: 0, unreadable: 0 }; for (const file of files) { - if (file.kind !== "file") continue; const path = `${folder}/${file.relativePath}`; - await eachWindow(filesystem, path, async (window, start) => { - const swapped = swapLineage(window, swap); - if (swapped.length !== window.length) { - throw new DriverCrashError(`swapping UDIDs in ${path} changed its length`); + if (file.kind === "unreadable") { + counts.unreadable += 1; + } else if (file.kind === "link") { + counts.links += 1; + const target = Buffer.from(await filesystem.readlink(path)); + const swapped = swapLineage(target, swap); + checkLength(path, target, swapped); + if (!target.equals(swapped)) { + await filesystem.replaceSymlink(path, Buffer.from(swapped).toString()); } - if (!Buffer.from(swapped).equals(window)) await filesystem.writeAt(path, start, swapped); - return false; - }); + } else { + counts.files += 1; + await eachWindow(filesystem, path, async (window, start) => { + const swapped = swapLineage(window, swap); + checkLength(path, window, swapped); + if (!window.equals(swapped)) await filesystem.writeAt(path, start, swapped); + return false; + }); + } + } + return counts; +} + +function checkLength(path: string, before: Uint8Array, after: Uint8Array): void { + if (before.length !== after.length) { + throw new DriverCrashError(`swapping UDIDs in ${path} changed its length`); } } diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index 7817af862..b09a4c033 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -232,14 +232,7 @@ export class Seeds { try { await this.#marks.writeMark(udid); await this.#dropLaunchServicesDatabase(this.#deviceRoot, udid); - if (seed.lineage !== undefined) { - await swapCopyFiles( - this.#filesystem, - join(this.#deviceRoot, udid), - this.#sources.filesOf(seed.sourceId), - { lineage: seed.lineage, own: udid }, - ); - } + if (seed.lineage !== undefined) await this.#swapLineage(seed.sourceId, seed.lineage, udid); } catch (error: unknown) { // why: a clone whose mark or database drop failed is not a device Simlock hands out, and // in Xcode's set an unmarked one would never be listed, reclaimed or deleted again. @@ -494,6 +487,19 @@ export class Seeds { }); } + async #swapLineage(sourceId: string, lineage: readonly string[], udid: string): Promise { + const started = this.#clock.now(); + const counts = await swapCopyFiles( + this.#filesystem, + join(this.#deviceRoot, udid), + this.#sources.filesOf(sourceId), + { lineage, own: udid }, + ); + this.#logger?.debug( + `swapped ${String(counts.files)} files and ${String(counts.links)} links in ${String(this.#clock.now() - started)} ms for ${udid}; ${String(counts.unreadable)} unreadable`, + ); + } + #recordFiles(udid: string, lineage: readonly string[]): ReturnType { return recordLineageFiles({ filesystem: this.#filesystem, diff --git a/src/drivers/ios/udid-swap.ts b/src/drivers/ios/udid-swap.ts index f9aadf950..f09635c20 100644 --- a/src/drivers/ios/udid-swap.ts +++ b/src/drivers/ios/udid-swap.ts @@ -21,21 +21,27 @@ export function namesLineage(chunk: Uint8Array, lineage: readonly string[]): boo return lineage.some((udid) => forms(udid).some((form) => bytes.includes(form))); } -/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own; a UDID has one length, so the output has the input's length, and a caller checks it. */ +/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own in the same case and encoding; a UDID has one length, so the output has the input's length, and a caller checks it. */ export function swapLineage(chunk: Uint8Array, swap: LineageSwap): Uint8Array { let out = Buffer.from(chunk); - const own = Buffer.from(swap.own, "latin1"); + const ownForms = forms(swap.own); for (const udid of swap.lineage) { - const needle = Buffer.from(udid, "latin1"); - if (needle.length === 0) continue; - const pieces: Buffer[] = []; - let from = 0; - for (let at = out.indexOf(needle); at !== -1; at = out.indexOf(needle, from)) { - pieces.push(out.subarray(from, at), own); - from = at + needle.length; + if (udid.toUpperCase() === swap.own.toUpperCase()) continue; + const replacements = forms(udid).map((form, index) => ({ + form, + own: ownForms[index] ?? form, + })); + for (const { form, own } of replacements) { + if (form.length === 0) continue; + const pieces: Buffer[] = []; + let from = 0; + for (let at = out.indexOf(form); at !== -1; at = out.indexOf(form, from)) { + pieces.push(out.subarray(from, at), own); + from = at + form.length; + } + pieces.push(out.subarray(from)); + out = Buffer.concat(pieces); } - pieces.push(out.subarray(from)); - out = Buffer.concat(pieces); } return out; } diff --git a/src/ports/filesystem.ts b/src/ports/filesystem.ts index ae51e78b6..b5046c63c 100644 --- a/src/ports/filesystem.ts +++ b/src/ports/filesystem.ts @@ -15,6 +15,7 @@ import { rm, stat, statfs, + symlink, writeFile, } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; @@ -124,6 +125,8 @@ export interface Filesystem { directorySize(path: string): Promise; /** why: what a symbolic link points at, without following it. */ readlink(path: string): Promise; + /** why: the link at `path` points at `target` from one step to the next, with no moment when it is missing. */ + replaceSymlink(path: string, target: string): Promise; /** why: `bytes` written into an existing file at `offset`, with no truncation, keeping its mode and modification time. */ writeAt(path: string, offset: number, bytes: Uint8Array): Promise; } @@ -415,6 +418,17 @@ export class NodeFilesystem implements Filesystem { return readlink(path); } + async replaceSymlink(path: string, target: string): Promise { + const staged = `${path}.simlock-swap-${String(process.pid)}`; + await symlink(target, staged); + try { + await rename(staged, path); + } catch (error: unknown) { + await rm(staged, { force: true }); + throw error; + } + } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { const handle = await open(path, "r+"); try { @@ -793,6 +807,14 @@ export class MemoryFilesystem implements Filesystem { return entry.target; } + async replaceSymlink(path: string, target: string): Promise { + this.#failIfDefined(path); + const at = this.#underResolvedParent(path); + const entry = this.#rawEntryAt(at); + if (entry.kind !== "symlink") throw errnoError("EINVAL", `Not a symbolic link: ${path}`); + this.#entries.set(at, { ...this.#newEntry({ kind: "symlink", target }), uid: entry.uid }); + } + async writeAt(path: string, offset: number, bytes: Uint8Array): Promise { this.#failIfDefined(path); const entry = this.#entryAt(path); From 16f41c78d5e7e036195da74febc4f8f61958259e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:34:25 +0100 Subject: [PATCH 33/48] test: a baseline the driver calls stale is checked at startup and after a save, and a source with no lineage is not copied (#1570) --- src/daemon/dispatcher-baseline-lease.test.ts | 105 ++++++++++++++++++- src/daemon/test-dispatcher.ts | 2 +- src/drivers/ios/index-seed-sources.test.ts | 72 +++++++++++-- 3 files changed, 169 insertions(+), 10 deletions(-) diff --git a/src/daemon/dispatcher-baseline-lease.test.ts b/src/daemon/dispatcher-baseline-lease.test.ts index ec43a2321..234c108e1 100644 --- a/src/daemon/dispatcher-baseline-lease.test.ts +++ b/src/daemon/dispatcher-baseline-lease.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { FakeDriver } from "../core/testing.js"; -import { FakeClock } from "../ports/index.js"; +import { FakeClock, JsonLinesLogger, MemoryLogSink } from "../ports/index.js"; import { DispatchError } from "./dispatcher.js"; import { classifyError } from "./error-code.js"; import { buildDispatcher, gibibyte, session } from "./test-dispatcher.js"; @@ -269,3 +269,104 @@ describe("Dispatcher: a baseline lease is released", () => { }, ); }); + +describe("Dispatcher: a baseline the driver calls stale", () => { + const reason = "saved before simulators stopped sharing folders; save it again"; + + async function built(logger?: JsonLinesLogger) { + const clock = new FakeClock(1_000); + const harness = await buildDispatcher({ + clock, + driverOptions: { knownModels: ["iPhone 17 Pro"] }, + ...(logger === undefined ? {} : { engine: { logger } }), + warmPool: { baselineLearnAfter: 1 }, + }); + const real = harness.driver.listSources.bind(harness.driver); + return { + ...harness, + clock, + markStale: () => + vi + .spyOn(harness.driver, "listSources") + .mockImplementation(async () => + (await real()).map((source) => ({ ...source, staleReason: reason })), + ), + provisions: () => + harness.driver.calls.filter((call) => call.operation === "provision").length, + request: () => + harness.dispatcher + .dispatch("lease.request", { baseline: "checkout", platform: "ios" }, session()) + .catch((error: unknown) => error), + restart: async () => harness.engine.core.converge(await harness.engine.core.readStartup()), + save: async () => { + const first = await harness.dispatcher.dispatch( + "lease.request", + { model: "iPhone 17 Pro", osVersion: "26.5", platform: "ios" }, + session(), + ); + await harness.dispatcher.dispatch( + "baseline.save", + { leaseId: first.lease.id, name: "checkout" }, + session({ role: "admin" }), + ); + }, + }; + } + + async function learned(harness: Awaited>): Promise { + await harness.restart(); + await harness.save(); + const granted = await harness.request(); + await harness.dispatcher.dispatch( + "lease.release", + { leaseId: (granted as { lease: { id: string } }).lease.id }, + session(), + ); + } + + it("answers BASELINE_STALE from the first request after the worker starts or after its save, and warms no copy of it", async () => { + const control = await built(); + await learned(control); + const warmed = control.provisions(); + await control.engine.core.passWarmPool(); + expect(control.provisions(), "a fresh baseline's learned kind is warmed").toBeGreaterThan( + warmed, + ); + + const restarted = await built(); + await learned(restarted); + restarted.markStale(); + await restarted.restart(); + const provisioned = restarted.provisions(); + await restarted.engine.core.passWarmPool(); + expect(restarted.provisions(), "no copy of a stale baseline is warmed").toBe(provisioned); + expect(await restarted.request()).toMatchObject({ + code: "BASELINE_STALE", + message: `baseline checkout is stale: ${reason}`, + }); + + const saved = await built(); + saved.markStale(); + await saved.save(); + expect(await saved.request()).toMatchObject({ + code: "BASELINE_STALE", + message: `baseline checkout is stale: ${reason}`, + }); + }); + + it("goes on, logging it, when the check at startup or after a save fails", async () => { + const sink = new MemoryLogSink(); + const harness = await built(new JsonLinesLogger({ clock: new FakeClock(0), sink })); + vi.spyOn(harness.engine.core.baselines, "checkStale").mockRejectedValue( + new Error("catalog unreadable"), + ); + + await expect(harness.restart()).resolves.toBeUndefined(); + await expect(harness.save()).resolves.toBeUndefined(); + + expect(sink.records.map((record) => record.message)).toEqual([ + "could not check baselines at startup: catalog unreadable", + "could not check baselines after a save: catalog unreadable", + ]); + }); +}); diff --git a/src/daemon/test-dispatcher.ts b/src/daemon/test-dispatcher.ts index 65e459870..add8d1a90 100644 --- a/src/daemon/test-dispatcher.ts +++ b/src/daemon/test-dispatcher.ts @@ -191,7 +191,7 @@ export async function buildDispatcher( readonly otherStallDrivers?: readonly FakeDriver[]; /** why: a second platform's driver for the leasing engine, for a test that leases on both. */ readonly extraDrivers?: readonly FakeDriver[]; - readonly engine?: Pick[0], "defaultModes">; + readonly engine?: Pick[0], "defaultModes" | "logger">; readonly physicalDevices?: readonly SeededPhysicalDevice[]; /** why: rows a test writes to `state.db` before the registry loads. */ readonly seedState?: (state: StateDatabase) => void; diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts index a1316bd29..531d8b5da 100644 --- a/src/drivers/ios/index-seed-sources.test.ts +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -2,7 +2,7 @@ import { afterEach, expect, it, describe } from "vitest"; import { DriverCrashError, BootTimeoutError } from "../../core/index.js"; import { ScriptedProcessRunner, FakeClock, MemoryFilesystem } from "../../ports/index.js"; import { promiseState } from "../../test-support/promise-state.js"; -import { plantSource, memorySourceDatabase } from "./testing.js"; +import { type PlantedSource, plantSource, memorySourceDatabase } from "./testing.js"; import { deviceRoot, driverData, @@ -146,9 +146,11 @@ describe("IosSimctlDriver", () => { { name: "a named source is copied instead of the seed", id: "planted-named", cloned: true }, { name: "an unknown source is refused", id: "nope", cloned: false }, { name: "a stale source is refused", id: "planted-old", cloned: false }, + { name: "a source with no lineage is refused", id: "planted-unlineaged", cloned: false }, ])("provision with a source: $name", async ({ id, cloned }) => { const namedUdid = "00000000-0000-0000-0000-0000000000bb"; const oldUdid = "00000000-0000-0000-0000-0000000000cc"; + const unlineagedUdid = "00000000-0000-0000-0000-0000000000dd"; const database = await seededDatabase(); const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; const planted = plantSource(database, { ...kind, kind: "baseline", udid: namedUdid }); @@ -159,6 +161,10 @@ describe("IosSimctlDriver", () => { database .prepare("UPDATE ios_source_data SET source_id = 'planted-old' WHERE udid = ?") .run(oldUdid); + plantSource(database, { ...kind, kind: "baseline", lineage: null, udid: unlineagedUdid }); + database + .prepare("UPDATE ios_source_data SET source_id = 'planted-unlineaged' WHERE udid = ?") + .run(unlineagedUdid); const filesystem = new MemoryFilesystem(); await plantManagedDevice(filesystem); await plantSeedClone(filesystem); @@ -391,9 +397,29 @@ describe("IosSimctlDriver", () => { runtimeBuild: undefined, remade: false, }, - ])( + { + name: "it has no lineage", + lineage: null, + seedBuild: "23F79", + runtimeBuild: "23F79", + remade: true, + }, + { + name: "its lineage is empty", + lineage: [], + seedBuild: "23F79", + runtimeBuild: "23F79", + remade: false, + }, + ] as readonly { + name: string; + lineage?: readonly string[] | null; + seedBuild: string | undefined; + runtimeBuild: string | undefined; + remade: boolean; + }[])( "makes the seed again before copying it only when it is stale: $name -> remade $remade", - async ({ seedBuild, runtimeBuild, remade }) => { + async ({ lineage, seedBuild, runtimeBuild, remade }) => { const staleUdid = "00000000-0000-0000-0000-0000000000bb"; const catalog = JSON.parse(listFixture) as { runtimes: { buildversion?: string; identifier: string }[]; @@ -429,6 +455,7 @@ describe("IosSimctlDriver", () => { filesystem, await seededDatabase({ udid: planted, + ...(lineage === undefined ? {} : { lineage }), ...(seedBuild === undefined ? {} : { runtimeBuild: seedBuild }), }), ); @@ -455,8 +482,38 @@ describe("IosSimctlDriver", () => { }, ); - it.each([ + it.each<{ + name: string; + edit: (catalog: Catalog) => void; + planted?: Partial; + reason: string | undefined; + }>([ { name: "healthy", edit: () => undefined, reason: undefined }, + { + name: "a seed has no lineage, which is remade and shows no reason", + edit: () => undefined, + planted: { lineage: null }, + reason: undefined, + }, + { + name: "a baseline was saved before lineages were kept", + edit: () => undefined, + planted: { kind: "baseline", lineage: null }, + reason: "saved before simulators stopped sharing folders; save it again", + }, + { + name: "a baseline was saved from a simulator made before lineages were kept", + edit: () => undefined, + planted: { kind: "baseline", lineage: null, savedWithoutLineage: true }, + reason: + "saved from a simulator made before simulators stopped sharing folders; save it again from a new lease", + }, + { + name: "a baseline has a lineage", + edit: () => undefined, + planted: { kind: "baseline", lineage: ["AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"] }, + reason: undefined, + }, { name: "its runtime is gone", edit: (catalog: Catalog) => { @@ -485,8 +542,8 @@ describe("IosSimctlDriver", () => { reason: `device type ${driverData.deviceTypeId} is not available`, }, ])( - "lists a ready seed as a source with its model, version and size, and its staleReason when $name, which clears once it is back", - async ({ edit, reason }) => { + "lists a ready source with its model, version and size, and its staleReason when $name, which clears once it is back", + async ({ edit, planted, reason }) => { const catalogOf = (change: (catalog: Catalog) => void) => { const catalog = JSON.parse(listFixture) as Catalog; for (const entry of catalog.runtimes) { @@ -508,12 +565,13 @@ describe("IosSimctlDriver", () => { runtimeId: driverData.runtimeId, sizeBytes: 1_073_741_824, udid: seedUdid, + ...planted, }); const driver = await createDriver(runner, new FakeClock(), undefined, database); expect(await driver.listSources()).toMatchObject([ { - kind: "seed", + kind: planted?.kind ?? "seed", model: "iPhone 16", osVersion: "26.5", sizeBytes: 1_073_741_824, From 93395fc2222ec5ba6a3b5209ac4b212f35f582d1 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:35:43 +0100 Subject: [PATCH 34/48] feat(baselines): a baseline with no lineage is stale, checked at startup and after a save, and a seed with none is remade (#1570) --- src/core/baselines.ts | 10 +++++++++- src/core/create-core.ts | 8 ++++++++ src/drivers/ios/index-save-source-lifecycle.test.ts | 2 +- src/drivers/ios/index-seed-sources.test.ts | 10 ++++++++-- src/drivers/ios/seeds.ts | 6 ++++++ 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/core/baselines.ts b/src/core/baselines.ts index 69aef6550..d5b45636c 100644 --- a/src/core/baselines.ts +++ b/src/core/baselines.ts @@ -1,6 +1,6 @@ import type { EventBus } from "../bus/index.js"; import type { BaselineEntry } from "../contract/index.js"; -import type { Clock } from "../ports/index.js"; +import type { Clock, Logger } from "../ports/index.js"; import type { StateDatabase } from "../state/index.js"; import { assertVirtual, type Platform, type VirtualDevice } from "./domain.js"; import type { Driver, DriverSource } from "./driver.js"; @@ -105,6 +105,7 @@ export interface BaselineServiceOptions { readonly decisions: SerializedDecision; readonly eventBus: Pick; readonly lifecycle: Pick; + readonly logger?: Logger | undefined; readonly reclaim: Pick; readonly registry: { readonly snapshot: Pick }; readonly state: StateDatabase; @@ -170,6 +171,13 @@ export class BaselineService { }); await this.#destroyIdleOlderDevices(baseline.name, baseline.version); await this.#dropReplaced(request.requesterId); + try { + await this.checkStale(); + } catch (error: unknown) { + this.options.logger?.warn( + `could not check baselines after a save: ${error instanceof Error ? error.message : String(error)}`, + ); + } return { baseline, leaseId: request.leaseId }; } diff --git a/src/core/create-core.ts b/src/core/create-core.ts index 78b3ed19e..d6a224895 100644 --- a/src/core/create-core.ts +++ b/src/core/create-core.ts @@ -259,6 +259,7 @@ export function createCore(options: CoreOptions): Core { eventBus: options.eventBus, leaseSave: () => port("leaseSave"), lifecycle: deviceLifecycle, + logger: options.logger, reclaim, registry, state: options.state, @@ -444,6 +445,13 @@ export function createCore(options: CoreOptions): Core { await startup.converge(read); await dropReplacedBaselines(); await listStartupSources(options.drivers, options.logger); + try { + await baselines.checkStale(); + } catch (error: unknown) { + options.logger?.warn( + `could not check baselines at startup: ${error instanceof Error ? error.message : String(error)}`, + ); + } warmPool.start(); capacityObserver.start(); }, diff --git a/src/drivers/ios/index-save-source-lifecycle.test.ts b/src/drivers/ios/index-save-source-lifecycle.test.ts index bc13dc694..8e1d87ca1 100644 --- a/src/drivers/ios/index-save-source-lifecycle.test.ts +++ b/src/drivers/ios/index-save-source-lifecycle.test.ts @@ -66,7 +66,7 @@ describe("IosSimctlDriver", () => { const device = { address: driverData.udid, deviceId: driverData.udid, - driverData, + driverData: { ...driverData, lineage: ["AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"] }, physical: false, } as const; const cloneInvocation = simctl("clone", driverData.udid, baselineName, sourcesSet); diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts index 531d8b5da..36652ee30 100644 --- a/src/drivers/ios/index-seed-sources.test.ts +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -486,6 +486,7 @@ describe("IosSimctlDriver", () => { name: string; edit: (catalog: Catalog) => void; planted?: Partial; + persists?: true; reason: string | undefined; }>([ { name: "healthy", edit: () => undefined, reason: undefined }, @@ -499,12 +500,14 @@ describe("IosSimctlDriver", () => { name: "a baseline was saved before lineages were kept", edit: () => undefined, planted: { kind: "baseline", lineage: null }, + persists: true, reason: "saved before simulators stopped sharing folders; save it again", }, { name: "a baseline was saved from a simulator made before lineages were kept", edit: () => undefined, planted: { kind: "baseline", lineage: null, savedWithoutLineage: true }, + persists: true, reason: "saved from a simulator made before simulators stopped sharing folders; save it again from a new lease", }, @@ -543,7 +546,7 @@ describe("IosSimctlDriver", () => { }, ])( "lists a ready source with its model, version and size, and its staleReason when $name, which clears once it is back", - async ({ edit, planted, reason }) => { + async ({ edit, persists, planted, reason }) => { const catalogOf = (change: (catalog: Catalog) => void) => { const catalog = JSON.parse(listFixture) as Catalog; for (const entry of catalog.runtimes) { @@ -578,7 +581,10 @@ describe("IosSimctlDriver", () => { ...(reason === undefined ? {} : { staleReason: reason }), }, ]); - expect((await driver.listSources())[0]).not.toHaveProperty("staleReason"); + // why: a lineage the source never had does not come back with the catalog, so its reason stays. + const again = (await driver.listSources())[0]; + if (persists === true) expect(again).toHaveProperty("staleReason", reason); + else expect(again).not.toHaveProperty("staleReason"); expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list"]); }, ); diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index b09a4c033..a73b38b49 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -553,12 +553,18 @@ export class Seeds { /** why: a seed built on a runtime whose build changed no longer matches it; with no build on either side only the runtime's presence is checked, and a provision always has the runtime. */ function seedIsStale(seed: ReadySeed, resolved: ResolvedIosSpec): boolean { + if (seed.lineage === undefined) return true; const current = resolved.runtime.build; return seed.runtimeBuild !== undefined && current !== undefined && seed.runtimeBuild !== current; } /** why: the phrase reads after "is stale:" in `doctor`, and is what a person acts on, so it names the runtime or device type by id. */ function sourceStaleReason(row: SourceRow, catalog: SimctlCatalog): string | undefined { + if (row.kind === "baseline" && row.lineage === undefined) { + return row.savedWithoutLineage + ? "saved from a simulator made before simulators stopped sharing folders; save it again from a new lease" + : "saved before simulators stopped sharing folders; save it again"; + } const runtime = catalog.runtimes.find((entry) => entry.identifier === row.runtimeId); if (runtime === undefined) return `runtime ${row.runtimeId} is not installed`; if ( From 8b221bf7ac32800e9ef4c3ab591c33ed18541e16 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:37:50 +0100 Subject: [PATCH 35/48] test: the lane keeps an iOS seed's lineage and file rows between flows (#1570) --- e2e/helpers/env.test.ts | 66 ++++++++++++++++++++++++++++++++++++++++ e2e/helpers/lane-home.ts | 36 +++++++++++++++++++--- 2 files changed, 98 insertions(+), 4 deletions(-) diff --git a/e2e/helpers/env.test.ts b/e2e/helpers/env.test.ts index 14423231b..bde30bb51 100644 --- a/e2e/helpers/env.test.ts +++ b/e2e/helpers/env.test.ts @@ -212,3 +212,69 @@ describe("withSharedDaemon with keepSeeds", () => { } }); }); + +describe("withSharedDaemon with keepSeeds and an iOS seed", () => { + it("gives the next env the iOS seed's lineage and its file rows, even when its simulator set is outside the home", async () => { + const laneHome = join(tmpdir(), `simlock-lane-${randomBytes(3).toString("hex")}`); + const sources = `${laneHome}-sources`; + process.env.SIMLOCK_E2E_LANE_HOME = laneHome; + const udid = "11111111-2222-3333-4444-555555555555"; + const lineage = JSON.stringify(["AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"]); + const files = [ + { kind: "file", relative_path: "data/Library/a.plist" }, + { kind: "link", relative_path: "data/Library/b" }, + { kind: "unreadable", relative_path: "data/Library/c" }, + ]; + try { + await mkdir(join(sources, udid), { recursive: true }); + const first = await laneEnv(); + try { + const database = new DatabaseSync(join(first.home, "state.db")); + try { + database.exec("PRAGMA busy_timeout=5000"); + database + .prepare( + `INSERT INTO ios_source_data (source_id, kind, state, spec_key, udid, set_path, + device_type_id, runtime_id, model, os_version, maker, lineage, created_at) + VALUES ('seed_ios', 'seed', 'ready', ?, ?, ?, 'type', 'runtime', 'iPhone 16', '26.5', 'maker', ?, 1)`, + ) + .run("iPhone 16\u0000runtime", udid, sources, lineage); + for (const file of files) { + database + .prepare( + "INSERT INTO ios_source_files (source_id, relative_path, kind) VALUES ('seed_ios', ?, ?)", + ) + .run(file.relative_path, file.kind); + } + } finally { + database.close(); + } + } finally { + await first.teardown(); + } + + const second = await laneEnv(); + try { + const database = new DatabaseSync(join(second.home, "state.db")); + try { + expect(database.prepare("SELECT source_id, lineage FROM ios_source_data").all()).toEqual([ + { lineage, source_id: "seed_ios" }, + ]); + expect( + database + .prepare("SELECT kind, relative_path FROM ios_source_files ORDER BY relative_path") + .all(), + ).toEqual(files); + } finally { + database.close(); + } + } finally { + await second.teardown(); + } + } finally { + delete process.env.SIMLOCK_E2E_LANE_HOME; + await rm(laneHome, { force: true, recursive: true }); + await rm(sources, { force: true, recursive: true }); + } + }); +}); diff --git a/e2e/helpers/lane-home.ts b/e2e/helpers/lane-home.ts index c0001ac7b..579d61032 100644 --- a/e2e/helpers/lane-home.ts +++ b/e2e/helpers/lane-home.ts @@ -13,6 +13,8 @@ type Row = Record; interface KeptSeed { readonly table: string; readonly row: Row; + /** why: ADR 0076: an iOS seed's file record is its lineage's other half, so it is kept with the row. */ + readonly files?: readonly Row[]; } interface SeedTable { @@ -77,13 +79,21 @@ export async function restoreKeptSeeds(home: string): Promise { const seeds = JSON.parse(await readFile(join(home, KEPT_SEEDS_FILE), "utf8")) as KeptSeed[]; const database = new DatabaseSync(join(home, "state.db")); try { - for (const { table, row } of seeds) { + for (const { table, row, files } of seeds) { const columns = Object.keys(row); database .prepare( `INSERT INTO ${table} (${columns.join(", ")}) VALUES (${columns.map(() => "?").join(", ")})`, ) .run(...columns.map((column) => row[column] ?? null)); + for (const file of files ?? []) { + const fileColumns = Object.keys(file); + database + .prepare( + `INSERT INTO ios_source_files (${fileColumns.join(", ")}) VALUES (${fileColumns.map(() => "?").join(", ")})`, + ) + .run(...fileColumns.map((column) => file[column] ?? null)); + } } } finally { database.close(); @@ -138,7 +148,13 @@ async function readySeeds(home: string): Promise { const database = new DatabaseSync(path, { readOnly: true }); try { return SEED_TABLES.flatMap(({ table }) => - readyRows(database, table).map((row) => ({ row, table })), + readyRows(database, table).map((row) => ({ + row, + table, + ...(table === "ios_source_data" + ? { files: textRows(database, "ios_source_files", "source_id", String(row["source_id"])) } + : {}), + })), ).filter((seed) => existsSync(folderOf(seed))); } finally { database.close(); @@ -151,6 +167,17 @@ async function readySeeds(home: string): Promise { * is read as bytes and decoded here. */ function readyRows(database: DatabaseSync, table: string): Row[] { + return textRows(database, table, undefined, undefined, "kind = 'seed' AND state = 'ready'"); +} + +/** why: the rows of a table with every text column read as bytes (see `readyRows`); a table the driver never made has none. */ +function textRows( + database: DatabaseSync, + table: string, + column?: string, + value?: string, + where = "1 = 1", +): Row[] { const columns = database.prepare(`PRAGMA table_info(${table})`).all() as { name: string; type: string; @@ -159,9 +186,10 @@ function readyRows(database: DatabaseSync, table: string): Row[] { const selected = columns.map(({ name, type }) => type === "TEXT" ? `CAST(${name} AS BLOB) AS ${name}` : name, ); + const filter = column === undefined ? where : `${column} = ?`; return database - .prepare(`SELECT ${selected.join(", ")} FROM ${table} WHERE kind = 'seed' AND state = 'ready'`) - .all() + .prepare(`SELECT ${selected.join(", ")} FROM ${table} WHERE ${filter}`) + .all(...(value === undefined ? [] : [value])) .map((row) => Object.fromEntries( Object.entries(row).map(([name, value]) => [ From 2150745fb7d5f60fe8b53a8e8c46042a9b20b5c0 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:39:25 +0100 Subject: [PATCH 36/48] feat(ios): the daemon gives the iOS driver the Node SQLite opener, and the slow lane proves a source stays unchanged and a copy keeps its own logs (#1570) --- e2e/slow-ios-source-isolation.test.ts | 193 ++++++++++++++++++++++++++ src/daemon/driver-discovery.ts | 2 + 2 files changed, 195 insertions(+) create mode 100644 e2e/slow-ios-source-isolation.test.ts diff --git a/e2e/slow-ios-source-isolation.test.ts b/e2e/slow-ios-source-isolation.test.ts new file mode 100644 index 000000000..f2b5aa101 --- /dev/null +++ b/e2e/slow-ios-source-isolation.test.ts @@ -0,0 +1,193 @@ +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readdir, readFile, readlink, stat, writeFile } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; + +import { withDaemon } from "./helpers/index.js"; +import type { TestEnv } from "./helpers/env.js"; +import { sweepStaleDeviceSets } from "./helpers/ios-device-set.js"; +import { DEVICE_READY_MS } from "./helpers/wait.js"; + +interface Grant { + readonly deviceSet: string; + readonly leaseId: string; + readonly udid: string; +} + +const LOG_ROOT = join(homedir(), "Library", "Logs", "CoreSimulator"); +const BASELINE = "isolation"; + +async function lease(env: TestEnv, extra: readonly string[] = []): Promise { + const granted = await env.cli( + ["lease", "--platform", "ios", "--agent-id", "isolation", "--detach", ...extra], + { timeout: DEVICE_READY_MS }, + ); + expect(granted.code, `lease failed: ${granted.stderr}`).toBe(0); + const grant = granted.json as { + device: { driverDeviceId: string }; + environment: Record; + lease: { id: string }; + }; + return { + deviceSet: grant.environment.SIMLOCK_IOS_DEVICE_SET as string, + leaseId: grant.lease.id, + udid: grant.device.driverDeviceId, + }; +} + +async function release(env: TestEnv, grant: Grant): Promise { + const released = await env.cli(["release", grant.leaseId]); + expect(released.code, `release failed: ${released.stderr}`).toBe(0); +} + +/** why: a file's size, modification time and hash are what "unchanged after the row turned ready" compares (ADR 0076). */ +async function fingerprint(folder: string): Promise> { + const result: Record = {}; + for (const entry of await readdir(folder, { recursive: true, withFileTypes: true })) { + const path = join(entry.parentPath, entry.name); + if (entry.isSymbolicLink()) result[path] = `link ${await readlink(path)}`; + else if (entry.isFile()) { + const { mtimeMs, size } = await stat(path); + const hash = createHash("sha256") + .update(await readFile(path)) + .digest("hex"); + result[path] = `${size} ${mtimeMs} ${hash}`; + } + } + return result; +} + +function sourceFolder(home: string, kind: "seed" | "baseline"): string { + const database = new DatabaseSync(join(home, "state.db"), { readOnly: true }); + try { + database.exec("PRAGMA busy_timeout=5000"); + const row = database + .prepare( + "SELECT set_path, udid FROM ios_source_data WHERE kind = ? AND state = 'ready' ORDER BY created_at DESC", + ) + .get(kind) as { set_path: string; udid: string } | undefined; + if (row === undefined) throw new Error(`no ready ${kind} source in state.db`); + return join(row.set_path, row.udid); + } finally { + database.close(); + } +} + +function seedThrowaway(home: string): string { + const database = new DatabaseSync(join(home, "state.db"), { readOnly: true }); + try { + const row = database + .prepare("SELECT lineage FROM ios_source_data WHERE kind = 'seed' AND state = 'ready'") + .get() as { lineage: string } | undefined; + const [throwaway] = JSON.parse(row?.lineage ?? "[]") as string[]; + if (throwaway === undefined) throw new Error("the ready seed has no lineage"); + return throwaway; + } finally { + database.close(); + } +} + +/** why: ADR 0076: a copy's `Logs` link names the copy's own folder, never the throwaway's or the saved-from lease's. */ +async function expectOwnLogs(grant: Grant, strangers: readonly string[]): Promise { + const target = await readlink(join(grant.deviceSet, grant.udid, "data", "Library", "Logs")); + expect(target.toUpperCase()).toContain(grant.udid.toUpperCase()); + for (const stranger of strangers) expect(target.toUpperCase()).not.toContain(stranger); +} + +async function expectNoFolderNamedAfter( + names: readonly string[], + deviceSet: string, +): Promise { + for (const root of [deviceSet, LOG_ROOT]) { + const entries = existsSync(root) ? await readdir(root) : []; + for (const name of names) { + expect( + entries.filter((entry) => entry.toUpperCase() === name), + `${root} holds a folder named after ${name}`, + ).toEqual([]); + } + } +} + +async function swapMilliseconds(env: TestEnv, udid: string): Promise { + const log = await readFile(env.logPath, "utf8"); + for (const line of log.split("\n")) { + const match = new RegExp( + `swapped \\d+ files and \\d+ links in (\\d+) ms for ${udid}`, + "i", + ).exec(line); + if (match !== null) return Number(match[1]); + } + throw new Error(`the daemon log has no swap line for ${udid}`); +} + +// why: this lane needs the real simctl toolchain on Xcode 27, hence darwin-only; it never installs a runtime. +describe.skipIf(process.platform !== "darwin")( + "iOS source isolation (real simctl)", + { tags: ["slow", "ios"] }, + () => { + it( + "leaves the seed unchanged after three leases and a saved baseline's source unchanged after two, gives every copy a Logs link to its own folder, brings back no folder named after the throwaway or the saved-from lease, and swaps a baseline's copy in under 1 s", + { timeout: 2_400_000 }, + async () => { + await sweepStaleDeviceSets(); + const env = await withDaemon({ + configOverrides: { log: { level: "debug" } }, + driver: "real", + }); + + // why: case 1, the seed, after three leases of one kind. + const first = await lease(env); + const seedFolder = sourceFolder(env.home, "seed"); + const throwaway = seedThrowaway(env.home).toUpperCase(); + const seedBefore = await fingerprint(seedFolder); + await expectOwnLogs(first, [throwaway]); + await release(env, first); + for (let more = 0; more < 2; more += 1) { + const next = await lease(env); + await expectOwnLogs(next, [throwaway]); + await release(env, next); + } + expect(await fingerprint(seedFolder), "the seed changed after its row was ready").toEqual( + seedBefore, + ); + + // why: case 2, a baseline saved from a used lease, after two leases of it. + const saved = await lease(env); + const marker = join(saved.deviceSet, saved.udid, "data", "Documents", "isolation.txt"); + await mkdir(join(saved.deviceSet, saved.udid, "data", "Documents"), { recursive: true }); + await writeFile(marker, "used"); + const baseline = await env.cli(["baseline", "save", BASELINE, "--lease", saved.leaseId], { + timeout: 600_000, + }); + expect(baseline.code, `baseline save failed: ${baseline.stderr}`).toBe(0); + const baselineFolder = sourceFolder(env.home, "baseline"); + const baselineBefore = await fingerprint(baselineFolder); + const strangers = [throwaway, saved.udid.toUpperCase()]; + const copies: Grant[] = []; + for (let n = 0; n < 2; n += 1) { + const copy = await lease(env, ["--baseline", BASELINE]); + copies.push(copy); + // why: case 3, each copy's Logs link names its own folder. + await expectOwnLogs(copy, strangers); + await release(env, copy); + } + expect( + await fingerprint(baselineFolder), + "the baseline's source changed after its row was ready", + ).toEqual(baselineBefore); + + // why: case 4, no folder named after the throwaway or the saved-from lease comes back. + await expectNoFolderNamedAfter(strangers, first.deviceSet); + + // why: case 5, a baseline's copy swaps in under 1 s, so a fall back to a full read is caught. + for (const copy of copies) { + expect(await swapMilliseconds(env, copy.udid)).toBeLessThan(1000); + } + }, + ); + }, +); diff --git a/src/daemon/driver-discovery.ts b/src/daemon/driver-discovery.ts index 3687c43dd..97d570bcd 100644 --- a/src/daemon/driver-discovery.ts +++ b/src/daemon/driver-discovery.ts @@ -36,6 +36,7 @@ import { type IdGenerator, type Logger, NodeBytePipeLauncher, + NodeSqliteOpener, NodeTcpListenerFactory, NodeTcpStreamConnector, type ProcessRunner, @@ -203,6 +204,7 @@ async function discoverIosDriver( onSilenceFailed: logSilenceFailure(logger), onWindowFailed: logWindowFailure(logger), onSlimmed: emitSlimDiagnostic(options.eventBus), + sqliteOpener: new NodeSqliteOpener(), screenCapture: { baseEnvironment: process.env, command: PACKAGED_SCREEN_CAPTURE, From 990f10b688848c757c3902b49ebe48ae671242f3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:39:51 +0100 Subject: [PATCH 37/48] docs: the lineage, the file record, the swap and when a baseline turns stale for it (#1570) --- docs/internal/flows/startup.md | 3 ++- src/core/AGENTS.md | 2 +- src/drivers/ios/AGENTS.md | 3 ++- website/src/content/docs/docs/reference/cli.mdx | 2 +- website/src/content/docs/docs/reference/errors.mdx | 2 +- website/src/content/docs/docs/using-devices/errors.mdx | 2 +- website/src/content/docs/docs/worker/baselines.mdx | 4 ++-- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/internal/flows/startup.md b/docs/internal/flows/startup.md index 6c09e3d67..e71155fbf 100644 --- a/docs/internal/flows/startup.md +++ b/docs/internal/flows/startup.md @@ -68,7 +68,8 @@ Before step 16, a driver with `deviceTcp` sweeps the reverse rules into 47000-47 `device.shutdown` (initiator `startup-interrupted-reclaim`). 3. Spent deletes: `shutdown` devices that may not be granted again are deleted inline. A failed delete is logged; the device stays `shutdown`. - 4. Warm pool and capacity observer start. The pool closes the lease rows of its learned-kind + 4. Baselines `checkStale` (a failure is logged), so a baseline the driver calls stale is known before the first lease. + 5. Warm pool and capacity observer start. The pool closes the lease rows of its learned-kind record whose lease the registry no longer holds, and its first pass boots the recorded kinds. 22. `queue.changed` with the first depth. Converge done → health `running`. diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 4f77c216a..3bf002db4 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -58,7 +58,7 @@ Baselines (ADR 0061 §4): - One save or remove at a time. A save: leasing's `releaseToSave` ends the lease, then `ReclaimCoordinator.retireSaved` calls `driver.saveSource` under the `reclaim` claim. A driver failure quarantines the device and answers `BASELINE_SAVE_FAILED`. - A replace keeps the leased devices of older versions. An older version is dropped only when no device record and no provision (`hold`) has it; that is checked after a save, on every `device.deleted` and at startup. - `remove` refuses a baseline a lease uses, and destroys each source before its row. `removeAll` (nuke) deletes rows first. -- `checkStale` runs before `list` and `doctor` and after `component.removed`. +- `checkStale` runs before `list` and `doctor`, after `component.removed`, after a save, and at startup before the warm pool starts. A failing check is logged and the save or the start goes on. ## Known gaps diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 0365b214b..3b0dd0a54 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -29,7 +29,8 @@ Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources set: `sources` in the owned set, an owned root of its own under `default`. It is in no `listManaged` or `simlock simctl list`. - A seed is made from a throwaway (ADR 0076): `simlock-seed-` is booted once, shut down, cloned unbooted into the sources set, then deleted with its log folder (`logsRoot`); `saveSource` deletes the saved-from lease's log folder too. A log folder that cannot be deleted is logged and the make or save goes on; a throwaway that cannot be deleted fails the make. Only its `making` row (`throwaway_udid`) deletes a throwaway, at the first `listSources` of a later process or the next make of its kind. -- Every clone (a seed's copy, `saveSource`'s own clone, a `provision` with `source`) has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted before its first boot: it records each app under the original device's folder. A clone whose drop fails is deleted. +- A source records its lineage (the UDIDs of the simulators it was copied from: a seed's is its throwaway, a baseline's is the saved-from device then that device's own) and, in `ios_source_files`, every file and link of its folder, after flushing its SQLite WALs and before the row turns `ready`. A device keeps the lineage of the source it was copied from. +- Every clone (a seed's copy, `saveSource`'s own clone, a `provision` with `source`) has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted and, in the recorded files, every form of its lineage's UDIDs (ASCII and UTF-16, either case) swapped for its own, and its links re-pointed, before its first boot. A clone whose drop or swap fails is deleted. A source with no lineage is never copied: a seed is made again, a baseline is stale (`saved before simulators stopped sharing folders; save it again`). - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. diff --git a/website/src/content/docs/docs/reference/cli.mdx b/website/src/content/docs/docs/reference/cli.mdx index 042b20b44..7fdf28eaf 100644 --- a/website/src/content/docs/docs/reference/cli.mdx +++ b/website/src/content/docs/docs/reference/cli.mdx @@ -453,7 +453,7 @@ simlock doctor [--fix] [--purge-orphans] [--yes] | `--purge-orphans` | Destroy every orphaned device: a device in Simlock's device folders with no record. Also destroy every orphan source: a seed simulator with no record, or a saved baseline's copy that no baseline names. Needs the admin role. Asks for confirmation. | Off | | `--yes` | Skip the confirmation for `--purge-orphans`. Required when there is no terminal to ask in. | Off | -The report also lists, under `sources`, the copies on disk that new devices are made from. Each has `platform`, `kind` (`seed` or `baseline`), `model`, `osVersion`, `sizeBytes` when known, and `staleReason` when it no longer matches the installed runtime. A baseline source also has `baselineName` and `baselineVersion`. A seed is a simulator Simlock copies to make each iOS simulator a lease gets: it takes disk space but is never leased. A seed with no record, or a baseline copy no baseline names, shows `orphan: true`. That is not a finding; only `--purge-orphans` deletes it. +The report also lists, under `sources`, the copies on disk that new devices are made from. Each has `platform`, `kind` (`seed` or `baseline`), `model`, `osVersion`, `sizeBytes` when known, and `staleReason` when it no longer matches the installed runtime, or when an iOS baseline was saved before simulators stopped sharing folders. A baseline source also has `baselineName` and `baselineVersion`. A seed is a simulator Simlock copies to make each iOS simulator a lease gets: it takes disk space but is never leased. A seed with no record, or a baseline copy no baseline names, shows `orphan: true`. That is not a finding; only `--purge-orphans` deletes it. On a worker whose login service names a Node or `main.js` that is gone, the report holds a `service-broken` finding and stderr gets `Service broken: () names missing . Run `simlock daemon install` again.` (`, ` for two). diff --git a/website/src/content/docs/docs/reference/errors.mdx b/website/src/content/docs/docs/reference/errors.mdx index f57ae212e..854959ce9 100644 --- a/website/src/content/docs/docs/reference/errors.mdx +++ b/website/src/content/docs/docs/reference/errors.mdx @@ -69,7 +69,7 @@ In a script, check `code`, not the message: the wording of a message can change, | `BASELINE_SAVE_FAILED` | 1 | 500 | `baseline save` ended the lease, but Simlock could not keep a copy of the device. The message gives the reason. No baseline was saved, and the device is set aside for cleanup. | Lease a fresh device, prepare it again and save again. | | `UNKNOWN_BASELINE` | 12 | 404 | `baseline remove`, or a lease request with `--baseline`, naming a baseline this worker does not have. Through a gateway: no worker holds the baseline, or the worker `--worker` names does not. | List the baselines with `simlock baseline list`. | | `BASELINE_MISMATCH` | 2 | 422 | A lease request names a baseline and also a platform, model, OS, mode or image tag that differs from it. The message names the first field that differs. | Drop the field, or change it to the baseline's own value. | -| `BASELINE_STALE` | 12 | 422 | A lease request names a baseline whose runtime or image has been removed since it was saved. The message gives the reason. | Install the runtime or image again, or save the baseline again from a new device. | +| `BASELINE_STALE` | 12 | 422 | A lease request names a baseline whose runtime or image has been removed since it was saved, or an iOS baseline saved before simulators stopped sharing folders. The message gives the reason. | Install the runtime or image again, or save the baseline again from a new device. | | `BASELINE_IN_USE` | 12 | 409 | `baseline remove` of a baseline a lease still uses. The message names the lease. | End that lease, or remove the baseline after it. | | `COMPONENT_BUSY` | 11 | 409 | `component remove` while an install or removal runs or waits on that platform. | Try again once it ends. | diff --git a/website/src/content/docs/docs/using-devices/errors.mdx b/website/src/content/docs/docs/using-devices/errors.mdx index ef2ce6f30..ddf20c698 100644 --- a/website/src/content/docs/docs/using-devices/errors.mdx +++ b/website/src/content/docs/docs/using-devices/errors.mdx @@ -45,7 +45,7 @@ In a script, check the exit code or `code`, never the message: the message is fo | `RUNTIME_MISSING` | 12 | The OS version, or the Android image, isn't installed. An OS range or `--image-tag` never downloads. | Pass `--allow-download` for an exact version, or ask an admin to install it. See [Choosing a device](/simlock/docs/using-devices/choosing-a-device#when-the-runtime-is-not-installed). | | `UNKNOWN_BASELINE` | 12 | `--baseline` names a baseline this machine doesn't have. | Check the name with `simlock baseline list`. | | `BASELINE_MISMATCH` | 2 | `--baseline` came with a platform, model, OS, mode or image tag that differs from the baseline. The message names the first one that differs. | Drop that option: the baseline already decides it. | -| `BASELINE_STALE` | 12 | The OS or image the baseline was saved on has been removed. | Ask an admin to install it again, or save the baseline again. | +| `BASELINE_STALE` | 12 | The OS or image the baseline was saved on has been removed, or an iOS baseline was saved before simulators stopped sharing folders. | Ask an admin to install it again, or save the baseline again. | | `LICENSE_NOT_ACCEPTED` | 12 | An Android download needs the SDK licences accepted, and this machine hasn't accepted them. | An admin sets `downloads.acceptAndroidLicenses` to `true`. | | `INSUFFICIENT_DISK_SPACE` | 12 | The download doesn't fit on the disk. The message says how much is needed and free. | Free disk space on that machine, or ask for an installed version. | | `DOWNLOAD_TIMEOUT` | 10 | The download, including time waiting behind another download, took longer than the machine allows. | Retry; the download may already be further along. Or ask an admin to install the runtime. | diff --git a/website/src/content/docs/docs/worker/baselines.mdx b/website/src/content/docs/docs/worker/baselines.mdx index c477df398..ea453e76d 100644 --- a/website/src/content/docs/docs/worker/baselines.mdx +++ b/website/src/content/docs/docs/worker/baselines.mdx @@ -72,7 +72,7 @@ login 1 ios iPhone 17 27.0 slim - 3.1 GB 2026-11 stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed ``` -A baseline is stale when the iOS runtime, Android system image or emulator it was saved with has changed or been removed. The reason is on the line under it. A lease that names a stale baseline fails with `BASELINE_STALE`. Install that runtime or image again and the baseline works again; or save a new version from a device on the runtime you have now. +A baseline is stale when the iOS runtime, Android system image or emulator it was saved with has changed or been removed. An iOS baseline saved before Simlock kept each simulator in its own folders is stale too; save it again from a new lease. The reason is on the line under it. A lease that names a stale baseline fails with `BASELINE_STALE`. Install that runtime or image again and the baseline works again; or save a new version from a device on the runtime you have now. ## Remove a baseline @@ -95,7 +95,7 @@ This deletes every version of `login` and the idle copies made from it. If an ag | --- | --- | --- | | `BASELINE_SAVE_FAILED` | The lease has ended, but Simlock could not copy the device. The message says why. No baseline was saved, and the device is kept aside for `simlock doctor`. | Lease a new device, prepare it again and save again. | | `UNKNOWN_BASELINE` | No baseline on this worker has that name. | Check the name with `simlock baseline list`. | -| `BASELINE_STALE` | The runtime or image the baseline needs is gone. | Install it again, or save a new version. | +| `BASELINE_STALE` | The runtime or image the baseline needs is gone, or an iOS baseline was saved before simulators stopped sharing folders. | Install it again, or save a new version. | | `BASELINE_IN_USE` | `remove` was refused because a lease uses a copy. | Wait for that lease to end, then remove. | | `DEVICE_BUSY` | `remove` was refused because a copy is being cleaned up or another operation holds it. | Try again in a moment. | | `FORBIDDEN` | The session does not have the admin role. | Run the command as an admin. | From 549d42276fbdef036a03e668b527d0776bef7af7 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:41:18 +0100 Subject: [PATCH 38/48] test: the filesystem port reads a link, repoints it and writes at an offset (#1570) --- src/ports/filesystem.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/ports/filesystem.test.ts b/src/ports/filesystem.test.ts index 61ea824a2..2cec8727b 100644 --- a/src/ports/filesystem.test.ts +++ b/src/ports/filesystem.test.ts @@ -235,6 +235,23 @@ describe.each(implementations)("Filesystem contract: $name", ({ create, link }) ); }); + it("reads where a link points, repoints it in one step, and writes bytes into a file at an offset without truncating it, and refuses to read a file as a link or write into a file that is not there", async () => { + const filesystem = create(); + const dir = temporaryDirectory; + await filesystem.mkdirp(dir); + await filesystem.writeFileAtomic(`${dir}/file`, "abcdef"); + await link(filesystem, `${dir}/link`, `${dir}/old`); + + await expect(filesystem.readlink(`${dir}/link`)).resolves.toBe(`${dir}/old`); + await filesystem.replaceSymlink(`${dir}/link`, `${dir}/new`); + await expect(filesystem.readlink(`${dir}/link`)).resolves.toBe(`${dir}/new`); + await filesystem.writeAt(`${dir}/file`, 2, new TextEncoder().encode("XY")); + await expect(filesystem.readFile(`${dir}/file`)).resolves.toBe("abXYef"); + + await expect(filesystem.readlink(`${dir}/file`)).rejects.toThrow(); + await expect(filesystem.writeAt(`${dir}/missing`, 0, new Uint8Array(1))).rejects.toThrow(); + }); + it("refuses to answer for a path that runs through a file", async () => { // why: A mistyped device root -- `/notes.txt/ios` -- is a different problem from one // that is simply not there yet, and callers can only tell them apart by the errno. From ed609c631170471fba5374d440290e177bdf6185 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:41:55 +0100 Subject: [PATCH 39/48] refactor(ios): the source record, the throwaway boot and the file walk are functions of their own (#1570) --- src/drivers/ios/lineage-files.ts | 60 ++++++++++++++++++++------------ src/drivers/ios/seeds.ts | 44 ++++++++++++++--------- 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts index 2ee6bf76a..c2083404f 100644 --- a/src/drivers/ios/lineage-files.ts +++ b/src/drivers/ios/lineage-files.ts @@ -74,31 +74,47 @@ export interface RecordOptions { */ export async function recordLineageFiles(options: RecordOptions): Promise { const files: SourceFile[] = []; - const visit = async (directory: string, relative: string): Promise => { - for (const name of (await options.filesystem.readdir(directory)).sort()) { - const path = `${directory}/${name}`; - const relativePath = relative === "" ? name : `${relative}/${name}`; - const details = await options.filesystem.lstat(path); - if (details.kind === "directory") { - await visit(path, relativePath); - } else if (details.kind === "symlink") { - const target = await options.filesystem.readlink(path); - if (namesLineage(Buffer.from(target), options.lineage)) { - files.push({ kind: "link", relativePath }); - } - } else if (details.kind === "file") { - if (name.endsWith(WAL_SUFFIX)) await flushWal(options, path); - else { - const kind = await classify(options, path); - if (kind !== undefined) files.push({ kind, relativePath }); - } - } - } - }; - await visit(options.folder, ""); + await visitFolder(options, options.folder, "", files); return files; } +async function visitFolder( + options: RecordOptions, + directory: string, + relative: string, + files: SourceFile[], +): Promise { + for (const name of (await options.filesystem.readdir(directory)).sort()) { + const path = `${directory}/${name}`; + const relativePath = relative === "" ? name : `${relative}/${name}`; + const details = await options.filesystem.lstat(path); + if (details.kind === "directory") { + await visitFolder(options, path, relativePath, files); + continue; + } + const kind = await recordedKind(options, path, details.kind); + if (kind !== undefined) files.push({ kind, relativePath }); + } +} + +/** why: a link is recorded when its target names the lineage; a WAL is flushed and never recorded; any other file is recorded when its bytes name it or cannot be read. */ +async function recordedKind( + options: RecordOptions, + path: string, + entry: string, +): Promise { + if (entry === "symlink") { + const target = await options.filesystem.readlink(path); + return namesLineage(Buffer.from(target), options.lineage) ? "link" : undefined; + } + if (entry !== "file") return undefined; + if (path.endsWith(WAL_SUFFIX)) { + await flushWal(options, path); + return undefined; + } + return classify(options, path); +} + async function flushWal(options: RecordOptions, wal: string): Promise { if (options.sqlite === undefined) return; const database = wal.slice(0, -WAL_SUFFIX.length); diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index a73b38b49..5ea753aa6 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -20,7 +20,12 @@ import type { ResolvedIosSpec } from "./catalog.js"; import { UDID_PATTERN } from "./device-set-leftovers.js"; -import { IosSourceTable, type ReadySeed, type SourceRow } from "./source-data.js"; +import { + IosSourceTable, + type ReadySeed, + type SourceRecord, + type SourceRow, +} from "./source-data.js"; import { recordLineageFiles, swapCopyFiles } from "./lineage-files.js"; @@ -165,16 +170,9 @@ export class Seeds { this.#sources.setUdid(sourceId, cloneUdid); await this.#slimBoot.copyOverrides(data.udid, cloneUdid); await this.#dropLaunchServicesDatabase(this.#sourcesSet, cloneUdid); - // why: ADR 0076 §2: the baseline's lineage is its lease's UDID and the lineage the lease was copied with; a lease made before lineages were kept has none, so the baseline is never copied. - const lineage = - data.lineage === undefined ? undefined : [data.udid.toUpperCase(), ...data.lineage]; - const files = lineage === undefined ? [] : await this.#recordFiles(cloneUdid, lineage); + const record = await this.#baselineRecord(data, cloneUdid); const sizeBytes = await this.#sizeOf(cloneUdid); - this.#sources.markReady(sourceId, sizeBytes, { - files, - lineage, - savedWithoutLineage: lineage === undefined, - }); + this.#sources.markReady(sourceId, sizeBytes, record); // why: ADR 0076: the lease is destroyed next, and `simctl delete` leaves its log folder behind for the baseline's copies to inherit. await this.#deleteLogFolder(data.udid); return { sourceId, ...(sizeBytes === undefined ? {} : { sizeBytes }) }; @@ -187,6 +185,14 @@ export class Seeds { } } + /** why: ADR 0076 §2: the baseline's lineage is its lease's UDID and the lineage the lease was copied with; a lease made before lineages were kept has none, so the baseline is never copied. */ + async #baselineRecord(data: IosDriverData, cloneUdid: string): Promise { + const lineage = + data.lineage === undefined ? undefined : [data.udid.toUpperCase(), ...data.lineage]; + const files = lineage === undefined ? [] : await this.#recordFiles(cloneUdid, lineage); + return { files, lineage, savedWithoutLineage: lineage === undefined }; + } + async destroySource(sourceId: string): Promise { const row = this.#sources.findRow(sourceId); if (row === undefined) { @@ -321,6 +327,16 @@ export class Seeds { return this.#makeSeed(resolved, specKey); } + async #bootThrowaway(throwaway: string, sourceId: string): Promise { + try { + await this.#boot.bootAndWait(throwaway, sourceId, this.#fullBootTimeoutMs); + } catch (error: unknown) { + // why: a seed that ran out of boot time is a boot timeout, as an Android seed's is. + if (error instanceof BootTimeoutError) throw error; + throw new DriverCrashError(`seed ${throwaway} did not boot: ${errorMessage(error)}`); + } + } + async #makeSeed(resolved: ResolvedIosSpec, specKey: string): Promise { // why: a `making` row of this process older than the longest a make can take belongs to a make that never ended. for (const row of this.#sources.makingRowsOfKindBefore( @@ -370,13 +386,7 @@ export class Seeds { await this.#marks.writeMark(throwaway); await this.#deviceHub.disablePasteboardSync(throwaway); } - try { - await this.#boot.bootAndWait(throwaway, sourceId, this.#fullBootTimeoutMs); - } catch (error: unknown) { - // why: a seed that ran out of boot time is a boot timeout, as an Android seed's is. - if (error instanceof BootTimeoutError) throw error; - throw new DriverCrashError(`seed ${throwaway} did not boot: ${errorMessage(error)}`); - } + await this.#bootThrowaway(throwaway, sourceId); await this.#simctl.shutdown(throwaway); const cloned = await this.#simctl.createSimulator( ["clone", throwaway, `simlock-seed-${sourceId}`, this.#sourcesSet], From c25fc5f87806d43dcbe1c8754a95d0abf3199a0e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:48:42 +0100 Subject: [PATCH 40/48] test: the edge of a swap window, a baseline copy's swap and a file's time kept by writeAt (#1570) Docs: a baseline saved from a lease made before lineages is stale too. --- src/drivers/ios/index-seed-sources.test.ts | 35 ++++++++++++++++--- src/drivers/ios/index-seed-swap.test.ts | 7 +++- src/ports/filesystem.test.ts | 6 +++- .../src/content/docs/docs/reference/cli.mdx | 2 +- .../content/docs/docs/reference/errors.mdx | 2 +- .../docs/docs/using-devices/errors.mdx | 2 +- .../content/docs/docs/worker/baselines.mdx | 4 +-- 7 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts index 36652ee30..995222c37 100644 --- a/src/drivers/ios/index-seed-sources.test.ts +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -2,7 +2,13 @@ import { afterEach, expect, it, describe } from "vitest"; import { DriverCrashError, BootTimeoutError } from "../../core/index.js"; import { ScriptedProcessRunner, FakeClock, MemoryFilesystem } from "../../ports/index.js"; import { promiseState } from "../../test-support/promise-state.js"; -import { type PlantedSource, plantSource, memorySourceDatabase } from "./testing.js"; +import { + type PlantedSource, + plantBytes, + plantSource, + memorySourceDatabase, + readBytes, +} from "./testing.js"; import { deviceRoot, driverData, @@ -143,7 +149,11 @@ describe("IosSimctlDriver", () => { }); it.each([ - { name: "a named source is copied instead of the seed", id: "planted-named", cloned: true }, + { + name: "a named source is copied instead of the seed, with its lineage swapped for the copy's own UDID", + id: "planted-named", + cloned: true, + }, { name: "an unknown source is refused", id: "nope", cloned: false }, { name: "a stale source is refused", id: "planted-old", cloned: false }, { name: "a source with no lineage is refused", id: "planted-unlineaged", cloned: false }, @@ -153,10 +163,18 @@ describe("IosSimctlDriver", () => { const unlineagedUdid = "00000000-0000-0000-0000-0000000000dd"; const database = await seededDatabase(); const kind = { deviceTypeId: driverData.deviceTypeId, runtimeId: driverData.runtimeId }; - const planted = plantSource(database, { ...kind, kind: "baseline", udid: namedUdid }); + const namedLineage = "AAAAAAAA-1111-2222-3333-BBBBBBBBBBBB"; + const planted = plantSource(database, { + ...kind, + files: [{ kind: "file", relativePath: "data/note.plist" }], + kind: "baseline", + lineage: [namedLineage], + udid: namedUdid, + }); database .prepare("UPDATE ios_source_data SET source_id = 'planted-named' WHERE udid = ?") .run(namedUdid); + database.prepare("UPDATE ios_source_files SET source_id = 'planted-named'").run(); plantSource(planted, { ...kind, kind: "baseline", runtimeBuild: "23F77", udid: oldUdid }); database .prepare("UPDATE ios_source_data SET source_id = 'planted-old' WHERE udid = ?") @@ -169,6 +187,8 @@ describe("IosSimctlDriver", () => { await plantManagedDevice(filesystem); await plantSeedClone(filesystem); await plantLaunchServices(filesystem, `${deviceRoot}/${driverData.udid}`); + const note = `${deviceRoot}/${driverData.udid}/data/note.plist`; + await plantBytes(filesystem, note, Buffer.from(`root /Devices/${namedLineage}/data`)); const catalog = JSON.parse(listFixture) as { runtimes: { buildversion?: string; identifier: string }[]; }; @@ -189,8 +209,15 @@ describe("IosSimctlDriver", () => { const provisioned = driver.provision(spec, { source: id }); if (cloned) { - await expect(provisioned).resolves.toMatchObject({ deviceId: driverData.udid }); + await expect(provisioned).resolves.toMatchObject({ + deviceId: driverData.udid, + driverData: { lineage: [namedLineage] }, + }); expect(runner.calls.map((call) => call.args[3])).toEqual(["list", "list", "clone"]); + expect( + (await readBytes(filesystem, note)).toString(), + "the copy of a baseline names its own folder in place of the baseline's lineage", + ).toBe(`root /Devices/${driverData.udid.toUpperCase()}/data`); expect( await launchServicesFiles(filesystem, `${deviceRoot}/${driverData.udid}`), "the copy of a source loses the LaunchServices database that names the source's folder", diff --git a/src/drivers/ios/index-seed-swap.test.ts b/src/drivers/ios/index-seed-swap.test.ts index 4d858a2cb..bb97b5b57 100644 --- a/src/drivers/ios/index-seed-swap.test.ts +++ b/src/drivers/ios/index-seed-swap.test.ts @@ -69,6 +69,11 @@ describe("IosSimctlDriver copy swap", () => { before: spanning(utf16le(lineageUdid), window - 36), after: spanning(utf16le(copyUdid), window - 36), }, + { + name: "edge-utf16", + before: spanning(utf16le(lineageUdid), window - 1), + after: spanning(utf16le(copyUdid), window - 1), + }, { name: "big", before: big, after: bigSwapped }, { name: "own", before: ascii(`own ${copyUdid}`), after: ascii(`own ${copyUdid}`) }, { @@ -117,7 +122,7 @@ describe("IosSimctlDriver copy swap", () => { ); expect(device.driverData).toMatchObject({ lineage: [lineageUdid] }); expect(logger.debugs).toEqual([ - `swapped 8 files and 1 links in 35 ms for ${copyUdid}; 1 unreadable`, + `swapped 9 files and 1 links in 40 ms for ${copyUdid}; 1 unreadable`, ]); }); diff --git a/src/ports/filesystem.test.ts b/src/ports/filesystem.test.ts index 2cec8727b..932f08414 100644 --- a/src/ports/filesystem.test.ts +++ b/src/ports/filesystem.test.ts @@ -235,11 +235,14 @@ describe.each(implementations)("Filesystem contract: $name", ({ create, link }) ); }); - it("reads where a link points, repoints it in one step, and writes bytes into a file at an offset without truncating it, and refuses to read a file as a link or write into a file that is not there", async () => { + it("reads where a link points, repoints it, and writes bytes into a file at an offset without truncating it or touching its modification time, and refuses to read a file as a link or write into a file that is not there", async () => { const filesystem = create(); const dir = temporaryDirectory; await filesystem.mkdirp(dir); await filesystem.writeFileAtomic(`${dir}/file`, "abcdef"); + // why: the memory filesystem has no disk file to age, and its write never touches the time. + if (filesystem instanceof NodeFilesystem) await utimes(`${dir}/file`, 1_000_000, 1_000_000); + const modifiedBefore = (await filesystem.stat(`${dir}/file`)).modifiedAtMs; await link(filesystem, `${dir}/link`, `${dir}/old`); await expect(filesystem.readlink(`${dir}/link`)).resolves.toBe(`${dir}/old`); @@ -247,6 +250,7 @@ describe.each(implementations)("Filesystem contract: $name", ({ create, link }) await expect(filesystem.readlink(`${dir}/link`)).resolves.toBe(`${dir}/new`); await filesystem.writeAt(`${dir}/file`, 2, new TextEncoder().encode("XY")); await expect(filesystem.readFile(`${dir}/file`)).resolves.toBe("abXYef"); + expect((await filesystem.stat(`${dir}/file`)).modifiedAtMs).toBe(modifiedBefore); await expect(filesystem.readlink(`${dir}/file`)).rejects.toThrow(); await expect(filesystem.writeAt(`${dir}/missing`, 0, new Uint8Array(1))).rejects.toThrow(); diff --git a/website/src/content/docs/docs/reference/cli.mdx b/website/src/content/docs/docs/reference/cli.mdx index 7fdf28eaf..88e8ba622 100644 --- a/website/src/content/docs/docs/reference/cli.mdx +++ b/website/src/content/docs/docs/reference/cli.mdx @@ -453,7 +453,7 @@ simlock doctor [--fix] [--purge-orphans] [--yes] | `--purge-orphans` | Destroy every orphaned device: a device in Simlock's device folders with no record. Also destroy every orphan source: a seed simulator with no record, or a saved baseline's copy that no baseline names. Needs the admin role. Asks for confirmation. | Off | | `--yes` | Skip the confirmation for `--purge-orphans`. Required when there is no terminal to ask in. | Off | -The report also lists, under `sources`, the copies on disk that new devices are made from. Each has `platform`, `kind` (`seed` or `baseline`), `model`, `osVersion`, `sizeBytes` when known, and `staleReason` when it no longer matches the installed runtime, or when an iOS baseline was saved before simulators stopped sharing folders. A baseline source also has `baselineName` and `baselineVersion`. A seed is a simulator Simlock copies to make each iOS simulator a lease gets: it takes disk space but is never leased. A seed with no record, or a baseline copy no baseline names, shows `orphan: true`. That is not a finding; only `--purge-orphans` deletes it. +The report also lists, under `sources`, the copies on disk that new devices are made from. Each has `platform`, `kind` (`seed` or `baseline`), `model`, `osVersion`, `sizeBytes` when known, and `staleReason` when it no longer matches the installed runtime, or when an iOS baseline was saved before simulators stopped sharing folders, or from a lease made before that. A baseline source also has `baselineName` and `baselineVersion`. A seed is a simulator Simlock copies to make each iOS simulator a lease gets: it takes disk space but is never leased. A seed with no record, or a baseline copy no baseline names, shows `orphan: true`. That is not a finding; only `--purge-orphans` deletes it. On a worker whose login service names a Node or `main.js` that is gone, the report holds a `service-broken` finding and stderr gets `Service broken: () names missing . Run `simlock daemon install` again.` (`, ` for two). diff --git a/website/src/content/docs/docs/reference/errors.mdx b/website/src/content/docs/docs/reference/errors.mdx index 854959ce9..4d2f54ff1 100644 --- a/website/src/content/docs/docs/reference/errors.mdx +++ b/website/src/content/docs/docs/reference/errors.mdx @@ -69,7 +69,7 @@ In a script, check `code`, not the message: the wording of a message can change, | `BASELINE_SAVE_FAILED` | 1 | 500 | `baseline save` ended the lease, but Simlock could not keep a copy of the device. The message gives the reason. No baseline was saved, and the device is set aside for cleanup. | Lease a fresh device, prepare it again and save again. | | `UNKNOWN_BASELINE` | 12 | 404 | `baseline remove`, or a lease request with `--baseline`, naming a baseline this worker does not have. Through a gateway: no worker holds the baseline, or the worker `--worker` names does not. | List the baselines with `simlock baseline list`. | | `BASELINE_MISMATCH` | 2 | 422 | A lease request names a baseline and also a platform, model, OS, mode or image tag that differs from it. The message names the first field that differs. | Drop the field, or change it to the baseline's own value. | -| `BASELINE_STALE` | 12 | 422 | A lease request names a baseline whose runtime or image has been removed since it was saved, or an iOS baseline saved before simulators stopped sharing folders. The message gives the reason. | Install the runtime or image again, or save the baseline again from a new device. | +| `BASELINE_STALE` | 12 | 422 | A lease request names a baseline whose runtime or image has been removed since it was saved, or an iOS baseline saved before simulators stopped sharing folders, or from a lease made before that. The message gives the reason. | Install the runtime or image again, or save the baseline again from a new device. | | `BASELINE_IN_USE` | 12 | 409 | `baseline remove` of a baseline a lease still uses. The message names the lease. | End that lease, or remove the baseline after it. | | `COMPONENT_BUSY` | 11 | 409 | `component remove` while an install or removal runs or waits on that platform. | Try again once it ends. | diff --git a/website/src/content/docs/docs/using-devices/errors.mdx b/website/src/content/docs/docs/using-devices/errors.mdx index ddf20c698..d4f2f2242 100644 --- a/website/src/content/docs/docs/using-devices/errors.mdx +++ b/website/src/content/docs/docs/using-devices/errors.mdx @@ -45,7 +45,7 @@ In a script, check the exit code or `code`, never the message: the message is fo | `RUNTIME_MISSING` | 12 | The OS version, or the Android image, isn't installed. An OS range or `--image-tag` never downloads. | Pass `--allow-download` for an exact version, or ask an admin to install it. See [Choosing a device](/simlock/docs/using-devices/choosing-a-device#when-the-runtime-is-not-installed). | | `UNKNOWN_BASELINE` | 12 | `--baseline` names a baseline this machine doesn't have. | Check the name with `simlock baseline list`. | | `BASELINE_MISMATCH` | 2 | `--baseline` came with a platform, model, OS, mode or image tag that differs from the baseline. The message names the first one that differs. | Drop that option: the baseline already decides it. | -| `BASELINE_STALE` | 12 | The OS or image the baseline was saved on has been removed, or an iOS baseline was saved before simulators stopped sharing folders. | Ask an admin to install it again, or save the baseline again. | +| `BASELINE_STALE` | 12 | The OS or image the baseline was saved on has been removed, or an iOS baseline was saved before simulators stopped sharing folders, or from a lease made before that. | Ask an admin to install it again, or save the baseline again. | | `LICENSE_NOT_ACCEPTED` | 12 | An Android download needs the SDK licences accepted, and this machine hasn't accepted them. | An admin sets `downloads.acceptAndroidLicenses` to `true`. | | `INSUFFICIENT_DISK_SPACE` | 12 | The download doesn't fit on the disk. The message says how much is needed and free. | Free disk space on that machine, or ask for an installed version. | | `DOWNLOAD_TIMEOUT` | 10 | The download, including time waiting behind another download, took longer than the machine allows. | Retry; the download may already be further along. Or ask an admin to install the runtime. | diff --git a/website/src/content/docs/docs/worker/baselines.mdx b/website/src/content/docs/docs/worker/baselines.mdx index ea453e76d..683273eab 100644 --- a/website/src/content/docs/docs/worker/baselines.mdx +++ b/website/src/content/docs/docs/worker/baselines.mdx @@ -72,7 +72,7 @@ login 1 ios iPhone 17 27.0 slim - 3.1 GB 2026-11 stale: runtime com.apple.CoreSimulator.SimRuntime.iOS-27-0 is not installed ``` -A baseline is stale when the iOS runtime, Android system image or emulator it was saved with has changed or been removed. An iOS baseline saved before Simlock kept each simulator in its own folders is stale too; save it again from a new lease. The reason is on the line under it. A lease that names a stale baseline fails with `BASELINE_STALE`. Install that runtime or image again and the baseline works again; or save a new version from a device on the runtime you have now. +A baseline is stale when the iOS runtime, Android system image or emulator it was saved with has changed or been removed. An iOS baseline is stale too when it was saved before Simlock kept each simulator in its own folders, or from a lease made before that; save it again from a new lease. The reason is on the line under it. A lease that names a stale baseline fails with `BASELINE_STALE`. Install that runtime or image again and the baseline works again; or save a new version from a device on the runtime you have now. ## Remove a baseline @@ -95,7 +95,7 @@ This deletes every version of `login` and the idle copies made from it. If an ag | --- | --- | --- | | `BASELINE_SAVE_FAILED` | The lease has ended, but Simlock could not copy the device. The message says why. No baseline was saved, and the device is kept aside for `simlock doctor`. | Lease a new device, prepare it again and save again. | | `UNKNOWN_BASELINE` | No baseline on this worker has that name. | Check the name with `simlock baseline list`. | -| `BASELINE_STALE` | The runtime or image the baseline needs is gone, or an iOS baseline was saved before simulators stopped sharing folders. | Install it again, or save a new version. | +| `BASELINE_STALE` | The runtime or image the baseline needs is gone, or an iOS baseline was saved before simulators stopped sharing folders, or from a lease made before that. | Install it again, or save a new version. | | `BASELINE_IN_USE` | `remove` was refused because a lease uses a copy. | Wait for that lease to end, then remove. | | `DEVICE_BUSY` | `remove` was refused because a copy is being cleaned up or another operation holds it. | Try again in a moment. | | `FORBIDDEN` | The session does not have the admin role. | Run the command as an admin. | From 1ea76edfb3f357affcfdbbc441e402fb4d056a66 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:52:11 +0100 Subject: [PATCH 41/48] docs: the iOS and core manuals fit their size limits and say what the code records (#1570) --- src/core/AGENTS.md | 2 +- src/drivers/ios/AGENTS.md | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 3bf002db4..104bff87c 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -58,7 +58,7 @@ Baselines (ADR 0061 §4): - One save or remove at a time. A save: leasing's `releaseToSave` ends the lease, then `ReclaimCoordinator.retireSaved` calls `driver.saveSource` under the `reclaim` claim. A driver failure quarantines the device and answers `BASELINE_SAVE_FAILED`. - A replace keeps the leased devices of older versions. An older version is dropped only when no device record and no provision (`hold`) has it; that is checked after a save, on every `device.deleted` and at startup. - `remove` refuses a baseline a lease uses, and destroys each source before its row. `removeAll` (nuke) deletes rows first. -- `checkStale` runs before `list` and `doctor`, after `component.removed`, after a save, and at startup before the warm pool starts. A failing check is logged and the save or the start goes on. +- `checkStale` runs before `list` and `doctor`, after `component.removed`, after a save, and at startup. A failure is logged and goes on. ## Known gaps diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index 3b0dd0a54..f8b942d34 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -23,17 +23,16 @@ Resolve and boot: - The overrides folder lives outside the device set (`launchdOverridesRoot`), so `destroy` removes it itself after `simctl delete`. - After every boot and before the readback, `runAfterBoot` (`boot-courtesies.ts`) runs the volume step, then the window step. Each only reports its failure (`onSilenceFailed`, `onWindowFailed`). - Volume (`ios.simulator.audio: false`): apps play at volume 0. A baseline source is not silenced. -- Window (`ios.simulator.headless: false`): DeviceHub is opened in the background (the front app stays in front) and told not to shut down started devices when it quits. Only a non-zero exit of that open means DeviceHub is missing: Simulator.app is then opened instead, detached, so quitting it leaves the simulator running. +- Window (`ios.simulator.headless: false`): DeviceHub is opened in the background and told not to shut down started devices when it quits. Only a non-zero exit of that open means DeviceHub is missing: Simulator.app is then opened instead, detached, so quitting it leaves the simulator running. Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources set: `sources` in the owned set, an owned root of its own under `default`. It is in no `listManaged` or `simlock simctl list`. -- A seed is made from a throwaway (ADR 0076): `simlock-seed-` is booted once, shut down, cloned unbooted into the sources set, then deleted with its log folder (`logsRoot`); `saveSource` deletes the saved-from lease's log folder too. A log folder that cannot be deleted is logged and the make or save goes on; a throwaway that cannot be deleted fails the make. Only its `making` row (`throwaway_udid`) deletes a throwaway, at the first `listSources` of a later process or the next make of its kind. -- A source records its lineage (the UDIDs of the simulators it was copied from: a seed's is its throwaway, a baseline's is the saved-from device then that device's own) and, in `ios_source_files`, every file and link of its folder, after flushing its SQLite WALs and before the row turns `ready`. A device keeps the lineage of the source it was copied from. -- Every clone (a seed's copy, `saveSource`'s own clone, a `provision` with `source`) has its `data/var/db/lsd/com.apple.LaunchServices-*.csstore` deleted and, in the recorded files, every form of its lineage's UDIDs (ASCII and UTF-16, either case) swapped for its own, and its links re-pointed, before its first boot. A clone whose drop or swap fails is deleted. A source with no lineage is never copied: a seed is made again, a baseline is stale (`saved before simulators stopped sharing folders; save it again`). +- A seed is made from a throwaway (ADR 0076): `simlock-seed-` is booted once, shut down, cloned unbooted into the sources set, then deleted with its log folder; `saveSource` deletes the saved-from lease's log folder too. A log folder that cannot be deleted is logged and the make or save goes on; a throwaway that cannot be deleted fails the make. +- A source records its lineage (a seed's throwaway; a baseline's saved-from device, then that device's own) and, in `ios_source_files`, the files and links naming it, after flushing its WALs. A clone keeps its source's lineage and, before its first boot, has its `com.apple.LaunchServices-*.csstore` deleted and those files and links swapped to its own UDID; a failed drop or swap deletes it. No lineage: a seed is remade, a baseline stale. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. -- A `making` row of another process is swept at the first `listSources`; one of this process, once older than any make takes, at the next make of its kind. +- A `making` row of another process is swept at the first `listSources`, with its throwaway; one of this process, once older than any make takes, at the next make of its kind. - A seed that `simctl delete` calls `Invalid device` is already gone: its row is dropped. `removeComponent` deletes the runtime's seeds first. Physical, passthrough, screen: From 3c09207d51a719c770b9cc56fb874323955d98d2 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 21:54:57 +0100 Subject: [PATCH 42/48] test: lineage round-trips and a non-UDID is refused; deleting a source deletes its file rows (#1570) --- src/drivers/ios/device-data.test.ts | 2 ++ src/drivers/ios/index-seed-sources.test.ts | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/drivers/ios/device-data.test.ts b/src/drivers/ios/device-data.test.ts index 40ad35a18..142312594 100644 --- a/src/drivers/ios/device-data.test.ts +++ b/src/drivers/ios/device-data.test.ts @@ -13,6 +13,7 @@ describeDeviceDataTable("iosDeviceDataTable", iosDeviceDataTable, [ base, { ...base, slimMarkToken: "tok-1", slimSignature: "sig-1" }, { ...base, slimSignature: "sig-2" }, + { ...base, lineage: ["9F2C0000-0000-0000-0000-000000000002", base.udid] }, ]); describe("iosDeviceDataTable write", () => { @@ -23,6 +24,7 @@ describe("iosDeviceDataTable write", () => { "required key of the wrong type": { ...base, udid: 5 }, "optional key of the wrong type": { ...base, slimSignature: 5 }, "extra key": { ...base, extra: "x" }, + "lineage entry that is not a UDID": { ...base, lineage: ["not-a-udid"] }, }; const state = await openMemoryState([iosDeviceDataTable]); for (const [label, value] of Object.entries(cases)) { diff --git a/src/drivers/ios/index-seed-sources.test.ts b/src/drivers/ios/index-seed-sources.test.ts index 995222c37..d923e9e0f 100644 --- a/src/drivers/ios/index-seed-sources.test.ts +++ b/src/drivers/ios/index-seed-sources.test.ts @@ -712,6 +712,7 @@ describe("IosSimctlDriver", () => { ])("destroySource $name", async ({ deleteCode, stderr, rows }) => { const database = plantSource(await memorySourceDatabase(), { deviceTypeId: driverData.deviceTypeId, + files: [{ kind: "file", relativePath: "data/a.plist" }], runtimeId: driverData.runtimeId, udid: seedUdid, }); @@ -734,6 +735,9 @@ describe("IosSimctlDriver", () => { expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_data").get()).toEqual({ n: rows, }); + expect(database.prepare("SELECT COUNT(*) AS n FROM ios_source_files").get()).toEqual({ + n: rows, + }); }); it("destroySource refuses an id that names no source, without running simctl", async () => { From 767222e2e041817f59d40fd9cfcc2e6159b16b1e Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 22:18:36 +0100 Subject: [PATCH 43/48] fix: every WAL is flushed before its database is read; section cites, baseline.stale and checkStale claims follow the code (#1570) --- docs/internal/EVENTS.md | 2 +- src/core/AGENTS.md | 2 +- src/drivers/ios/index-seed-record.test.ts | 8 +++++++- src/drivers/ios/lineage-files.ts | 14 ++++++++++---- src/drivers/ios/udid-swap.ts | 8 ++++---- src/ports/sqlite.ts | 16 +++++++++++++--- .../src/content/docs/docs/reference/events.mdx | 2 +- 7 files changed, 37 insertions(+), 15 deletions(-) diff --git a/docs/internal/EVENTS.md b/docs/internal/EVENTS.md index 9cefb77da..f783d44b6 100644 --- a/docs/internal/EVENTS.md +++ b/docs/internal/EVENTS.md @@ -124,7 +124,7 @@ consumer, just operator diagnostics, so it's a `warn` log line (`daemon.driver-d |---|---|---|---|---| | `baseline.saved` | name, version, platform, model, OS version, mode, image tag (when the device has one), size in bytes (when it could be counted), lease id (the lease the save ended), requester id (the admin principal that ran `baseline save`) | once per save, after the `baselines` row is committed and before the saved device is deleted | BaselineService | implemented | | `baseline.removed` | name, version, reason (`operator` for `baseline remove`, `nuke` for `nuke --delete-devices`, `replaced` when a newer save of the name dropped it), requester id (the principal that ran the command; absent for a removal nobody asked for) | once per version deleted, after its row is deleted | BaselineService | implemented | -| `baseline.stale` | name, version, reason (the driver's words: the runtime, image or emulator that changed) | when a check finds a baseline stale that had no stale reason before (again after it was fresh in between). The check runs at start, after `component.removed`, and before `baseline list` and `doctor` answer | BaselineService | implemented | +| `baseline.stale` | name, version, reason (the driver's words: the runtime, image or emulator that changed, or an iOS baseline saved without a lineage record) | when a check finds a baseline stale that had no stale reason before (again after it was fresh in between). The check runs at start, after `component.removed`, after a save, and before `baseline list` and `doctor` answer | BaselineService | implemented | `BaselineService` (`src/core/baselines.ts`) is the only emitter; drivers keep and destroy the sources but never emit (architecture rule 5). diff --git a/src/core/AGENTS.md b/src/core/AGENTS.md index 104bff87c..32bf432c7 100644 --- a/src/core/AGENTS.md +++ b/src/core/AGENTS.md @@ -58,7 +58,7 @@ Baselines (ADR 0061 §4): - One save or remove at a time. A save: leasing's `releaseToSave` ends the lease, then `ReclaimCoordinator.retireSaved` calls `driver.saveSource` under the `reclaim` claim. A driver failure quarantines the device and answers `BASELINE_SAVE_FAILED`. - A replace keeps the leased devices of older versions. An older version is dropped only when no device record and no provision (`hold`) has it; that is checked after a save, on every `device.deleted` and at startup. - `remove` refuses a baseline a lease uses, and destroys each source before its row. `removeAll` (nuke) deletes rows first. -- `checkStale` runs before `list` and `doctor`, after `component.removed`, after a save, and at startup. A failure is logged and goes on. +- `checkStale` runs before `list` and `doctor`, after `component.removed`, after a save, and at startup. A failure is logged and goes on, except before `list`, which fails with it. ## Known gaps diff --git a/src/drivers/ios/index-seed-record.test.ts b/src/drivers/ios/index-seed-record.test.ts index fd5d9d4c6..03b6d9a9c 100644 --- a/src/drivers/ios/index-seed-record.test.ts +++ b/src/drivers/ios/index-seed-record.test.ts @@ -203,6 +203,8 @@ describe("IosSimctlDriver source records", () => { { kind: "link", relative_path: "data/Logs" }, { kind: "file", relative_path: "data/a.plist" }, { kind: "file", relative_path: "data/big.bin" }, + { kind: "file", relative_path: "data/one.db" }, + { kind: "file", relative_path: "data/two.db" }, { kind: "file", relative_path: "data/wide.bin" }, ]); expect(seed.logger.warnings).toEqual([]); @@ -222,7 +224,10 @@ describe("IosSimctlDriver source records", () => { `could not flush ${seedFolder}/data/one.db: unable to open database file ${seedFolder}/data/one.db`, `WAL ${seedFolder}/data/two.db-wal still holds frames after the flush; left unpatched`, ]); - expect(recorded(unflushed.database), "a WAL is never recorded").toEqual([]); + expect( + recorded(unflushed.database), + "a WAL is never recorded, and a database whose frames stayed names nothing", + ).toEqual([]); const unreadable = await provisionNewSeed({ plant: async (filesystem) => { @@ -287,6 +292,7 @@ describe("IosSimctlDriver source records", () => { }); expect(recorded(fromUsedLease.database)).toEqual([ { kind: "file", relative_path: "data/lease.plist" }, + { kind: "file", relative_path: "data/one.db" }, { kind: "file", relative_path: "data/seed.plist" }, ]); diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts index c2083404f..648dabce9 100644 --- a/src/drivers/ios/lineage-files.ts +++ b/src/drivers/ios/lineage-files.ts @@ -9,7 +9,7 @@ import { errorMessage } from "./command-run.js"; import type { SourceFile } from "./source-data.js"; import { SWAP_OVERLAP_BYTES, namesLineage, swapLineage, type LineageSwap } from "./udid-swap.js"; -/** why: a window is read whole, so this bounds memory at any file size (ADR 0076 §3). */ +/** why: a window is read whole, so this bounds memory at any file size. */ const WINDOW_BYTES = 1_048_576; const WAL_SUFFIX = "-wal"; @@ -20,7 +20,7 @@ export interface SwapCounts { readonly unreadable: number; } -/** why: ADR 0076 §4: before its first boot a copy's recorded files and links name its own UDID in place of the lineage's, each keeping its length; an unreadable file is left. */ +/** why: ADR 0076 §3: before its first boot a copy's recorded files and links name its own UDID in place of the lineage's, each keeping its length; an unreadable file is left. */ export async function swapCopyFiles( filesystem: Filesystem, folder: string, @@ -70,7 +70,7 @@ export interface RecordOptions { } /** - * why: ADR 0076 §3: when a source is made every WAL in it is flushed into its database, and the files and links that name a lineage UDID are found once and recorded, so a copy patches only those. + * why: ADR 0076 §2: when a source is made every WAL in it is flushed into its database, and the files and links that name a lineage UDID are found once and recorded, so a copy patches only those. */ export async function recordLineageFiles(options: RecordOptions): Promise { const files: SourceFile[] = []; @@ -84,7 +84,13 @@ async function visitFolder( relative: string, files: SourceFile[], ): Promise { - for (const name of (await options.filesystem.readdir(directory)).sort()) { + const names = (await options.filesystem.readdir(directory)).sort(); + // why: a database and its WAL are siblings, and a database is read only after its WAL is flushed into it (ADR 0076 §2). + const ordered = [ + ...names.filter((name) => name.endsWith(WAL_SUFFIX)), + ...names.filter((name) => !name.endsWith(WAL_SUFFIX)), + ]; + for (const name of ordered) { const path = `${directory}/${name}`; const relativePath = relative === "" ? name : `${relative}/${name}`; const details = await options.filesystem.lstat(path); diff --git a/src/drivers/ios/udid-swap.ts b/src/drivers/ios/udid-swap.ts index f09635c20..462241b66 100644 --- a/src/drivers/ios/udid-swap.ts +++ b/src/drivers/ios/udid-swap.ts @@ -1,4 +1,4 @@ -/** why: a UDID is 36 characters, so its UTF-16 form is 72 bytes; a window overlaps the next by one byte less, and a match straddling two windows lies whole in one of them (ADR 0076 §4). */ +/** why: a UDID is 36 characters, so its UTF-16 form is 72 bytes; a window overlaps the next by one byte less, and a match straddling two windows lies whole in one of them (ADR 0076 §3). */ export const SWAP_OVERLAP_BYTES = 71; export interface LineageSwap { @@ -6,7 +6,7 @@ export interface LineageSwap { readonly own: string; } -/** why: ADR 0076 §4: a UDID is stored upper or lower case, as ASCII or as UTF-16 of either byte order, and no other way is looked for. */ +/** why: ADR 0076 §3: a UDID is stored upper or lower case, as ASCII or as UTF-16 of either byte order, and no other way is looked for. */ function forms(udid: string): readonly Buffer[] { return [udid.toUpperCase(), udid.toLowerCase()].flatMap((text) => [ Buffer.from(text, "latin1"), @@ -15,13 +15,13 @@ function forms(udid: string): readonly Buffer[] { ]); } -/** why: the finder of ADR 0076 §3: whether any lineage UDID is in the bytes in any of its forms. */ +/** why: the finder of ADR 0076 §2: whether any lineage UDID is in the bytes in any of its forms. */ export function namesLineage(chunk: Uint8Array, lineage: readonly string[]): boolean { const bytes = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); return lineage.some((udid) => forms(udid).some((form) => bytes.includes(form))); } -/** why: the UDID swap (ADR 0076 §4) replaces each lineage UDID with the copy's own in the same case and encoding; a UDID has one length, so the output has the input's length, and a caller checks it. */ +/** why: the UDID swap (ADR 0076 §3) replaces each lineage UDID with the copy's own in the same case and encoding; a UDID has one length, so the output has the input's length, and a caller checks it. */ export function swapLineage(chunk: Uint8Array, swap: LineageSwap): Uint8Array { let out = Buffer.from(chunk); const ownForms = forms(swap.own); diff --git a/src/ports/sqlite.ts b/src/ports/sqlite.ts index bdfa62215..acc8b4f9d 100644 --- a/src/ports/sqlite.ts +++ b/src/ports/sqlite.ts @@ -50,8 +50,8 @@ export class NodeSqliteOpener implements SqliteOpener { /** * why: a fake for a caller that opens a database only to flush its WAL: it records each database - * a checkpoint ran on, can fail one, and empties the `-wal` file in a memory filesystem unless told - * to leave its frames. + * a checkpoint ran on, can fail one, and a checkpoint moves the `-wal` file's bytes onto the end of + * its database in a memory filesystem and empties the WAL, unless told to leave its frames. */ export class FakeSqliteOpener implements SqliteOpener { readonly checkpoints: string[] = []; @@ -73,9 +73,19 @@ export class FakeSqliteOpener implements SqliteOpener { async open(path: string): Promise { if (this.#failing.has(path)) throw new Error(`unable to open database file ${path}`); + // why: `exec` is synchronous, so what a checkpoint moves is read here, and the filesystem calls + // below run their bodies at once. + const wal = `${path}-wal`; + const frames = await this.#filesystem.readFile(wal).then( + (text) => Buffer.from(text), + () => undefined, + ); + const size = frames === undefined ? 0 : (await this.#filesystem.stat(path)).size; const checkpoint = (): void => { this.checkpoints.push(path); - if (!this.#keepingFrames.has(path)) void this.#filesystem.rm(`${path}-wal`); + if (this.#keepingFrames.has(path)) return; + if (frames !== undefined) void this.#filesystem.writeAt(path, size, frames); + void this.#filesystem.rm(wal); }; return { close: () => undefined, diff --git a/website/src/content/docs/docs/reference/events.mdx b/website/src/content/docs/docs/reference/events.mdx index f262e5246..5dcbe32e3 100644 --- a/website/src/content/docs/docs/reference/events.mdx +++ b/website/src/content/docs/docs/reference/events.mdx @@ -168,7 +168,7 @@ release and records nothing. | --- | --- | --- | --- | | `baseline.saved` | `simlock baseline save` kept a device as a baseline. | `name`, `version`, `platform`, `model`, `osVersion`, `mode`, `imageTag` when the device has one, `sizeBytes` when known, `leaseId`, `requesterId` | Relayed | | `baseline.removed` | One version of a baseline was deleted, by `baseline remove`, by `nuke --delete-devices` or because a newer save of the name replaced it. | `name`, `version`, `reason` (`operator`, `nuke` or `replaced`), `requesterId` when someone asked | Relayed | -| `baseline.stale` | A baseline became stale: the runtime, system image or emulator it was made with changed after the save. | `name`, `version`, `reason` | Relayed | +| `baseline.stale` | A baseline became stale: the runtime, system image or emulator it was made with changed, or an iOS baseline was saved in a way that cannot be copied safely and must be saved again. A check runs after a save too. | `name`, `version`, `reason` | Relayed | ## Workers From 36a7353f6616ce853daf0e9625c344704b68b369 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 22:25:22 +0100 Subject: [PATCH 44/48] fix: every WAL in a folder is flushed before it is listed again, since closing SQLite deletes the -shm; the fake does too; manuals fit their total (#1570) --- src/drivers/ios/index-seed-record.test.ts | 1 + src/drivers/ios/lineage-files.ts | 20 +++++++++----------- src/ports/sqlite.ts | 7 ++++++- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/drivers/ios/index-seed-record.test.ts b/src/drivers/ios/index-seed-record.test.ts index 03b6d9a9c..ee7e400a6 100644 --- a/src/drivers/ios/index-seed-record.test.ts +++ b/src/drivers/ios/index-seed-record.test.ts @@ -184,6 +184,7 @@ describe("IosSimctlDriver source records", () => { await plantBytes(filesystem, `${seedFolder}/data/none.txt`, Buffer.from("nothing")); await plantBytes(filesystem, `${seedFolder}/data/one.db`, SQLITE_HEADER); await plantBytes(filesystem, `${seedFolder}/data/one.db-wal`, naming); + await plantBytes(filesystem, `${seedFolder}/data/one.db-shm`, Buffer.from("index")); await plantBytes(filesystem, `${seedFolder}/data/two.db`, SQLITE_HEADER); await plantBytes(filesystem, `${seedFolder}/data/two.db-wal`, naming); filesystem.defineSymlink(`${seedFolder}/data/Logs`, `${logsRoot}/${lineageUdid}`); diff --git a/src/drivers/ios/lineage-files.ts b/src/drivers/ios/lineage-files.ts index 648dabce9..c7cf6b485 100644 --- a/src/drivers/ios/lineage-files.ts +++ b/src/drivers/ios/lineage-files.ts @@ -84,13 +84,14 @@ async function visitFolder( relative: string, files: SourceFile[], ): Promise { + // why: a database and its WAL are siblings, and a database is read only after its WAL is flushed into it (ADR 0076 §2); a flush closes SQLite, which deletes the `-shm`, so the folder is listed again once every WAL in it is flushed. + for (const name of (await options.filesystem.readdir(directory)).sort()) { + if (!name.endsWith(WAL_SUFFIX)) continue; + const path = `${directory}/${name}`; + if ((await options.filesystem.lstat(path)).kind === "file") await flushWal(options, path); + } const names = (await options.filesystem.readdir(directory)).sort(); - // why: a database and its WAL are siblings, and a database is read only after its WAL is flushed into it (ADR 0076 §2). - const ordered = [ - ...names.filter((name) => name.endsWith(WAL_SUFFIX)), - ...names.filter((name) => !name.endsWith(WAL_SUFFIX)), - ]; - for (const name of ordered) { + for (const name of names) { const path = `${directory}/${name}`; const relativePath = relative === "" ? name : `${relative}/${name}`; const details = await options.filesystem.lstat(path); @@ -103,7 +104,7 @@ async function visitFolder( } } -/** why: a link is recorded when its target names the lineage; a WAL is flushed and never recorded; any other file is recorded when its bytes name it or cannot be read. */ +/** why: a link is recorded when its target names the lineage; a WAL is never recorded; any other file is recorded when its bytes name it or cannot be read. */ async function recordedKind( options: RecordOptions, path: string, @@ -114,10 +115,7 @@ async function recordedKind( return namesLineage(Buffer.from(target), options.lineage) ? "link" : undefined; } if (entry !== "file") return undefined; - if (path.endsWith(WAL_SUFFIX)) { - await flushWal(options, path); - return undefined; - } + if (path.endsWith(WAL_SUFFIX)) return undefined; return classify(options, path); } diff --git a/src/ports/sqlite.ts b/src/ports/sqlite.ts index acc8b4f9d..38d641d3b 100644 --- a/src/ports/sqlite.ts +++ b/src/ports/sqlite.ts @@ -81,14 +81,19 @@ export class FakeSqliteOpener implements SqliteOpener { () => undefined, ); const size = frames === undefined ? 0 : (await this.#filesystem.stat(path)).size; + let emptied = false; const checkpoint = (): void => { this.checkpoints.push(path); if (this.#keepingFrames.has(path)) return; if (frames !== undefined) void this.#filesystem.writeAt(path, size, frames); void this.#filesystem.rm(wal); + emptied = true; }; return { - close: () => undefined, + // why: like SQLite, the last connection to close after a full checkpoint removes the `-shm` beside the emptied WAL. + close: () => { + if (emptied) void this.#filesystem.rm(`${path}-shm`); + }, exec: (sql) => { if (sql === "PRAGMA wal_checkpoint(TRUNCATE)") checkpoint(); }, From 07141482c76cbeae81d8e427347c760c8d8aac7d Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 22:25:49 +0100 Subject: [PATCH 45/48] docs: the iOS manual drops the lineage's contents so the manuals fit their total (#1570) --- src/drivers/ios/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/ios/AGENTS.md b/src/drivers/ios/AGENTS.md index f8b942d34..52a027220 100644 --- a/src/drivers/ios/AGENTS.md +++ b/src/drivers/ios/AGENTS.md @@ -29,7 +29,7 @@ Seeds (ADR 0061): - A simulator is a clone of its kind's seed. A seed is known by its `ios_source_data` row, never by name, and lives in the sources set: `sources` in the owned set, an owned root of its own under `default`. It is in no `listManaged` or `simlock simctl list`. - A seed is made from a throwaway (ADR 0076): `simlock-seed-` is booted once, shut down, cloned unbooted into the sources set, then deleted with its log folder; `saveSource` deletes the saved-from lease's log folder too. A log folder that cannot be deleted is logged and the make or save goes on; a throwaway that cannot be deleted fails the make. -- A source records its lineage (a seed's throwaway; a baseline's saved-from device, then that device's own) and, in `ios_source_files`, the files and links naming it, after flushing its WALs. A clone keeps its source's lineage and, before its first boot, has its `com.apple.LaunchServices-*.csstore` deleted and those files and links swapped to its own UDID; a failed drop or swap deletes it. No lineage: a seed is remade, a baseline stale. +- A source records its lineage and, in `ios_source_files`, the files and links naming it, after flushing its WALs. A clone keeps its source's lineage and, before its first boot, has its `com.apple.LaunchServices-*.csstore` deleted and those files and links swapped to its own UDID; a failed drop or swap deletes it. No lineage: a seed is remade, a baseline stale. - Two provisions of a kind make one seed; clones of a seed and its deletion take turns. - A `clone` or `create` waits `ios.simulator.createTimeoutMs`. At the limit it fails but `simctl` runs on: the UDID it later prints is deleted, the seed's deletion waits for it, and one silent 5 minutes more is killed and logged. - A `making` row of another process is swept at the first `listSources`, with its throwaway; one of this process, once older than any make takes, at the next make of its kind. From ba0e0759abd3e48c45c8167865043d108e5241c3 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 22:33:17 +0100 Subject: [PATCH 46/48] fix: a copy's swap skips the LaunchServices database it just dropped (#1570) --- src/drivers/ios/index-seed-swap.test.ts | 24 ++++++++++++++++++++++++ src/drivers/ios/seeds.ts | 6 +++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/drivers/ios/index-seed-swap.test.ts b/src/drivers/ios/index-seed-swap.test.ts index bb97b5b57..3acd83383 100644 --- a/src/drivers/ios/index-seed-swap.test.ts +++ b/src/drivers/ios/index-seed-swap.test.ts @@ -8,6 +8,7 @@ import { createDriver, listFixture, listInvocation, + plantLaunchServices, scopedRunners, seedUdid, seededDatabase, @@ -188,4 +189,27 @@ describe("IosSimctlDriver copy swap", () => { } } }); + + it("a LaunchServices database the seed recorded is dropped from the copy and is not swapped", async () => { + const csstore = "data/var/db/lsd/com.apple.LaunchServices-33554481-v2.csstore"; + const database = await seededDatabase({ + files: [{ kind: "file", relativePath: csstore }], + lineage: [lineageUdid], + }); + const runner = new ScriptedProcessRunner([ + { match: listInvocation, result: { code: 0, stderr: "", stdout: listFixture } }, + { + match: sourcesSimctl("clone", seedUdid, driverData.name, deviceRoot), + result: { code: 0, stderr: "", stdout: `${copyUdid}\n` }, + }, + ]); + const filesystem = new MemoryFilesystem(); + await plantManagedDevice(filesystem); + await plantLaunchServices(filesystem, `${deviceRoot}/${copyUdid}`); + const driver = await createDriver(runner, new FakeClock(), filesystem, database); + + await driver.provision(spec); + + expect(await filesystem.exists(`${deviceRoot}/${copyUdid}/${csstore}`)).toBe(false); + }); }); diff --git a/src/drivers/ios/seeds.ts b/src/drivers/ios/seeds.ts index 5ea753aa6..b29926dc1 100644 --- a/src/drivers/ios/seeds.ts +++ b/src/drivers/ios/seeds.ts @@ -39,6 +39,7 @@ import type { SqliteDatabase, SqliteOpener } from "../../ports/index.js"; const STALE_MAKING_MS = 300_000; const LAUNCH_SERVICES_DATABASE = /^com\.apple\.LaunchServices-.*\.csstore$/; +const LAUNCH_SERVICES_PATH = /^data\/var\/db\/lsd\/com\.apple\.LaunchServices-[^/]*\.csstore$/; interface SeedsOptions { readonly boot: SimulatorBoot; @@ -502,7 +503,10 @@ export class Seeds { const counts = await swapCopyFiles( this.#filesystem, join(this.#deviceRoot, udid), - this.#sources.filesOf(sourceId), + // why: the copy's LaunchServices database was dropped just before, so a seed that recorded one has nothing of it left to swap. + this.#sources + .filesOf(sourceId) + .filter((file) => !LAUNCH_SERVICES_PATH.test(file.relativePath)), { lineage, own: udid }, ); this.#logger?.debug( From 17f358dd91516bb493f47f7ed7337da939b57541 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 22:40:36 +0100 Subject: [PATCH 47/48] test: the isolation fingerprint takes an unreadable system cache file by its size and time (#1570) --- e2e/slow-ios-source-isolation.test.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/e2e/slow-ios-source-isolation.test.ts b/e2e/slow-ios-source-isolation.test.ts index f2b5aa101..a0a8d68b2 100644 --- a/e2e/slow-ios-source-isolation.test.ts +++ b/e2e/slow-ios-source-isolation.test.ts @@ -51,9 +51,14 @@ async function fingerprint(folder: string): Promise> { if (entry.isSymbolicLink()) result[path] = `link ${await readlink(path)}`; else if (entry.isFile()) { const { mtimeMs, size } = await stat(path); - const hash = createHash("sha256") - .update(await readFile(path)) - .digest("hex"); + // why: a system service's cache file inside a source can be unreadable to the test (EPERM); the product records it as unreadable too, so its size and time stand for it. + const hash = await readFile(path).then( + (bytes) => createHash("sha256").update(bytes).digest("hex"), + (error: NodeJS.ErrnoException) => { + if (error.code === "EPERM" || error.code === "EACCES") return "unreadable"; + throw error; + }, + ); result[path] = `${size} ${mtimeMs} ${hash}`; } } From 7fef0f856a56ef0640601ef841e534988b7ed412 Mon Sep 17 00:00:00 2001 From: Szymon Chmal Date: Mon, 16 Nov 2026 23:16:58 +0100 Subject: [PATCH 48/48] docs(adr): accept 0076 (#1570) --- ...os-clone-names-only-its-own-folders-before-its-first-boot.md | 2 +- docs/internal/adr/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internal/adr/0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md b/docs/internal/adr/0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md index 2a632bfef..f2006aa57 100644 --- a/docs/internal/adr/0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md +++ b/docs/internal/adr/0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md @@ -1,6 +1,6 @@ # 0076. An iOS clone names only its own folders before its first boot -- **Status:** Accepted — not yet implemented +- **Status:** Accepted - **Date:** 2026-11-15, revised 2026-11-16 - **Issue:** [#1561](https://github.com/callstackincubator/simlock/issues/1561), bug [#1548](https://github.com/callstackincubator/simlock/issues/1548) - **Narrows:** [ADR 0061](0061-every-virtual-device-is-a-copy-of-a-source-and-a-baseline-is-a-named-source.md) diff --git a/docs/internal/adr/README.md b/docs/internal/adr/README.md index aa0d55fae..b413bc3e1 100644 --- a/docs/internal/adr/README.md +++ b/docs/internal/adr/README.md @@ -110,7 +110,7 @@ the status is stale. | [0073](0073-a-worker-may-enrol-the-phones-plugged-into-it.md) | A worker may enrol the phones plugged into it | Accepted; narrows 0022 §3, §5 and §6 | | [0074](0074-an-admission-command-ranks-or-refuses-a-gateway-request-and-an-ended-chosen-lease-id-is-remembered.md) | An admission command ranks or refuses a gateway request, and an ended chosen lease id is remembered | Accepted; narrows 0005 items 10 and 11, decision 5 and requirements 14 and 30; supersedes 0020 §1 in part | | [0075](0075-a-foldable-simulator-streams-the-panel-in-use-and-a-stream-may-change-size.md) | A foldable simulator streams the panel in use, and a stream may change size while it runs | Accepted; narrows 0031's Consequences and 0052 §4 | -| [0076](0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md) | An iOS clone names only its own folders before its first boot | Accepted — not yet implemented; narrows 0061 §1, §2 and §4 | +| [0076](0076-an-ios-clone-names-only-its-own-folders-before-its-first-boot.md) | An iOS clone names only its own folders before its first boot | Accepted; narrows 0061 §1, §2 and §4 | | [0077](0077-the-repository-is-a-workspace-and-its-packages-release-together-under-one-version.md) | The repository is a workspace, and its packages release together under one version | Accepted | | [0078](0078-an-android-worker-tries-its-default-image-tag-first-and-falls-back-to-none.md) | An Android worker tries its default image tag first, and a lease falls back to none | Accepted | | [0079](0079-one-capture-per-device-is-shared-by-every-viewer-and-every-capture-sends-a-key-frame-every-5-s.md) | One capture per device is shared by every viewer, and every capture sends a key frame at least every 5 s | Accepted; narrows 0045 §5 |