From 00eeb4ddf20c8e75284bf1878bc8e38907692298 Mon Sep 17 00:00:00 2001 From: Peter Abbondanzo Date: Sun, 11 Oct 2026 22:47:52 -0700 Subject: [PATCH] Shard iOS RNTester E2E tests with isolated retries (#58992) Summary: Run the RNTester iOS suite in four independent flow shards for each of Debug and Release. Each runner owns its simulator; flows remain serial within a shard. Before every attempt, uninstall and reinstall the original app artifact, reset Keychain and app privacy permissions, and restore light appearance and portrait orientation. Retry only failed flows, continue collecting results after an exhausted flow, and retain separate reports and recordings for every attempt. Subprocess timeouts hard-kill hung commands before retry and cleanup. Allow up to ten minutes for cold simulator boot and data migration before installing the app or running flows. Remove RNTester's whole-workflow retries and use the matrix job result to propagate any shard failure. The shared action defaults keep Templateapp unsharded with its existing state behavior. Changelog: [Internal] Differential Revision: D124671885 --- .github/actions/maestro-ios/action.yml | 35 +- .../__tests__/maestro-ios-test.js | 402 +++++++++++++++--- .github/workflow-scripts/maestro-ios.js | 251 ++++++++--- .github/workflows/e2e-ios-rntester.yml | 14 +- .github/workflows/test-all.yml | 12 - 5 files changed, 561 insertions(+), 153 deletions(-) diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index b5e4036d6280..b85b483a2f1a 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -26,6 +26,18 @@ inputs: required: false default: '' description: Maestro device OS name, such as iOS-26-2 + shard-index: + required: false + default: '0' + description: Zero-based index of the flow shard to run + shard-count: + required: false + default: '1' + description: Number of flow shards + reset-app: + required: false + default: 'false' + description: Whether to reset the app before each flow attempt runs: using: composite @@ -60,6 +72,10 @@ runs: - name: Run tests id: run-tests shell: bash + env: + MAESTRO_SHARD_INDEX: ${{ inputs.shard-index }} + MAESTRO_SHARD_COUNT: ${{ inputs.shard-count }} + MAESTRO_RESET_APP: ${{ inputs.reset-app }} run: | # Avoid exit from the job if one of the command returns an error. # Maestro can fail in case of flakyness, we have some retry logic. @@ -74,23 +90,10 @@ runs: "${{ inputs.working-directory }}" \ "${{ inputs.device-model }}" \ "${{ inputs.device-os }}" - - name: Store video record + - name: Store Maestro reports if: always() uses: ./.github/actions/upload-artifact with: - name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch - overwrite: true - path: | - video_record_1.mov - video_record_2.mov - video_record_3.mov - video_record_4.mov - video_record_5.mov - report.xml - - name: Store Logs - if: failure() && steps.run-tests.outcome == 'failure' - uses: ./.github/actions/upload-artifact - with: - name: maestro-logs-${{ inputs.app-id }}-${{ inputs.flavor }}-NewArch + name: e2e_ios_${{ inputs.app-id }}_report_${{ inputs.flavor }}_NewArch_shard_${{ inputs.shard-index }}_attempt_${{ github.run_attempt }} overwrite: true - path: /tmp/MaestroLogs + path: maestro-reports diff --git a/.github/workflow-scripts/__tests__/maestro-ios-test.js b/.github/workflow-scripts/__tests__/maestro-ios-test.js index f5f9686e4e09..7fdf2ecc1fa1 100644 --- a/.github/workflow-scripts/__tests__/maestro-ios-test.js +++ b/.github/workflow-scripts/__tests__/maestro-ios-test.js @@ -9,23 +9,44 @@ jest.mock('child_process', () => ({ execSync: jest.fn(), + execFileSync: jest.fn(), spawn: jest.fn(), })); jest.mock('fs', () => ({ existsSync: jest.fn(), lstatSync: jest.fn(), readdirSync: jest.fn(), + mkdirSync: jest.fn(), + writeFileSync: jest.fn(), })); const childProcess = require('child_process'); const {EventEmitter} = require('events'); const fs = require('fs'); +const path = require('path'); -const {executeFlows, findAvailableSimulator} = require('../maestro-ios'); +const {executeFlows, findAvailableSimulator, main} = require('../maestro-ios'); describe('Maestro iOS runner', () => { + function flowCalls() { + return childProcess.execFileSync.mock.calls.filter( + ([file]) => file !== 'xcrun', + ); + } + + function mockFlows(files) { + fs.existsSync.mockReturnValue(true); + fs.lstatSync.mockImplementation(file => ({ + isDirectory: () => file === 'flows/', + })); + fs.readdirSync.mockReturnValue(files); + } + beforeEach(() => { jest.resetAllMocks(); + jest.spyOn(console, 'log').mockImplementation(() => {}); + jest.spyOn(console, 'info').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); childProcess.spawn.mockImplementation(() => { const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; @@ -37,45 +58,139 @@ describe('Maestro iOS runner', () => { }); }); + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('waits for cold boot before installing, launching, and running flows', async () => { + jest.replaceProperty(process, 'env', { + ...process.env, + MAESTRO_RESET_APP: 'true', + MAESTRO_SHARD_INDEX: '0', + MAESTRO_SHARD_COUNT: '1', + }); + childProcess.execSync.mockReturnValueOnce( + JSON.stringify({ + devices: { + 'com.apple.CoreSimulator.SimRuntime.iOS-26-2': [ + {name: 'iPhone 17 Pro', udid: 'device-id'}, + ], + }, + }), + ); + childProcess.execFileSync.mockImplementation((file, args, options) => { + if (args[1] === 'bootstatus' && options.timeout < 3 * 60 * 1000) { + throw new Error('Cold simulator data migration exceeded the deadline'); + } + }); + + await main([ + '/tmp/RNTester.app', + 'com.example', + 'flow.yml', + 'Hermes', + 'Release', + '/tmp/workspace', + 'iPhone-17-Pro', + 'iOS-26-2', + ]); + + const calls = childProcess.execFileSync.mock.calls; + expect(calls[0]).toEqual([ + 'xcrun', + ['simctl', 'bootstatus', 'device-id', '-b'], + {stdio: 'inherit', timeout: 10 * 60 * 1000, killSignal: 'SIGKILL'}, + ]); + expect(calls.slice(1, -1).map(([, args]) => args)).toEqual([ + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'launch', 'device-id', 'com.example'], + ['simctl', 'uninstall', 'device-id', 'com.example'], + ['simctl', 'keychain', 'device-id', 'reset'], + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'privacy', 'device-id', 'reset', 'all', 'com.example'], + ['simctl', 'ui', 'device-id', 'appearance', 'light'], + ]); + expect(childProcess.execSync).toHaveBeenLastCalledWith('open -a simulator'); + expect(flowCalls()).toHaveLength(1); + expect(flowCalls()[0][1]).toContain('APP_ID=com.example'); + expect(console.log).toHaveBeenLastCalledWith('Test finished'); + }); + + it('hard-kills a hung boot without installing the app or running flows', async () => { + const actualChildProcess = jest.requireActual('child_process'); + childProcess.execSync.mockReturnValueOnce( + JSON.stringify({ + devices: { + 'iOS 26.2': [{name: 'iPhone 17 Pro', udid: 'device-id'}], + }, + }), + ); + let timeoutError; + childProcess.execFileSync.mockImplementationOnce((file, args, options) => { + expect(args).toEqual(['simctl', 'bootstatus', 'device-id', '-b']); + expect(options.timeout).toBe(10 * 60 * 1000); + expect(options.killSignal).toBe('SIGKILL'); + try { + actualChildProcess.execFileSync( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stderr.write("ready"); setInterval(() => {}, 1000);', + ], + {...options, timeout: 500, stdio: 'pipe'}, + ); + } catch (error) { + timeoutError = error; + throw error; + } + }); + + await expect( + main([ + '/tmp/RNTester.app', + 'com.example', + 'flow.yml', + 'Hermes', + 'Release', + '/tmp/workspace', + ]), + ).rejects.toMatchObject({code: 'ETIMEDOUT', signal: 'SIGKILL'}); + + expect(String(timeoutError.stderr)).toContain('ready'); + expect(childProcess.execFileSync).toHaveBeenCalledTimes(1); + expect(childProcess.execSync).toHaveBeenCalledTimes(1); + expect(childProcess.spawn).not.toHaveBeenCalled(); + expect(fs.writeFileSync).not.toHaveBeenCalled(); + }); + it('executes each YAML flow separately and skips other files', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['second.yaml', 'image.png', 'first.yml']); + mockFlows(['second.yaml', 'image.png', 'first.yml']); await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/first.yml"', - ); - expect(childProcess.execSync.mock.calls[1][0]).toContain( - 'test "flows/second.yaml"', - ); + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + 'flows/first.yml', + 'flows/second.yaml', + ]); }); it('retries only the failing flow', async () => { - fs.existsSync.mockReturnValue(false); - childProcess.execSync.mockImplementationOnce(() => { + mockFlows(['first.yml', 'second.yml']); + childProcess.execFileSync.mockImplementationOnce(() => { throw new Error('Maestro driver failed'); }); - await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); - expect(childProcess.execSync).toHaveBeenCalledTimes(2); - for (const call of childProcess.execSync.mock.calls) { - expect(call[0]).toContain('test "flow.yml"'); - } + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + 'flows/first.yml', + 'flows/first.yml', + 'flows/second.yml', + ]); }); it('waits for the recorder to exit before starting the next flow', async () => { - fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => path === 'flows/', - })); - fs.readdirSync.mockReturnValue(['first.yml', 'second.yml']); - + mockFlows(['first.yml', 'second.yml']); const recordingProcess = new EventEmitter(); recordingProcess.pid = 1; recordingProcess.kill = jest.fn(() => true); @@ -87,61 +202,249 @@ describe('Maestro iOS runner', () => { 'flows/', 'Hermes', ); - await new Promise(resolve => jest.requireActual('timers').setImmediate(resolve), ); expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); + expect(flowCalls()).toHaveLength(1); expect(childProcess.spawn).toHaveBeenCalledTimes(1); recordingProcess.emit('exit', 0, null); await execution; - expect(childProcess.execSync).toHaveBeenCalledTimes(2); + expect(flowCalls()).toHaveLength(2); expect(childProcess.spawn).toHaveBeenCalledTimes(2); }); it('skips helper directories while recursing into flow directories', async () => { fs.existsSync.mockReturnValue(true); - fs.lstatSync.mockImplementation(path => ({ - isDirectory: () => !path.endsWith('.yml'), + fs.lstatSync.mockImplementation(file => ({ + isDirectory: () => !file.endsWith('.yml'), })); - fs.readdirSync.mockImplementation(path => - path === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], + fs.readdirSync.mockImplementation(directory => + directory === 'flows/' ? ['helpers', 'nested'] : ['flow.yml'], ); await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); expect(fs.readdirSync).not.toHaveBeenCalledWith('flows/helpers'); - expect(childProcess.execSync).toHaveBeenCalledTimes(1); - expect(childProcess.execSync.mock.calls[0][0]).toContain( - 'test "flows/nested/flow.yml"', + expect(flowCalls()).toHaveLength(1); + expect(flowCalls()[0][1][3]).toBe('flows/nested/flow.yml'); + }); + + it('partitions the sorted flow list without omissions or duplicates', async () => { + const files = ['e.yml', 'd.yaml', 'c.yml', 'b.yml', 'a.yml']; + const partitions = []; + for (let shardIndex = 0; shardIndex < 4; shardIndex++) { + mockFlows(files); + childProcess.execFileSync.mockClear(); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + shardIndex, + shardCount: 4, + }); + partitions.push(flowCalls().map(([, args]) => args[3])); + } + + expect(partitions).toEqual([ + ['flows/a.yml', 'flows/e.yml'], + ['flows/b.yml'], + ['flows/c.yml'], + ['flows/d.yaml'], + ]); + expect(new Set(partitions.flat()).size).toBe(files.length); + }); + + it.each([ + {shardIndex: -1, shardCount: 4}, + {shardIndex: 4, shardCount: 4}, + {shardIndex: 0, shardCount: 0}, + {shardIndex: 0.5, shardCount: 4}, + {shardIndex: 0, shardCount: NaN}, + ])('rejects invalid sharding options: %p', async options => { + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes', options), + ).rejects.toThrow('Invalid shard'); + expect(childProcess.execFileSync).not.toHaveBeenCalled(); + }); + + it('fails an empty suite instead of passing without coverage', async () => { + mockFlows(['readme.txt']); + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes'), + ).rejects.toThrow('No Maestro flows'); + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + + it('allows empty shards when there are fewer flows than shards', async () => { + mockFlows(['first.yml']); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + shardIndex: 3, + shardCount: 4, + }); + expect(childProcess.spawn).not.toHaveBeenCalled(); + expect(JSON.parse(fs.writeFileSync.mock.calls[0][1])).toEqual({ + shardIndex: 3, + shardCount: 4, + flows: [], + }); + }); + + it('resets before every retry and every subsequent flow', async () => { + mockFlows(['first.yml', 'second.yml']); + let attempts = 0; + childProcess.execFileSync.mockImplementation(file => { + if (file !== 'xcrun' && attempts++ === 0) { + throw new Error('Flow left the app dirty'); + } + }); + + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes', { + resetApp: true, + appPath: '/tmp/RNTester.app', + }); + + const resetCommands = [ + ['simctl', 'uninstall', 'device-id', 'com.example'], + ['simctl', 'keychain', 'device-id', 'reset'], + ['simctl', 'install', 'device-id', '/tmp/RNTester.app'], + ['simctl', 'privacy', 'device-id', 'reset', 'all', 'com.example'], + ['simctl', 'ui', 'device-id', 'appearance', 'light'], + ]; + const calls = childProcess.execFileSync.mock.calls; + expect(calls).toHaveLength(18); + for (let attempt = 0; attempt < 3; attempt++) { + expect( + calls.slice(attempt * 6, attempt * 6 + 5).map(([, args]) => args), + ).toEqual(resetCommands); + expect(calls[attempt * 6 + 5][0]).not.toBe('xcrun'); + } + const wrappers = fs.writeFileSync.mock.calls.filter(([file]) => + file.endsWith('/flow.yml'), + ); + expect(wrappers).toHaveLength(3); + for (const [, content] of wrappers) { + expect(content).toContain( + '- setOrientation: portrait\n- launchApp\n- runFlow:', + ); + expect(content).not.toContain('clearState'); + } + expect(wrappers[0][1]).toContain( + JSON.stringify(path.resolve('flows/first.yml')), + ); + expect(wrappers[2][1]).toContain( + JSON.stringify(path.resolve('flows/second.yml')), ); }); - it('rejects after exhausting retries and stops every recorder', async () => { - const consoleError = jest - .spyOn(console, 'error') - .mockImplementation(() => {}); + it('does not execute a flow when reset fails', async () => { fs.existsSync.mockReturnValue(false); - const error = new Error('Maestro driver failed'); - childProcess.execSync.mockImplementation(() => { + const error = new Error('Unable to uninstall app'); + childProcess.execFileSync.mockImplementation(() => { throw error; }); await expect( - executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'), - ).rejects.toBe(error); + executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes', { + resetApp: true, + appPath: '/tmp/RNTester.app', + }), + ).rejects.toMatchObject({errors: [error]}); + + expect(childProcess.execFileSync).toHaveBeenCalledTimes(5); + expect(flowCalls()).toHaveLength(0); + expect(childProcess.spawn).not.toHaveBeenCalled(); + }); + + it('requires the original app artifact for reset', async () => { + await expect( + executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes', { + resetApp: true, + }), + ).rejects.toThrow('requires an app path'); + expect(childProcess.execFileSync).not.toHaveBeenCalled(); + }); + + it('keeps reports, logs, screenshots, and videos unique for every attempt', async () => { + mockFlows(['first.yml', 'second.yml']); + childProcess.execFileSync.mockImplementationOnce(() => { + throw new Error('Retry'); + }); + await executeFlows('com.example', 'device-id', 'flows/', 'Hermes'); + + for (const flag of ['--output', '--debug-output', '--test-output-dir']) { + const outputs = flowCalls().map( + ([, args]) => args[args.indexOf(flag) + 1], + ); + expect(new Set(outputs).size).toBe(3); + expect(outputs.every(output => output.includes('maestro-reports'))).toBe( + true, + ); + } + const videos = childProcess.spawn.mock.calls.map(([, args]) => args.at(-1)); + expect(new Set(videos).size).toBe(3); + expect(videos[0]).toContain('0-first.yml/attempt-1/video.mov'); + expect(videos[1]).toContain('0-first.yml/attempt-2/video.mov'); + expect(videos[2]).toContain('1-second.yml/attempt-1/video.mov'); + for (const [, args] of flowCalls()) { + expect(args).toContain('APP_ID=com.example'); + } + }); + + it('hard-kills a hung flow and retries after stopping its recorder', async () => { + const actualChildProcess = jest.requireActual('child_process'); + let timeoutError; + childProcess.execFileSync.mockImplementationOnce((file, args, options) => { + expect(options.killSignal).toBe('SIGKILL'); + try { + actualChildProcess.execFileSync( + process.execPath, + [ + '-e', + 'process.on("SIGTERM", () => {}); process.stderr.write("ready"); setInterval(() => {}, 1000);', + ], + {...options, timeout: 500, stdio: 'pipe'}, + ); + } catch (error) { + timeoutError = error; + throw error; + } + }); + + await executeFlows('com.example', 'device-id', 'flow.yml', 'Hermes'); + + expect(timeoutError).toMatchObject({code: 'ETIMEDOUT', signal: 'SIGKILL'}); + expect(String(timeoutError.stderr)).toContain('ready'); + expect(flowCalls()).toHaveLength(2); + expect(childProcess.spawn).toHaveBeenCalledTimes(2); + for (const {value: recordingProcess} of childProcess.spawn.mock.results) { + expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); + } + }); + + it('continues after an exhausted flow and reports the failure at the end', async () => { + mockFlows(['first.yml', 'second.yml']); + const error = new Error('Maestro driver failed'); + childProcess.execFileSync.mockImplementation((file, args) => { + if (args[3] === 'flows/first.yml') { + throw error; + } + }); + + await expect( + executeFlows('com.example', 'device-id', 'flows/', 'Hermes'), + ).rejects.toMatchObject({errors: [error]}); - expect(childProcess.execSync).toHaveBeenCalledTimes(5); - expect(childProcess.spawn).toHaveBeenCalledTimes(5); + expect(flowCalls().map(([, args]) => args[3])).toEqual([ + ...Array(5).fill('flows/first.yml'), + 'flows/second.yml', + ]); + expect(childProcess.spawn).toHaveBeenCalledTimes(6); for (const {value: recordingProcess} of childProcess.spawn.mock.results) { expect(recordingProcess.kill).toHaveBeenCalledWith('SIGINT'); } - expect(consoleError).toHaveBeenCalledWith( - 'Failed to execute flow flow.yml after 5 attempts.', + expect(console.error).toHaveBeenCalledWith( + 'Failed to execute flow flows/first.yml after 5 attempts.', ); }); @@ -157,7 +460,6 @@ describe('Maestro iOS runner', () => { }, }), ); - expect(findAvailableSimulator()).toEqual({ name: 'iPhone 17 Pro', udid: 'new-pro', @@ -178,7 +480,6 @@ describe('Maestro iOS runner', () => { }, }), ); - expect(findAvailableSimulator('iPhone-17-Pro', 'iOS-26-2')).toEqual({ name: 'iPhone 17 Pro', udid: 'expected', @@ -187,7 +488,6 @@ describe('Maestro iOS runner', () => { it('fails when the configured simulator is unavailable', () => { childProcess.execSync.mockReturnValue(JSON.stringify({devices: {}})); - expect(() => findAvailableSimulator('iPhone-17-Pro', 'iOS-26-2')).toThrow( 'Unable to find iPhone 17 Pro simulator on iOS-26-2', ); diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index a9782ec6506f..838afdbdd865 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -9,6 +9,7 @@ const childProcess = require('child_process'); const fs = require('fs'); +const path = require('path'); const usage = ` === Usage === @@ -65,20 +66,24 @@ function findAvailableSimulator(deviceModel, deviceOS) { function launchSimulator(simulator) { console.log(`Launching simulator ${simulator.name} (${simulator.udid})`); - try { - childProcess.execSync(`xcrun simctl boot "${simulator.udid}"`); - } catch (error) { - if ( - !error.message.includes('Unable to boot device in current state: Booted') - ) { - throw error; - } - } + childProcess.execFileSync( + 'xcrun', + ['simctl', 'bootstatus', simulator.udid, '-b'], + { + stdio: 'inherit', + timeout: 10 * 60 * 1000, + killSignal: 'SIGKILL', + }, + ); } function installAppOnSimulator(appPath, udid) { console.log(`Installing app at path ${appPath}`); - childProcess.execSync(`xcrun simctl install "${udid}" "${appPath}"`); + childProcess.execFileSync('xcrun', ['simctl', 'install', udid, appPath], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); } function bringSimulatorInForeground() { @@ -94,7 +99,11 @@ function sleep(ms) { async function launchAppOnSimulator(appId, udid, isDebug) { console.log('Launch the app'); - childProcess.execSync(`xcrun simctl launch "${udid}" "${appId}"`); + childProcess.execFileSync('xcrun', ['simctl', 'launch', udid, appId], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); if (isDebug) { console.log('Wait for metro to warm'); @@ -102,19 +111,17 @@ async function launchAppOnSimulator(appId, udid, isDebug) { } } -function startVideoRecording(udid, currentAttempt) { - console.log( - `Start video record using pid: video_record_${currentAttempt}.pid`, - ); +function startVideoRecording(udid, outputPath) { + console.log(`Start video recording: ${outputPath}`); - const recordingArgs = - `simctl io ${udid} recordVideo --force video_record_${currentAttempt}.mov`.split( - ' ', - ); - const recordingProcess = childProcess.spawn('xcrun', recordingArgs, { - detached: true, - stdio: 'ignore', - }); + const recordingProcess = childProcess.spawn( + 'xcrun', + ['simctl', 'io', udid, 'recordVideo', '--force', outputPath], + { + detached: true, + stdio: 'ignore', + }, + ); return recordingProcess; } @@ -179,60 +186,167 @@ async function executeFlowWithRetries( appId, udid, flow, - jsengine, - currentAttempt, + reportDirectory, + options, ) { - const recProcess = startVideoRecording(udid, currentAttempt); - try { - const timeout = 1000 * 60 * 10; // 10 minutes - const command = `$HOME/.maestro/bin/maestro --udid="${udid}" test "${flow}" --format junit -e APP_ID="${appId}"`; - console.info(`Executing flow: ${flow} (attempt ${currentAttempt})`); - console.log(command); - childProcess.execSync(`MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 ${command}`, { - stdio: 'inherit', - timeout, - }); - - await stopVideoRecording(recProcess); - } catch (error) { - await stopVideoRecording(recProcess); + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + const attemptDirectory = path.resolve( + reportDirectory, + `attempt-${attempt}`, + ); + fs.mkdirSync(attemptDirectory, {recursive: true}); + let recordingProcess; + + try { + let flowPath = flow; + if (options.resetApp) { + // Uninstall first so the old process cannot repopulate the keychain. + for (const args of [ + ['uninstall', udid, appId], + ['keychain', udid, 'reset'], + ['install', udid, options.appPath], + ['privacy', udid, 'reset', 'all', appId], + ['ui', udid, 'appearance', 'light'], + ]) { + childProcess.execFileSync('xcrun', ['simctl', ...args], { + stdio: 'inherit', + timeout: 60 * 1000, + killSignal: 'SIGKILL', + }); + } + + flowPath = path.join(attemptDirectory, 'flow.yml'); + fs.writeFileSync( + flowPath, + `appId: ${JSON.stringify(appId)} +--- +- setOrientation: portrait +- launchApp +- runFlow: ${JSON.stringify(path.resolve(flow))} +`, + ); + } - if (currentAttempt < MAX_ATTEMPTS) { - console.info(`Retrying flow: ${flow}`); - await executeFlowWithRetries( - appId, + recordingProcess = startVideoRecording( udid, - flow, - jsengine, - currentAttempt + 1, + path.join(attemptDirectory, 'video.mov'), ); - } else { - console.error( - `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, + console.info(`Executing flow: ${flow} (attempt ${attempt})`); + childProcess.execFileSync( + path.join(process.env.HOME, '.maestro', 'bin', 'maestro'), + [ + '--udid', + udid, + 'test', + flowPath, + '--format', + 'junit', + '--output', + path.join(attemptDirectory, 'report.xml'), + '--debug-output', + path.join(attemptDirectory, 'logs'), + '--test-output-dir', + attemptDirectory, + '-e', + `APP_ID=${appId}`, + ], + { + env: {...process.env, MAESTRO_DRIVER_STARTUP_TIMEOUT: '1500000'}, + stdio: 'inherit', + timeout: 1000 * 60 * 10, + killSignal: 'SIGKILL', + }, ); - throw error; + return; + } catch (error) { + fs.writeFileSync(path.join(attemptDirectory, 'error.txt'), String(error)); + if (attempt === MAX_ATTEMPTS) { + console.error( + `Failed to execute flow ${flow} after ${MAX_ATTEMPTS} attempts.`, + ); + throw error; + } + console.info(`Retrying flow: ${flow}`); + } finally { + await stopVideoRecording(recordingProcess); } } } -async function executeFlows(appId, udid, maestroFlow, jsengine) { - if (!fs.existsSync(maestroFlow) || !fs.lstatSync(maestroFlow).isDirectory()) { - await executeFlowWithRetries(appId, udid, maestroFlow, jsengine, 1); - return; +async function executeFlows(appId, udid, maestroFlow, jsengine, options = {}) { + const { + shardIndex = 0, + shardCount = 1, + reportDirectory = 'maestro-reports', + } = options; + if ( + !Number.isInteger(shardCount) || + shardCount < 1 || + !Number.isInteger(shardIndex) || + shardIndex < 0 || + shardIndex >= shardCount + ) { + throw new Error(`Invalid shard ${shardIndex} of ${shardCount}`); + } + if (options.resetApp && !options.appPath) { + throw new Error('Resetting app state requires an app path'); } - for (const file of fs.readdirSync(maestroFlow).sort()) { - const filePath = `${maestroFlow.replace(/\/$/, '')}/${file}`; - if (fs.lstatSync(filePath).isDirectory()) { - // Fragments pulled in via `runFlow`; they have no `launchApp` of their - // own and fail when run standalone. - if (file === 'helpers') { - continue; + function collectFlows(directory) { + if (!fs.existsSync(directory) || !fs.lstatSync(directory).isDirectory()) { + return [directory]; + } + const flows = []; + for (const file of fs.readdirSync(directory).sort()) { + const filePath = path.join(directory, file); + if (fs.lstatSync(filePath).isDirectory()) { + // Helper fragments are only runnable via their parent flow. + if (file !== 'helpers') { + flows.push(...collectFlows(filePath)); + } + } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { + flows.push(filePath); } - await executeFlows(appId, udid, filePath, jsengine); - } else if (file.endsWith('.yml') || file.endsWith('.yaml')) { - await executeFlowWithRetries(appId, udid, filePath, jsengine, 1); } + return flows; + } + + const flows = collectFlows(maestroFlow); + if (flows.length === 0) { + throw new Error(`No Maestro flows found at ${maestroFlow}`); + } + const selectedFlows = flows.filter( + (_, index) => index % shardCount === shardIndex, + ); + fs.mkdirSync(reportDirectory, {recursive: true}); + fs.writeFileSync( + path.join(reportDirectory, 'manifest.json'), + JSON.stringify({shardIndex, shardCount, flows: selectedFlows}, null, 2), + ); + console.info( + `Shard ${shardIndex + 1}/${shardCount}: ${selectedFlows.length}/${flows.length} flows`, + ); + + const failures = []; + for (let index = 0; index < selectedFlows.length; index++) { + const flow = selectedFlows[index]; + try { + await executeFlowWithRetries( + appId, + udid, + flow, + path.join(reportDirectory, `${index}-${path.basename(flow)}`), + options, + ); + } catch (error) { + failures.push(error); + } + } + if (failures.length > 0) { + throw new AggregateError( + failures, + `${failures.length} Maestro flow(s) failed`, + ); } } @@ -249,6 +363,12 @@ async function main(args = process.argv.slice(2)) { const workingDirectory = args[5]; const deviceModel = args[6] || null; const deviceOS = args[7] || null; + const options = { + appPath, + resetApp: process.env.MAESTRO_RESET_APP === 'true', + shardIndex: Number(process.env.MAESTRO_SHARD_INDEX ?? 0), + shardCount: Number(process.env.MAESTRO_SHARD_COUNT ?? 1), + }; console.info('\n=============================='); console.info('Running tests for iOS with the following parameters:'); @@ -267,7 +387,7 @@ async function main(args = process.argv.slice(2)) { installAppOnSimulator(appPath, simulator.udid); bringSimulatorInForeground(); await launchAppOnSimulator(appId, simulator.udid, isDebug); - await executeFlows(appId, simulator.udid, maestroFlow, jsengine); + await executeFlows(appId, simulator.udid, maestroFlow, jsengine, options); console.log('Test finished'); } @@ -278,4 +398,5 @@ if (require.main === module) { module.exports = { executeFlows, findAvailableSimulator, + main, }; diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index 0660257487bf..895e771e1790 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -17,12 +17,11 @@ on: jobs: test: runs-on: macos-26-large - outputs: - status: ${{ steps.report-status.outputs.status }} strategy: fail-fast: false matrix: flavor: [Debug, Release] + shard: [0, 1, 2, 3] steps: - name: Checkout uses: actions/checkout@v6 @@ -37,7 +36,6 @@ jobs: run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests - continue-on-error: true uses: ./.github/actions/maestro-ios with: app-path: '/tmp/RNTesterBuild/RNTester.app' @@ -46,10 +44,9 @@ jobs: flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 - - name: Report status - id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT + shard-index: ${{ matrix.shard }} + shard-count: '4' + reset-app: 'true' report: needs: test @@ -60,8 +57,7 @@ jobs: steps: - id: check run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then + if [[ "${{ needs.test.result }}" != "success" ]]; then echo "status=failure" >> $GITHUB_OUTPUT else echo "status=success" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index afa299e09a1c..d04ffc9fb4ac 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -193,18 +193,6 @@ jobs: needs: test_ios_rntester if: ${{ needs.test_ios_rntester.result == 'success' }} uses: ./.github/workflows/e2e-ios-rntester.yml - secrets: inherit - - test_e2e_ios_rntester_retry_1: - needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} - uses: ./.github/workflows/e2e-ios-rntester.yml - secrets: inherit - - test_e2e_ios_rntester_retry_2: - needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} - uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true secrets: inherit