e2e: device coverage for the first beta

examples/bare/App.tsx

@@ -22,7 +22,8 @@
  * `breadcrumb-filter` is scenarios/breadcrumb-filter.ts,
  * `network-filter` is scenarios/network-filter.ts,
  * `add-network-event` is scenarios/add-network-event.ts,
- * `feedback` is scenarios/feedback.ts, and `apm` is scenarios/apm.ts.
+ * `feedback` is scenarios/feedback.ts, `apm` is scenarios/apm.ts, and
+ * `cov-*` / `opt-<case>` are scenarios/coverage.ts.
  */
 import { useEffect, useState } from 'react';
 import {
@@ -115,6 +116,12 @@ import {
 import { isFeedbackScenario, runFeedbackScenario } from './scenarios/feedback';
 import { isApmScenario, runApmScenario } from './scenarios/apm';
 import { isAppearanceScenario, runAppearanceScenario } from './scenarios/appearance';
+import {
+  isCoverageScenario,
+  launchOverrides,
+  preLaunchCoverage,
+  runCoverageScenario,
+} from './scenarios/coverage';
 
 const STATUS_NAMES: Record<number, string> = {
   [Status.Stopped]: 'Stopped',
@@ -147,6 +154,10 @@ function launchOptions(endpoint: string, scenario: string): LaunchOptions {
   if (isBreadcrumbFilterScenario(scenario)) {
     options.captureBreadcrumbs = true;
   }
+  // The `opt-<case>` scenarios (scenarios/coverage.ts): each case's option
+  // set, through setCustomOption -- the typed model's escape hatch, which
+  // writes the same native payload a first-class accessor does.
+  launchOverrides(scenario, (key, value) => options.setCustomOption(key, value));
   const serialized = BugseeLaunchOptions.serialize(options) as LaunchOptions;
   // This scenario only. APM is on by default; setting the key makes the
   // launch explicit about the capture this scenario is here to produce.
@@ -370,7 +381,7 @@ export default function App() {
       // Before launch(): a crash recovered at the next launch only reaches a
       // handler that is already registered (Task 7.6b).
       if (isNativeScenario(choice.scenario)) {
-        installNativeHandler(choice.scenario);
+        installNativeHandler(choice.scenario, choice.nonce);
       }
       // Before launch(): app ErrorUtils handler (so Bugsee chains to it), and
       // the pre-launch exception probe (must not reach a bundle).
@@ -393,6 +404,11 @@ export default function App() {
       if (isAddNetworkEventScenario(choice.scenario)) {
         installAddNetworkEventFilter(choice.nonce);
       }
+      // Before launch(): cov-lifecycle's onLifecycleEvent/onStatusChange
+      // subscriptions, so Launching and Launched are seen.
+      if (isCoverageScenario(choice.scenario)) {
+        preLaunchCoverage(choice.scenario, choice.nonce);
+      }
       console.log(`BUGSEE_E2E launching on ${Platform.OS}`);
       // Polling starts before launch() is awaited, not after. The SDK brings
       // capture up off the main thread, so by the time the promise resolves the
@@ -493,7 +509,7 @@ export default function App() {
         }
 
         if (isFeedbackScenario(choice.scenario)) {
-          runFeedbackScenario(choice.nonce);
+          runFeedbackScenario(choice.scenario, choice.nonce);
           return;
         }
 
@@ -502,6 +518,11 @@ export default function App() {
           return;
         }
 
+        if (isCoverageScenario(choice.scenario)) {
+          await runCoverageScenario(choice.scenario, choice.nonce);
+          return;
+        }
+
         if (isAppearanceScenario(choice.scenario)) {
           await runAppearanceScenario(choice.scenario, choice.nonce);
           return;

examples/bare/e2e/feedback.test.ts

@@ -0,0 +1,135 @@
+/**
+ * FB-01, FB-02, FB-04, FB-05: `@bugsee/react-native-feedback` on a device.
+ *
+ * Drives scenarios/feedback.ts. Both scenarios set a greeting carrying the
+ * nonce (`hello <nonce>`), a chat background of #112233, a listener, and
+ * call `showFeedbackUI()`:
+ *   feedback       as the app ships it. The first time ever, the SDKs may
+ *                  open on a "notify me by e-mail" screen.
+ *   feedback-chat  first sets an e-mail-shaped user identifier, which both
+ *                  SDKs take as the chat's e-mail, so the chat itself opens
+ *                  with no tap.
+ *
+ * What is read, and from where -- nothing here taps the screen:
+ *   - Android: the top-resumed activity (`dumpsys activity`) is the SDK's
+ *     FeedbackActivity, in this app's task; a screen capture.
+ *   - iOS: a screen capture (simctl / devicectl), read by macOS Vision
+ *     (e2e/ocr.swift) -- iOS has no accessibility dump the e2e can read.
+ *
+ * Needs a human (MANUAL, see the beta-coverage report): sending a message
+ * and receiving a reply (setListener's two callbacks, FB-03), which also
+ * needs a real backend.
+ */
+import { colourOf, colourPixels } from './media';
+import { ON_ANDROID, ON_IOS, type Run, TARGET_NAME, describeDevice, must, report, startRun } from './harness';
+import { androidTopActivity, beginRetainingSuite, captureScreen, endRetainingSuite, ocrLines } from './observe';
+import { type DeviceLog, adbStatus } from './scenario';
+
+jest.setTimeout(6 * 60_000);
+
+/** Folds the characters OCR confuses (1/l/I/|, 0/O/o) together. */
+function ocrFold(text: string): string {
+  return text.replace(/[lI|]/g, '1').replace(/[Oo]/g, '0');
+}
+
+const FEEDBACK_ACTIVITY = 'com.bareexample/com.bugsee.library.feedback.FeedbackActivity';
+/** scenarios/feedback.ts `appearance.backgroundColor`. */
+const CHAT_BACKGROUND = '#112233';
+
+/**
+ * The iOS SwiftUI chat (7.0.0-beta3 and beta4) ignores BugseeTheme, a
+ * documented limitation (packages/react-native-feedback/README.md): seen
+ * white (#ffffff) on the simulator with #112233 set. Pinned until it does.
+ */
+const itColour = ON_IOS ? it.failing : it;
+/**
+ * Android 7.3.0 stores the greeting (`setDefaultFeedbackGreeting` writes the
+ * feedback preferences' `greeting`) and nothing ever reads it back: the chat
+ * opens empty, offline. iOS shows it. Pinned until the Android SDK renders it.
+ */
+const itGreeting = ON_ANDROID ? it.failing : it;
+
+describeDevice(`the feedback chat on ${TARGET_NAME}`, () => {
+  let log: DeviceLog | undefined;
+
+  beforeAll(async () => {
+    log = await beginRetainingSuite('beta-feedback');
+  });
+
+  afterAll(async () => {
+    try {
+      // feedback-chat set an e-mail user identifier, which outlives the
+      // container wipe (the iOS Keychain; Android's SDK store): clear it
+      // the way attributes.test.ts does, with its `attributes-persist`.
+      const clearer = await startRun('attributes-persist');
+      must(
+        await log!.waitFor(/BUGSEE_E2E attr cleared-id \{"type":"undefined"\}/, 30_000, clearer.start),
+        'the user identifier cleared after the feedback-chat runs',
+        clearer.start,
+      );
+    } finally {
+      await endRetainingSuite(log);
+    }
+  });
+
+  /** Starts `scenario`, waits for its `shown` marker, lets the UI settle. */
+  async function show(scenario: 'feedback' | 'feedback-chat'): Promise<Run> {
+    const run = await startRun(scenario);
+    must(
+      await log!.waitFor(new RegExp(`BUGSEE_E2E feedback shown nonce=${run.scenario.nonce}`), 20_000, run.start),
+      'showFeedbackUI() returning',
+      run.start,
+    );
+    // The chat is presented on the next main-thread turn (Android after a
+    // storage-thread hop); a capture right away can still see the app.
+    await new Promise(resolve => setTimeout(resolve, 6_000));
+    return run;
+  }
+
+  async function screenText(label: string): Promise<string[]> {
+    const shot = await captureScreen(label);
+    const lines = await ocrLines(shot);
+    report(`${label} text`, lines);
+    return lines;
+  }
+
+  it('showFeedbackUI() puts the feedback screen in front of the app', async () => {
+    await show('feedback');
+    if (ON_ANDROID) {
+      const top = await androidTopActivity();
+      report('top activity', top);
+      expect(top).toBe(FEEDBACK_ACTIVITY);
+    }
+    // Both platforms title the screen "Feedback", in every locale tried.
+    const text = await screenText('feedback');
+    expect(text).toContain('Feedback');
+    // The app's own screen is gone from view: its status label is not read.
+    expect(text).not.toContain('Bugsee React Native');
+  });
+
+  itGreeting('the chat shows the greeting setGreeting() set', async () => {
+    const run = await show('feedback-chat');
+    if (ON_ANDROID) {
+      expect(await androidTopActivity()).toBe(FEEDBACK_ACTIVITY);
+    }
+    const text = await screenText('feedback-chat');
+    // OCR reads a hex nonce's 1 as l or I now and then: compare on the
+    // characters it can tell apart.
+    expect(text.map(ocrFold)).toContain(ocrFold(`hello ${run.scenario.nonce}`));
+  });
+
+  itColour('the chat is painted in the appearance background color', async () => {
+    await show('feedback-chat');
+    if (ON_ANDROID) {
+      expect(await androidTopActivity()).toBe(FEEDBACK_ACTIVITY);
+    }
+    const shot = await captureScreen('feedback-chat-colour');
+    const found = await colourPixels(shot, colourOf(CHAT_BACKGROUND), 4);
+    report('chat background pixels', { count: found.count, of: found.width * found.height, box: found.box });
+    // Most of the screen: everything but the status bar, title bar and input.
+    expect(found.count).toBeGreaterThan(0.5 * found.width * found.height);
+    if (ON_ANDROID) {
+      await adbStatus('shell', 'input', 'keyevent', 'KEYCODE_BACK');
+    }
+  });
+});

examples/bare/e2e/ios-native-crash.test.ts

@@ -0,0 +1,146 @@
+/**
+ * FLOW-10 / FLOW-09 on iOS: a real signal crash -- not an NSException -- is
+ * recovered at the next launch, and the recovery handler's edits reach it.
+ *
+ * `crashNative('segv' | 'abort')` (examples/e2e-native, iOS since this
+ * suite) stores through a bad pointer (EXC_BAD_ACCESS, SIGSEGV) or calls
+ * abort() (SIGABRT). report-handler.test.ts's iOS recovery case uses
+ * `testNativeCrash()`, an uncaught NSException; nothing before this proved a
+ * signal with no Objective-C exception behind it reaches a report.
+ *
+ * Per kind:
+ *   1. `native-crash-<kind>`: Launched, then `BUGSEE_E2E native crashing
+ *      kind=<kind>`; devicectl's own last line names the signal the app
+ *      died of, and the device's process list no longer has it.
+ *   2. `native-crash-recover`: relaunch with a report handler that labels
+ *      the recovered crash `['e2e-recovered', <nonce>]` from whichever phase
+ *      the SDK offers it in (scenarios/native.ts).
+ *   3. One `crash` bundle; its crash capture names the signal; its labels
+ *      carry the relaunch's nonce.
+ *
+ * iPhone only: the simulator slice of the iOS SDK has no crash reporter
+ * (report-handler.test.ts), so a crash there is never recovered.
+ */
+import { type PulledBundle, crashOf } from './bundles';
+import { iosTarget } from './device';
+import { ON_IOS, type Run, TARGET_NAME, awaitBundles, clearBundles, describeDevice, must, report, startRun } from './harness';
+import { beginRetainingSuite, endRetainingSuite, iosCrashQueueFiles } from './observe';
+import { type DeviceLog, devicePidsOfApp, deviceTerminationSignal } from './scenario';
+
+jest.setTimeout(10 * 60_000);
+
+const ON_IPHONE = ON_IOS && iosTarget() === 'device';
+const describeIphone = ON_IPHONE ? describeDevice : describe.skip;
+
+interface KindOutcome {
+  readonly crash: Run;
+  readonly signal: number | undefined;
+  readonly stillRunning: boolean;
+  readonly recover: Run;
+  readonly crashes: readonly PulledBundle[];
+  readonly handlerLines: readonly string[];
+}
+
+describeIphone(`a native signal crash recovered on ${TARGET_NAME}`, () => {
+  let log: DeviceLog | undefined;
+  const outcomes = new Map<string, KindOutcome>();
+
+  beforeAll(async () => {
+    log = await beginRetainingSuite('beta-ios-crash');
+  });
+
+  afterAll(() => endRetainingSuite(log));
+
+  async function crashAndRecover(kind: 'segv' | 'abort'): Promise<KindOutcome> {
+    const cached = outcomes.get(kind);
+    if (cached !== undefined) {
+      return cached;
+    }
+    await clearBundles();
+    const crash = await startRun(`native-crash-${kind}`);
+    const pid = Number(/BareExample\[(\d+):/.exec(crash.banner.text)?.[1]);
+    must(
+      await log!.waitFor(new RegExp(`BUGSEE_E2E native crashing kind=${kind}`), 20_000, crash.start),
+      `the app calling crashNative('${kind}')`,
+      crash.start,
+    );
+    // The console stream ends only when the process does.
+    const ended = await Promise.race([
+      crash.launch!.ended.then(() => true),
+      new Promise<boolean>(resolve => setTimeout(() => resolve(false), 30_000)),
+    ]);
+    expect(ended).toBe(true);
+    const signal = deviceTerminationSignal(crash.launch!.output);
+    const stillRunning = (await devicePidsOfApp()).includes(pid);
+    // What the crash left for the next launch to claim.
+    report(`${kind} crash queue before the relaunch`, await iosCrashQueueFiles());
+    report(`${kind} died`, { pid, signal, consoleEnd: crash.launch!.output.slice(-3).map(line => line.text.trim()) });
+
+    const recover = await startRun('native-crash-recover');
+    const bundles = await awaitBundles(1, 90_000);
+    // Let a late handler edit land before reading the bundle again.
+    await new Promise(resolve => setTimeout(resolve, 5_000));
+    const crashes = (await awaitBundles(1, 1_000)).filter(b => b.request.type === 'crash');
+    const handlerLines = log!.all(new RegExp(`BUGSEE_E2E native recover .*nonce=${recover.scenario.nonce}`), recover.start).map(l => l.text.trim());
+    report(`${kind} bundles`, bundles.map(b => ({ file: b.file, type: b.request.type, labels: b.request.labels })));
+    report(`${kind} handler`, handlerLines);
+    for (const bundle of crashes) {
+      report(`${kind} crash capture`, JSON.stringify(crashOf(bundle) ?? {}).slice(0, 1500));
+    }
+    const outcome = { crash, signal, stillRunning, recover, crashes, handlerLines };
+    outcomes.set(kind, outcome);
+    return outcome;
+  }
+
+  const SIGNALS = { segv: { number: 11, name: 'SIGSEGV' }, abort: { number: 6, name: 'SIGABRT' } } as const;
+
+  function assertRecovered(outcome: KindOutcome, name: string): void {
+    expect(outcome.crashes).toHaveLength(1);
+    const text = JSON.stringify(crashOf(outcome.crashes[0]!));
+    expect(text).toContain(name);
+    // A signal, not an Objective-C exception: no NSException behind it.
+    expect(text).not.toContain('NSGenericException');
+  }
+
+  function assertLabelled(outcome: KindOutcome): void {
+    expect(outcome.handlerLines.some(line => / labels-set /.test(line))).toBe(true);
+    expect(outcome.crashes).toHaveLength(1);
+    expect(outcome.crashes[0]!.request.labels).toEqual(expect.arrayContaining(['e2e-recovered', outcome.recover.scenario.nonce]));
+  }
+
+  it("crashNative('segv') kills the app with SIGSEGV", async () => {
+    const outcome = await crashAndRecover('segv');
+    expect(outcome.signal).toBe(SIGNALS.segv.number);
+    expect(outcome.stillRunning).toBe(false);
+  });
+
+  it('the next launch recovers one crash report naming SIGSEGV', async () => {
+    assertRecovered(await crashAndRecover('segv'), SIGNALS.segv.name);
+  });
+
+  it("the recovery handler's labels reach the segv crash report", async () => {
+    assertLabelled(await crashAndRecover('segv'));
+  });
+
+  it("crashNative('abort') kills the app with SIGABRT", async () => {
+    const outcome = await crashAndRecover('abort');
+    expect(outcome.signal).toBe(SIGNALS.abort.number);
+    expect(outcome.stillRunning).toBe(false);
+  });
+
+  // iOS 7.0.0-beta4 on the XS loses a SIGABRT now and then: in 3 of 16
+  // abort runs on 2026-10-06 the next launch claimed nothing and the handler
+  // was never called (SIGSEGV: 0 of 11). It is not the order of the kinds
+  // (seen abort-first and segv-first) and not a stale queue (the container,
+  // crash queue included, is wiped and asserted gone before each kind). An
+  // intermittent loss cannot be pinned with it.failing without making the
+  // pin flaky, so these two cases state the expected behaviour and go red
+  // when the SDK drops the crash. Open in the beta-coverage report.
+  it('the next launch recovers one crash report naming SIGABRT', async () => {
+    assertRecovered(await crashAndRecover('abort'), SIGNALS.abort.name);
+  });
+
+  it("the recovery handler's labels reach the abort crash report", async () => {
+    assertLabelled(await crashAndRecover('abort'));
+  });
+});