Setup guide: wait for this computer before loading install info, and send each step's load event once

apps/app/src/components/onboarding/OnboardingFlow.tsx

@@ -118,7 +118,7 @@ import {
 } from "./onboarding-telemetry";
 import { useDeviceSetupTracking } from "./useDeviceSetupTracking";
 import { useAgentStateTelemetry } from "./useAgentStateTelemetry";
-import { useLoadTelemetry } from "./useLoadTelemetry";
+import { useLoadTelemetry, type StepLoadTimes } from "./useLoadTelemetry";
 import { useOnboardingLeftTelemetry } from "./useOnboardingLeftTelemetry";
 import { useOnboardingErrorStep } from "./OnboardingErrorTelemetry";
 import {
@@ -397,6 +397,7 @@ function AgentStepContainer({
   hostId,
   hostWaitFailure,
   hostName,
+  loadTimes,
   onContinue,
 }: {
   chrome: StepChrome;
@@ -405,6 +406,7 @@ function AgentStepContainer({
   hostId: string | null;
   hostWaitFailure: HostWaitFailure | null;
   hostName: string | null;
+  loadTimes: StepLoadTimes;
   onContinue: () => void;
 }) {
   const statesQuery = useSystemProviderStates({
@@ -433,6 +435,7 @@ function AgentStepContainer({
   useLoadTelemetry({
     entry,
     step: "agent",
+    loadTimes,
     hostWaitFailure,
     query: statesQuery,
     loaded: states !== undefined,
@@ -1061,6 +1064,7 @@ function PluginsStepContainer({
   entry: onboardingEntry,
   trackInstall,
   switches,
+  loadTimes,
   onBack,
   onContinue,
   onSkip,
@@ -1070,6 +1074,7 @@ function PluginsStepContainer({
   entry: OnboardingEntry;
   trackInstall: TrackPluginInstall;
   switches: OnboardingPluginSwitches;
+  loadTimes: StepLoadTimes;
   onBack: () => void;
   onContinue: () => void;
   onSkip: () => void;
@@ -1078,6 +1083,7 @@ function PluginsStepContainer({
   useLoadTelemetry({
     entry: onboardingEntry,
     step: "plugins",
+    loadTimes,
     hostWaitFailure: null,
     query: catalogQuery,
     loaded: catalogQuery.data !== undefined,
@@ -1422,6 +1428,7 @@ export function OnboardingFlow({
   }, [entry, step]);
   const trackPluginInstall = useOnboardingPluginInstallOutcomes(entry);
   const pluginSwitches = useOnboardingPluginSwitches();
+  const [loadTimes] = useState<StepLoadTimes>(() => new Map());
   const finishedRef = useRef(false);
   useOnboardingLeftTelemetry({ entry, step, finishedRef });
   useOnboardingErrorStep(step);
@@ -1489,6 +1496,7 @@ export function OnboardingFlow({
           hostId={hostId}
           hostWaitFailure={hostWaitFailure}
           hostName={readyHost?.name ?? null}
+          loadTimes={loadTimes}
           onContinue={advance(
             hasReadyAgent(agentStates) ? "completed" : "skipped",
           )}
@@ -1516,6 +1524,7 @@ export function OnboardingFlow({
           entry={entry}
           trackInstall={trackPluginInstall}
           switches={pluginSwitches}
+          loadTimes={loadTimes}
           onBack={goTo(-1)}
           onContinue={advance("completed")}
           onSkip={advance("skipped")}

apps/app/src/components/onboarding/OnboardingGate.test.tsx

@@ -59,7 +59,7 @@ function renderGate() {
 
 beforeEach(() => {
   window.localStorage.clear();
-  mocks.usePrimaryHost.mockReturnValue({ id: "host-1" });
+  mocks.usePrimaryHost.mockReturnValue({ id: "host-1", status: "connected" });
   mocks.useSetupGuideAssignment.mockReturnValue({
     isPending: false,
     data: { variant: "control" },
@@ -223,6 +223,34 @@ describe("OnboardingGate", () => {
     });
   });
 
+  it("waits for this computer to connect before loading agent install info", () => {
+    configWith({ onboardingCompletedAt: null });
+    mocks.useSetupGuideAssignment.mockReturnValue({
+      isPending: false,
+      data: { variant: "new" },
+    });
+    mocks.usePrimaryHost.mockReturnValue({
+      id: "host-1",
+      status: "disconnected",
+    });
+    const { view } = renderGate();
+    expect(mocks.useHostProviderCliStatus).toHaveBeenLastCalledWith({
+      hostId: null,
+      enabled: true,
+    });
+
+    mocks.usePrimaryHost.mockReturnValue({ id: "host-1", status: "connected" });
+    view.rerender(
+      <OnboardingGate>
+        <div>App shell</div>
+      </OnboardingGate>,
+    );
+    expect(mocks.useHostProviderCliStatus).toHaveBeenLastCalledWith({
+      hostId: "host-1",
+      enabled: true,
+    });
+  });
+
   it("falls back to the app when settings cannot be loaded", () => {
     mocks.useSystemConfig.mockReturnValue({ data: undefined, isError: true });
 

apps/app/src/components/onboarding/OnboardingGate.tsx

@@ -52,7 +52,7 @@ export function OnboardingGate({ children }: { children: ReactNode }) {
     completedAt === null && setupGuideQuery.data?.variant === "new";
   const primaryHost = usePrimaryHost({ enabled: prefetchInstallInfo });
   useHostProviderCliStatus({
-    hostId: primaryHost?.id ?? null,
+    hostId: primaryHost?.status === "connected" ? primaryHost.id : null,
     enabled: prefetchInstallInfo,
   });
   useEffect(() => {

apps/app/src/components/onboarding/useLoadTelemetry.test.tsx

@@ -3,7 +3,7 @@ import { cleanup, renderHook } from "@testing-library/react";
 import { BbHttpError } from "@bb/sdk/browser";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { recordTelemetryEvent } from "./onboarding-telemetry";
-import { useLoadTelemetry } from "./useLoadTelemetry";
+import { useLoadTelemetry, type StepLoadTimes } from "./useLoadTelemetry";
 
 vi.mock("./onboarding-telemetry", () => ({ recordTelemetryEvent: vi.fn() }));
 
@@ -43,6 +43,7 @@ describe("load telemetry", () => {
           useLoadTelemetry({
             entry: "replay",
             step,
+            loadTimes: new Map(),
             hostWaitFailure: null,
             query,
             loaded: false,
@@ -71,6 +72,7 @@ describe("load telemetry", () => {
       useLoadTelemetry({
         entry: "first_run",
         step: "agent",
+        loadTimes: new Map(),
         hostWaitFailure: "no_host",
         query: loaded,
         loaded: false,
@@ -94,6 +96,7 @@ describe("load telemetry", () => {
         useLoadTelemetry({
           entry: "first_run",
           step: "agent",
+          loadTimes: new Map(),
           hostWaitFailure: null,
           query: props.query,
           loaded: false,
@@ -119,11 +122,13 @@ describe("load telemetry", () => {
 
   it("records the first successful load once with how long it took", () => {
     vi.useFakeTimers();
+    const loadTimes: StepLoadTimes = new Map();
     const view = renderHook(
       (loadedNow: boolean) =>
         useLoadTelemetry({
           entry: "first_run",
           step: "agent",
+          loadTimes,
           hostWaitFailure: null,
           query: loaded,
           loaded: loadedNow,
@@ -147,4 +152,44 @@ describe("load telemetry", () => {
       },
     ]);
   });
+
+  it("records one successful load per step per guide run, timed from when the step first opened", () => {
+    vi.useFakeTimers();
+    const loadTimes: StepLoadTimes = new Map();
+    const renderStep = (loadedNow: boolean) =>
+      renderHook(
+        (props: boolean) =>
+          useLoadTelemetry({
+            entry: "first_run",
+            step: "agent",
+            loadTimes,
+            hostWaitFailure: null,
+            query: loaded,
+            loaded: props,
+            timedOut: false,
+          }),
+        { initialProps: loadedNow },
+      );
+
+    const firstVisit = renderStep(false);
+    vi.advanceTimersByTime(1_000);
+    firstVisit.unmount();
+    vi.advanceTimersByTime(2_000);
+    const secondVisit = renderStep(false);
+    vi.advanceTimersByTime(500);
+    secondVisit.rerender(true);
+    secondVisit.unmount();
+    renderStep(true).unmount();
+    vi.useRealTimers();
+
+    expect(recorded()).toEqual([
+      {
+        entry: "first_run",
+        step: "agent",
+        operation: "load",
+        outcome: "succeeded",
+        duration_ms: 3_500,
+      },
+    ]);
+  });
 });

apps/app/src/components/onboarding/useLoadTelemetry.ts

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useRef } from "react";
 import {
   operationFailureReason,
   type OnboardingStepId,
@@ -10,16 +10,23 @@ import {
 } from "./onboarding-telemetry";
 import type { HostWaitFailure } from "./useReadyHost";
 
+export type StepLoadTimes = Map<
+  OnboardingStepId,
+  { shownAt: number; reported: boolean }
+>;
+
 export function useLoadTelemetry({
   entry,
   step,
+  loadTimes,
   hostWaitFailure,
   query,
   loaded,
   timedOut,
 }: {
   entry: OnboardingEntry;
   step: OnboardingStepId;
+  loadTimes: StepLoadTimes;
   hostWaitFailure: HostWaitFailure | null;
   query: { isError: boolean; error: unknown };
   loaded: boolean;
@@ -51,20 +58,24 @@ export function useLoadTelemetry({
       },
     });
   }, [entry, failureReason, step]);
-  const [shownAt] = useState(() => Date.now());
-  const succeededRef = useRef(false);
   useEffect(() => {
-    if (!loaded || succeededRef.current) return;
-    succeededRef.current = true;
+    if (!loadTimes.has(step)) {
+      loadTimes.set(step, { shownAt: Date.now(), reported: false });
+    }
+  }, [loadTimes, step]);
+  useEffect(() => {
+    const timing = loadTimes.get(step);
+    if (!loaded || timing === undefined || timing.reported) return;
+    loadTimes.set(step, { ...timing, reported: true });
     recordTelemetryEvent({
       name: "onboarding_operation",
       properties: {
         entry,
         step,
         operation: "load",
         outcome: "succeeded",
-        duration_ms: Math.max(0, Math.round(Date.now() - shownAt)),
+        duration_ms: Math.max(0, Math.round(Date.now() - timing.shownAt)),
       },
     });
-  }, [entry, loaded, shownAt, step]);
+  }, [entry, loadTimes, loaded, step]);
 }