From 0058fb12f404508761bb68320040591239e607b6 Mon Sep 17 00:00:00 2001 From: Coy Geek <65363919+coygeek@users.noreply.github.com> Date: Fri, 8 May 2026 09:29:28 -0700 Subject: [PATCH 1/4] feat(web): add assistant turn stats footer Show compact assistant turn stats with model, elapsed time, token count, throughput, TTFT, and tool-call count when those fields are available. Derive Codex timing from response boundaries so completed throughput does not use tiny final delta gaps as the denominator. Closes #2518 --- .../Layers/ProjectionPipeline.test.ts | 138 ++++++ .../Layers/ProjectionPipeline.ts | 2 +- .../Layers/ProviderRuntimeIngestion.test.ts | 467 ++++++++++++++++++ .../Layers/ProviderRuntimeIngestion.ts | 243 ++++++++- .../src/provider/Layers/CodexAdapter.test.ts | 3 +- .../src/provider/Layers/CodexAdapter.ts | 15 +- apps/web/src/components/ChatView.browser.tsx | 37 ++ apps/web/src/components/ChatView.tsx | 24 + .../chat/MessagesTimeline.browser.tsx | 54 ++ .../chat/MessagesTimeline.logic.test.ts | 65 +++ .../components/chat/MessagesTimeline.logic.ts | 8 + .../components/chat/MessagesTimeline.test.tsx | 1 + .../src/components/chat/MessagesTimeline.tsx | 13 +- .../src/components/chat/TurnStatsFooter.tsx | 33 ++ apps/web/src/lib/contextWindow.test.ts | 81 ++- apps/web/src/lib/contextWindow.ts | 39 +- apps/web/src/lib/turnStats.test.ts | 456 +++++++++++++++++ apps/web/src/lib/turnStats.ts | 279 +++++++++++ apps/web/src/proposedPlan.test.ts | 10 +- apps/web/src/proposedPlan.ts | 5 +- .../contracts/src/providerRuntime.test.ts | 2 + packages/contracts/src/providerRuntime.ts | 1 + 22 files changed, 1961 insertions(+), 15 deletions(-) create mode 100644 apps/web/src/components/chat/TurnStatsFooter.tsx create mode 100644 apps/web/src/lib/turnStats.test.ts create mode 100644 apps/web/src/lib/turnStats.ts diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts index 7f364c717a7..c05dde50beb 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts @@ -170,6 +170,144 @@ it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => { } }), ); + + it.effect("keeps the latest turn pointer when a completed session becomes ready", () => + Effect.gen(function* () { + const projectionPipeline = yield* OrchestrationProjectionPipeline; + const eventStore = yield* OrchestrationEventStore; + const sql = yield* SqlClient.SqlClient; + const createdAt = "2026-05-08T08:00:00.000Z"; + const startedAt = "2026-05-08T08:00:01.000Z"; + const completedAt = "2026-05-08T08:00:06.000Z"; + const turnId = TurnId.make("turn-latest-pointer"); + + yield* eventStore.append({ + type: "project.created", + eventId: EventId.make("evt-latest-project"), + aggregateKind: "project", + aggregateId: ProjectId.make("project-latest-pointer"), + occurredAt: createdAt, + commandId: CommandId.make("cmd-latest-project"), + causationEventId: null, + correlationId: CommandId.make("cmd-latest-project"), + metadata: {}, + payload: { + projectId: ProjectId.make("project-latest-pointer"), + title: "Project latest pointer", + workspaceRoot: "/tmp/project-latest-pointer", + defaultModelSelection: null, + scripts: [], + createdAt, + updatedAt: createdAt, + }, + }); + + yield* eventStore.append({ + type: "thread.created", + eventId: EventId.make("evt-latest-thread"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-latest-pointer"), + occurredAt: createdAt, + commandId: CommandId.make("cmd-latest-thread"), + causationEventId: null, + correlationId: CommandId.make("cmd-latest-thread"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-latest-pointer"), + projectId: ProjectId.make("project-latest-pointer"), + title: "Latest pointer", + modelSelection: { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5.4", + }, + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt, + updatedAt: createdAt, + }, + }); + + yield* eventStore.append({ + type: "thread.session-set", + eventId: EventId.make("evt-latest-running"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-latest-pointer"), + occurredAt: startedAt, + commandId: CommandId.make("cmd-latest-running"), + causationEventId: null, + correlationId: CommandId.make("cmd-latest-running"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-latest-pointer"), + session: { + threadId: ThreadId.make("thread-latest-pointer"), + status: "running", + providerName: "codex", + runtimeMode: "full-access", + activeTurnId: turnId, + lastError: null, + updatedAt: startedAt, + }, + }, + }); + + yield* eventStore.append({ + type: "thread.message-sent", + eventId: EventId.make("evt-latest-assistant"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-latest-pointer"), + occurredAt: completedAt, + commandId: CommandId.make("cmd-latest-assistant"), + causationEventId: null, + correlationId: CommandId.make("cmd-latest-assistant"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-latest-pointer"), + messageId: MessageId.make("message-latest-assistant"), + role: "assistant", + text: "done", + turnId, + streaming: false, + createdAt: completedAt, + updatedAt: completedAt, + }, + }); + + yield* eventStore.append({ + type: "thread.session-set", + eventId: EventId.make("evt-latest-ready"), + aggregateKind: "thread", + aggregateId: ThreadId.make("thread-latest-pointer"), + occurredAt: completedAt, + commandId: CommandId.make("cmd-latest-ready"), + causationEventId: null, + correlationId: CommandId.make("cmd-latest-ready"), + metadata: {}, + payload: { + threadId: ThreadId.make("thread-latest-pointer"), + session: { + threadId: ThreadId.make("thread-latest-pointer"), + status: "ready", + providerName: "codex", + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: completedAt, + }, + }, + }); + + yield* projectionPipeline.bootstrap; + + const rows = yield* sql<{ readonly latestTurnId: string | null }>` + SELECT latest_turn_id AS "latestTurnId" + FROM projection_threads + WHERE thread_id = 'thread-latest-pointer' + `; + assert.deepEqual(rows, [{ latestTurnId: "turn-latest-pointer" }]); + }), + ); }); it.layer(Layer.fresh(makeProjectionPipelinePrefixedTestLayer("t3-base-")))( diff --git a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts index 3ef8b38d642..e84283b6381 100644 --- a/apps/server/src/orchestration/Layers/ProjectionPipeline.ts +++ b/apps/server/src/orchestration/Layers/ProjectionPipeline.ts @@ -709,7 +709,7 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti } yield* projectionThreadRepository.upsert({ ...existingRow.value, - latestTurnId: event.payload.session.activeTurnId, + latestTurnId: event.payload.session.activeTurnId ?? existingRow.value.latestTurnId, updatedAt: event.occurredAt, }); yield* refreshThreadShellSummary(event.payload.threadId); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 2fe0e406d66..9be138e0173 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -2720,6 +2720,473 @@ describe("ProviderRuntimeIngestion", () => { }); }); + it("derives context window duration from Codex assistant response boundaries", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-throughput-single-span"); + const startedAt = "2026-05-08T12:00:00.000Z"; + const firstDeltaAt = "2026-05-08T12:04:19.000Z"; + const lastDeltaAt = "2026-05-08T12:04:44.000Z"; + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-throughput-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: startedAt, + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-delta-first"), + provider: ProviderDriverKind.make("codex"), + createdAt: firstDeltaAt, + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput"), + payload: { + streamKind: "assistant_text", + delta: "hello", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-delta-last"), + provider: ProviderDriverKind.make("codex"), + createdAt: lastDeltaAt, + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput"), + payload: { + streamKind: "assistant_text", + delta: " world", + }, + }); + harness.emit({ + type: "turn.completed", + eventId: asEventId("evt-throughput-turn-completed"), + provider: ProviderDriverKind.make("codex"), + createdAt: lastDeltaAt, + threadId: asThreadId("thread-1"), + turnId, + payload: { + state: "completed", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: lastDeltaAt, + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-throughput-token-usage", + ), + ); + + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-throughput-token-usage", + ); + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 25_000, + timeToFirstTokenMs: 259_000, + }); + expect((usageActivity?.payload as { durationMs?: number } | undefined)?.durationMs).not.toBe( + 284_000, + ); + }); + + it("uses the assistant completion boundary instead of a tiny final delta span", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-throughput-tiny-final-delta"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-throughput-tiny-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-tiny-delta-first"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:06.186Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-tiny"), + payload: { + streamKind: "assistant_text", + delta: "ok", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-tiny-delta-last"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:06.195Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-tiny"), + payload: { + streamKind: "assistant_text", + delta: ".", + }, + }); + harness.emit({ + type: "turn.completed", + eventId: asEventId("evt-throughput-tiny-turn-completed"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:06.700Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + state: "completed", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-tiny-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:06.700Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 72_193, + lastOutputTokens: 16, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-tiny-token-usage", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-throughput-tiny-token-usage", + ); + + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 72_193, + lastOutputTokens: 16, + durationMs: 514, + timeToFirstTokenMs: 6_186, + }); + expect((usageActivity?.payload as { durationMs?: number } | undefined)?.durationMs).not.toBe(9); + }); + + it("preserves provider duration instead of overwriting it with derived Codex stream duration", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-throughput-provider-duration"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-throughput-provider-duration-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-provider-duration-delta-first"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:10.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-provider-duration"), + payload: { + streamKind: "assistant_text", + delta: "hello", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-provider-duration-delta-last"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:35.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-provider-duration"), + payload: { + streamKind: "assistant_text", + delta: " world", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-provider-duration-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:36.000Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 12_345, + timeToFirstTokenMs: 1_234, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-provider-duration-token-usage", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-provider-duration-token-usage", + ); + + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 12_345, + timeToFirstTokenMs: 1_234, + }); + expect((usageActivity?.payload as { durationMs?: number } | undefined)?.durationMs).not.toBe( + 25_000, + ); + expect( + (usageActivity?.payload as { timeToFirstTokenMs?: number } | undefined)?.timeToFirstTokenMs, + ).not.toBe(10_000); + }); + + it("omits context window duration when no positive assistant text span was observed", async () => { + const harness = await createHarness(); + + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-token-usage-without-delta"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:04:44.000Z", + threadId: asThreadId("thread-1"), + turnId: asTurnId("turn-throughput-without-delta"), + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-token-usage-without-delta", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-token-usage-without-delta", + ); + + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 150_000, + lastOutputTokens: 1_790, + }); + expect((usageActivity?.payload as { durationMs?: number } | undefined)?.durationMs).toBe( + undefined, + ); + expect( + (usageActivity?.payload as { timeToFirstTokenMs?: number } | undefined)?.timeToFirstTokenMs, + ).toBe(undefined); + }); + + it("omits context window TTFT when the first assistant text delta is not after turn start", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-ttft-non-positive"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-ttft-non-positive-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-ttft-non-positive-delta"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-ttft-non-positive"), + payload: { + streamKind: "assistant_text", + delta: "hello", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-ttft-non-positive-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:01.000Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-ttft-non-positive-token-usage", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-ttft-non-positive-token-usage", + ); + + expect( + (usageActivity?.payload as { timeToFirstTokenMs?: number } | undefined)?.timeToFirstTokenMs, + ).toBe(undefined); + }); + + it("sums assistant response segments without including tool gaps", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-throughput-tool-gap"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-throughput-gap-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-gap-delta-1-first"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:10.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-gap"), + payload: { + streamKind: "assistant_text", + delta: "before", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-gap-delta-1-last"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:20.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-gap"), + payload: { + streamKind: "assistant_text", + delta: " approval", + }, + }); + harness.emit({ + type: "request.opened", + eventId: asEventId("evt-throughput-gap-request"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:25.000Z", + threadId: asThreadId("thread-1"), + turnId, + requestId: ApprovalRequestId.make("req-throughput-gap"), + payload: { + requestType: "command_execution_approval", + detail: "sleep 120", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-gap-delta-2-first"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:03:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-gap"), + payload: { + streamKind: "assistant_text", + delta: "after", + }, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-gap-delta-2-last"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:03:15.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-gap"), + payload: { + streamKind: "assistant_text", + delta: " approval", + }, + }); + harness.emit({ + type: "turn.completed", + eventId: asEventId("evt-throughput-gap-turn-completed"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:03:16.000Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + state: "completed", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-gap-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:03:16.000Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-throughput-gap-token-usage", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => activity.id === "evt-throughput-gap-token-usage", + ); + + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 31_000, + timeToFirstTokenMs: 10_000, + }); + }); + it("projects compacted thread state into context compaction activities", async () => { const harness = await createHarness(); const now = new Date().toISOString(); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index 2e86623f8dc..ac77e3e8fd3 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -42,6 +42,17 @@ interface AssistantSegmentState { activeMessageId: MessageId | null; } +interface AssistantResponseSegmentTiming { + startedAtMs: number; +} + +interface AssistantStreamTimingState { + accumulatedResponseDurationMs: number; + activeResponseSegment?: AssistantResponseSegmentTiming | undefined; + turnStartedAtMs?: number; + firstAssistantDeltaAtMs?: number; +} + const TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY = 10_000; const TURN_MESSAGE_IDS_BY_TURN_TTL = Duration.minutes(120); const BUFFERED_MESSAGE_TEXT_BY_MESSAGE_ID_CACHE_CAPACITY = 20_000; @@ -197,13 +208,83 @@ function assistantSegmentMessageId(baseKey: string, segmentIndex: number): Messa segmentIndex === 0 ? `assistant:${baseKey}` : `assistant:${baseKey}:segment:${segmentIndex}`, ); } + +function parseTimestampMs(value: string): number | undefined { + const timestampMs = Date.parse(value); + return Number.isFinite(timestampMs) ? timestampMs : undefined; +} + +function toPositiveFiniteMs(value: number | undefined): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined; +} + +function deriveAssistantStreamDurationMs( + state: AssistantStreamTimingState | undefined, +): number | undefined { + if (!state) { + return undefined; + } + + const roundedDurationMs = Math.round(state.accumulatedResponseDurationMs); + return roundedDurationMs > 0 ? roundedDurationMs : undefined; +} + +function deriveAssistantTimeToFirstTokenMs( + state: AssistantStreamTimingState | undefined, +): number | undefined { + if (!state) { + return undefined; + } + const { turnStartedAtMs, firstAssistantDeltaAtMs } = state; + if ( + turnStartedAtMs === undefined || + firstAssistantDeltaAtMs === undefined || + !Number.isFinite(turnStartedAtMs) || + !Number.isFinite(firstAssistantDeltaAtMs) + ) { + return undefined; + } + const durationMs = firstAssistantDeltaAtMs - turnStartedAtMs; + return durationMs > 0 && Number.isFinite(durationMs) ? Math.round(durationMs) : undefined; +} + function buildContextWindowActivityPayload( event: ProviderRuntimeEvent, + options?: { + readonly assistantStreamDurationMs?: number | undefined; + readonly assistantTimeToFirstTokenMs?: number | undefined; + }, ): ThreadTokenUsageSnapshot | undefined { if (event.type !== "thread.token-usage.updated" || event.payload.usage.usedTokens <= 0) { return undefined; } - return event.payload.usage; + const usage = event.payload.usage; + const providerDurationMs = toPositiveFiniteMs(usage.durationMs); + const providerTimeToFirstTokenMs = toPositiveFiniteMs(usage.timeToFirstTokenMs); + const assistantStreamDurationMs = toPositiveFiniteMs(options?.assistantStreamDurationMs); + const assistantTimeToFirstTokenMs = toPositiveFiniteMs(options?.assistantTimeToFirstTokenMs); + if (providerDurationMs !== undefined) { + return providerTimeToFirstTokenMs !== undefined + ? usage + : { + ...usage, + ...(assistantTimeToFirstTokenMs !== undefined + ? { timeToFirstTokenMs: assistantTimeToFirstTokenMs } + : {}), + }; + } + + if (assistantStreamDurationMs === undefined && assistantTimeToFirstTokenMs === undefined) { + return usage; + } + + return { + ...usage, + ...(assistantStreamDurationMs !== undefined ? { durationMs: assistantStreamDurationMs } : {}), + ...(assistantTimeToFirstTokenMs !== undefined + ? { timeToFirstTokenMs: assistantTimeToFirstTokenMs } + : {}), + }; } function normalizeRuntimeTurnState( @@ -259,6 +340,10 @@ function requestKindFromCanonicalRequestType( function runtimeEventToActivities( event: ProviderRuntimeEvent, + options?: { + readonly assistantStreamDurationMs?: number | undefined; + readonly assistantTimeToFirstTokenMs?: number | undefined; + }, ): ReadonlyArray { const maybeSequence = (() => { const eventWithSequence = event as ProviderRuntimeEvent & { sessionSequence?: number }; @@ -508,7 +593,7 @@ function runtimeEventToActivities( } case "thread.token-usage.updated": { - const payload = buildContextWindowActivityPayload(event); + const payload = buildContextWindowActivityPayload(event, options); if (!payload) { return []; } @@ -628,6 +713,15 @@ const make = Effect.gen(function* () { ), }); + const assistantStreamTimingByTurnKey = yield* Cache.make({ + capacity: TURN_MESSAGE_IDS_BY_TURN_CACHE_CAPACITY, + timeToLive: TURN_MESSAGE_IDS_BY_TURN_TTL, + lookup: () => + Effect.die( + new Error("assistant stream timing should be read through getOption before initialization"), + ), + }); + const bufferedProposedPlanById = yield* Cache.make({ capacity: BUFFERED_PROPOSED_PLAN_BY_ID_CACHE_CAPACITY, timeToLive: BUFFERED_PROPOSED_PLAN_BY_ID_TTL, @@ -703,6 +797,99 @@ const make = Effect.gen(function* () { const clearAssistantSegmentStateForTurn = (threadId: ThreadId, turnId: TurnId) => Cache.invalidate(assistantSegmentStateByTurnKey, providerTurnKey(threadId, turnId)); + const rememberTurnStartedAt = (input: { + threadId: ThreadId; + turnId: TurnId; + createdAt: string; + }) => + Effect.gen(function* () { + const startedAtMs = parseTimestampMs(input.createdAt); + if (startedAtMs === undefined) { + return; + } + + const turnKey = providerTurnKey(input.threadId, input.turnId); + const existingState = yield* Cache.getOption(assistantStreamTimingByTurnKey, turnKey); + yield* Cache.set(assistantStreamTimingByTurnKey, turnKey, { + ...(Option.isSome(existingState) + ? existingState.value + : { accumulatedResponseDurationMs: 0 }), + turnStartedAtMs: startedAtMs, + }); + }); + + const rememberAssistantStreamDelta = (input: { + threadId: ThreadId; + turnId: TurnId; + createdAt: string; + }) => + Effect.gen(function* () { + const deltaAtMs = parseTimestampMs(input.createdAt); + if (deltaAtMs === undefined) { + return; + } + + const turnKey = providerTurnKey(input.threadId, input.turnId); + const existingState = yield* Cache.getOption(assistantStreamTimingByTurnKey, turnKey); + const currentState = Option.isSome(existingState) + ? existingState.value + : ({ accumulatedResponseDurationMs: 0 } satisfies AssistantStreamTimingState); + const activeResponseSegment = currentState.activeResponseSegment + ? { + startedAtMs: Math.min(currentState.activeResponseSegment.startedAtMs, deltaAtMs), + } + : { + startedAtMs: deltaAtMs, + }; + + yield* Cache.set(assistantStreamTimingByTurnKey, turnKey, { + ...currentState, + activeResponseSegment, + firstAssistantDeltaAtMs: + currentState.firstAssistantDeltaAtMs !== undefined + ? Math.min(currentState.firstAssistantDeltaAtMs, deltaAtMs) + : deltaAtMs, + }); + }); + + const closeAssistantResponseSegment = (input: { + threadId: ThreadId; + turnId: TurnId; + createdAt: string; + }) => + Effect.gen(function* () { + const closedAtMs = parseTimestampMs(input.createdAt); + if (closedAtMs === undefined) { + return; + } + + const turnKey = providerTurnKey(input.threadId, input.turnId); + const existingState = yield* Cache.getOption(assistantStreamTimingByTurnKey, turnKey); + if (Option.isNone(existingState) || !existingState.value.activeResponseSegment) { + return; + } + + const { activeResponseSegment, ...state } = existingState.value; + const segmentDurationMs = closedAtMs - activeResponseSegment.startedAtMs; + yield* Cache.set(assistantStreamTimingByTurnKey, turnKey, { + ...state, + accumulatedResponseDurationMs: + segmentDurationMs > 0 && Number.isFinite(segmentDurationMs) + ? state.accumulatedResponseDurationMs + segmentDurationMs + : state.accumulatedResponseDurationMs, + }); + }); + + const getAssistantStreamDurationMsForTurn = (threadId: ThreadId, turnId: TurnId) => + Cache.getOption(assistantStreamTimingByTurnKey, providerTurnKey(threadId, turnId)).pipe( + Effect.map((state) => deriveAssistantStreamDurationMs(Option.getOrUndefined(state))), + ); + + const getAssistantTimeToFirstTokenMsForTurn = (threadId: ThreadId, turnId: TurnId) => + Cache.getOption(assistantStreamTimingByTurnKey, providerTurnKey(threadId, turnId)).pipe( + Effect.map((state) => deriveAssistantTimeToFirstTokenMs(Option.getOrUndefined(state))), + ); + const getActiveAssistantMessageIdForTurn = (threadId: ThreadId, turnId: TurnId) => getAssistantSegmentStateForTurn(threadId, turnId).pipe( Effect.map((state) => @@ -1057,6 +1244,7 @@ const make = Effect.gen(function* () { const proposedPlanPrefix = `plan:${threadId}:`; const turnKeys = Array.from(yield* Cache.keys(turnMessageIdsByTurnKey)); const assistantSegmentKeys = Array.from(yield* Cache.keys(assistantSegmentStateByTurnKey)); + const assistantTimingKeys = Array.from(yield* Cache.keys(assistantStreamTimingByTurnKey)); const proposedPlanKeys = Array.from(yield* Cache.keys(bufferedProposedPlanById)); yield* Effect.forEach( turnKeys, @@ -1085,6 +1273,14 @@ const make = Effect.gen(function* () { : Effect.void, { concurrency: 1 }, ).pipe(Effect.asVoid); + yield* Effect.forEach( + assistantTimingKeys, + (key) => + key.startsWith(prefix) + ? Cache.invalidate(assistantStreamTimingByTurnKey, key) + : Effect.void, + { concurrency: 1 }, + ).pipe(Effect.asVoid); yield* Effect.forEach( proposedPlanKeys, (key) => @@ -1189,6 +1385,14 @@ const make = Effect.gen(function* () { const eventTurnId = toTurnId(event.turnId); const activeTurnId = thread.session?.activeTurnId ?? null; + if (event.type === "turn.started" && eventTurnId) { + yield* rememberTurnStartedAt({ + threadId: thread.id, + turnId: eventTurnId, + createdAt: now, + }); + } + const conflictsWithActiveTurn = activeTurnId !== null && eventTurnId !== undefined && !sameId(activeTurnId, eventTurnId); const missingTurnForActiveTurn = activeTurnId !== null && eventTurnId === undefined; @@ -1324,6 +1528,11 @@ const make = Effect.gen(function* () { ...(turnId ? { turnId } : {}), }); if (turnId) { + yield* rememberAssistantStreamDelta({ + threadId: thread.id, + turnId, + createdAt: now, + }); yield* rememberAssistantMessageId(thread.id, turnId, assistantMessageId); } @@ -1362,6 +1571,11 @@ const make = Effect.gen(function* () { ? toTurnId(event.turnId) : undefined; if (pauseForUserTurnId) { + yield* closeAssistantResponseSegment({ + threadId: thread.id, + turnId: pauseForUserTurnId, + createdAt: now, + }); const detailedThread = yield* getLoadedThreadDetail(); const assistantDeliveryMode: AssistantDeliveryMode = yield* Effect.map( serverSettingsService.getSettings, @@ -1429,6 +1643,13 @@ const make = Effect.gen(function* () { const detailedThread = yield* getLoadedThreadDetail(); const messages = detailedThread?.messages ?? []; const turnId = toTurnId(event.turnId); + if (turnId) { + yield* closeAssistantResponseSegment({ + threadId: thread.id, + turnId, + createdAt: now, + }); + } const activeAssistantMessageId = turnId ? yield* getActiveAssistantMessageIdForTurn(thread.id, turnId) : Option.none(); @@ -1496,6 +1717,11 @@ const make = Effect.gen(function* () { const proposedPlans = detailedThread?.proposedPlans ?? []; const turnId = toTurnId(event.turnId); if (turnId) { + yield* closeAssistantResponseSegment({ + threadId: thread.id, + turnId, + createdAt: now, + }); const assistantMessageIds = yield* getAssistantMessageIdsForTurn(thread.id, turnId); yield* Effect.forEach( assistantMessageIds, @@ -1605,7 +1831,18 @@ const make = Effect.gen(function* () { } } - const activities = runtimeEventToActivities(event); + const assistantStreamDurationMs = + event.type === "thread.token-usage.updated" && eventTurnId + ? yield* getAssistantStreamDurationMsForTurn(thread.id, eventTurnId) + : undefined; + const assistantTimeToFirstTokenMs = + event.type === "thread.token-usage.updated" && eventTurnId + ? yield* getAssistantTimeToFirstTokenMsForTurn(thread.id, eventTurnId) + : undefined; + const activities = runtimeEventToActivities(event, { + assistantStreamDurationMs, + assistantTimeToFirstTokenMs, + }); yield* Effect.forEach(activities, (activity) => orchestrationEngine.dispatch({ type: "thread.activity.append", diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 486e149166b..50754d3ae68 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -964,7 +964,6 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { kind: "notification", provider: ProviderDriverKind.make("codex"), threadId: asThreadId("thread-1"), - turnId: asTurnId("turn-1"), createdAt: new Date().toISOString(), method: "thread/tokenUsage/updated", payload: { @@ -1015,6 +1014,8 @@ lifecycleLayer("CodexAdapterLive lifecycle", (it) => { lastReasoningOutputTokens: 0, compactsAutomatically: true, }); + assert.equal(firstEvent.value.turnId, "turn-1"); + assert.equal(firstEvent.value.providerRefs?.providerTurnId, "turn-1"); }), ); }); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 7486f2b9bb8..98c12c997a6 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -22,6 +22,7 @@ import { RuntimeRequestId, ProviderApprovalDecision, ThreadId, + TurnId, ProviderSendTurnInput, } from "@t3tools/contracts"; import { Effect, Exit, Fiber, FileSystem, Queue, Schema, Scope, Stream } from "effect"; @@ -715,14 +716,24 @@ function mapToRuntimeEvents( EffectCodexSchema.V2ThreadTokenUsageUpdatedNotification, event.payload, ); - const normalizedUsage = payload ? normalizeCodexTokenUsage(payload.tokenUsage) : undefined; + if (!payload) { + return []; + } + const normalizedUsage = normalizeCodexTokenUsage(payload.tokenUsage); if (!normalizedUsage) { return []; } + const turnId = TurnId.make(payload.turnId); + const baseEvent = runtimeEventBase(event, canonicalThreadId); return [ { type: "thread.token-usage.updated", - ...runtimeEventBase(event, canonicalThreadId), + ...baseEvent, + turnId, + providerRefs: { + ...baseEvent.providerRefs, + providerTurnId: turnId, + }, payload: { usage: normalizedUsage, }, diff --git a/apps/web/src/components/ChatView.browser.tsx b/apps/web/src/components/ChatView.browser.tsx index 578dc7c045d..ef22ffaf9f5 100644 --- a/apps/web/src/components/ChatView.browser.tsx +++ b/apps/web/src/components/ChatView.browser.tsx @@ -5666,6 +5666,10 @@ describe("ChatView timeline estimator parity (full app)", () => { ...snapshot, threads, }, + resolveRpc: (body) => + body._tag === WS_METHODS.projectsWriteFile + ? { relativePath: body.relativePath } + : undefined, }); try { @@ -5692,6 +5696,39 @@ describe("ChatView timeline estimator parity (full app)", () => { }, { timeout: 8_000, interval: 16 }, ); + + const savePathInput = await waitForElement( + () => + Array.from(document.querySelectorAll("input")).find( + (input) => input.value === "plan-ship-plan-mode-follow-up.md", + ) ?? null, + "Unable to find default root plan-artifact save path.", + ); + expect(savePathInput.value).toBe("plan-ship-plan-mode-follow-up.md"); + + const saveButton = await waitForElement( + () => + Array.from(document.querySelectorAll("button")).find( + (button) => button.textContent?.trim() === "Save", + ) ?? null, + "Unable to find proposed plan Save button.", + ); + saveButton.click(); + + await vi.waitFor( + () => { + expect(wsRequests).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + _tag: WS_METHODS.projectsWriteFile, + cwd: "/repo/worktrees/plan-thread", + relativePath: "plan-ship-plan-mode-follow-up.md", + }), + ]), + ); + }, + { timeout: 8_000, interval: 16 }, + ); } finally { await mounted.cleanup(); } diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 6b84aa11ca6..9af76ef306d 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -120,6 +120,7 @@ import { useSettings } from "../hooks/useSettings"; import { resolveAppModelSelectionForInstance } from "../modelSelection"; import { isTerminalFocused } from "../lib/terminalFocus"; import { deriveLogicalProjectKeyFromSettings } from "../logicalProject"; +import { buildLatestAssistantTurnStatsMap } from "../lib/turnStats"; import { reconnectSavedEnvironment, useSavedEnvironmentRegistryStore, @@ -1562,6 +1563,28 @@ export default function ChatView(props: ChatViewProps) { deriveTimelineEntries(timelineMessages, activeThread?.proposedPlans ?? [], workLogEntries), [activeThread?.proposedPlans, timelineMessages, workLogEntries], ); + const latestAssistantTurnStatsByMessageId = useMemo(() => { + const assistantMessageId = activeLatestTurn?.assistantMessageId; + const assistantMessage = + assistantMessageId && latestTurnSettled + ? (timelineMessages.find( + (message) => message.id === assistantMessageId && message.role === "assistant", + ) ?? null) + : null; + + return buildLatestAssistantTurnStatsMap({ + latestTurn: latestTurnSettled ? activeLatestTurn : null, + assistantMessage, + activities: threadActivities, + modelSelection: activeThread?.modelSelection, + }); + }, [ + activeLatestTurn, + activeThread?.modelSelection, + latestTurnSettled, + threadActivities, + timelineMessages, + ]); const { turnDiffSummaries, inferredCheckpointTurnCountByTurnId } = useTurnDiffSummaries(activeThread); const turnDiffSummaryByAssistantMessageId = useMemo(() => { @@ -3564,6 +3587,7 @@ export default function ChatView(props: ChatViewProps) { completionDividerBeforeEntryId={completionDividerBeforeEntryId} completionSummary={completionSummary} turnDiffSummaryByAssistantMessageId={turnDiffSummaryByAssistantMessageId} + assistantTurnStatsByMessageId={latestAssistantTurnStatsByMessageId} activeThreadEnvironmentId={activeThread.environmentId} routeThreadKey={routeThreadKey} onOpenTurnDiff={onOpenTurnDiff} diff --git a/apps/web/src/components/chat/MessagesTimeline.browser.tsx b/apps/web/src/components/chat/MessagesTimeline.browser.tsx index 0eb5c8a1fc0..6a12cb25547 100644 --- a/apps/web/src/components/chat/MessagesTimeline.browser.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.browser.tsx @@ -61,6 +61,7 @@ function buildProps() { routeThreadKey: "environment-local:thread-1", onOpenTurnDiff: vi.fn(), revertTurnCountByUserMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), onRevertUserMessage: vi.fn(), isRevertingCheckpoint: false, onImageExpand: vi.fn(), @@ -157,4 +158,57 @@ describe("MessagesTimeline", () => { await screen.unmount(); } }); + + it("renders a compact assistant turn stats footer and omits missing metrics", async () => { + const assistantTurnStatsByMessageId = new Map([ + [ + "assistant-1", + { + summaryLabel: "gpt-5.4 (Low). 321 tokens. 1 tool call", + items: [ + { id: "model", label: "gpt-5.4 (Low)" }, + { id: "tokens", label: "321 tokens" }, + { id: "tools", label: "1 tool call" }, + ], + }, + ], + ]); + const screen = await render( + , + ); + + try { + await expect.element(page.getByText("Completed response")).toBeVisible(); + await expect.element(page.getByText("gpt-5.4 (Low)")).toBeVisible(); + await expect.element(page.getByText("321 tokens")).toBeVisible(); + await expect.element(page.getByText("1 tool call")).toBeVisible(); + await expect.element(page.getByText("0 tok/sec")).not.toBeInTheDocument(); + await expect + .element( + page.getByLabelText("Assistant turn stats: gpt-5.4 (Low). 321 tokens. 1 tool call"), + ) + .toBeVisible(); + } finally { + await screen.unmount(); + } + }); }); diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts index 633fb5d6bef..27596cec036 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.test.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { type AssistantTurnStats } from "../../lib/turnStats"; import { computeStableMessagesTimelineRows, computeMessageDurationStart, @@ -7,6 +8,15 @@ import { resolveAssistantMessageCopyState, } from "./MessagesTimeline.logic"; +const ASSISTANT_TURN_STATS: AssistantTurnStats = { + summaryLabel: "gpt-5.4 (High). 321 tokens. 1 tool call", + items: [ + { id: "model", label: "gpt-5.4 (High)" }, + { id: "tokens", label: "321 tokens" }, + { id: "tools", label: "1 tool call" }, + ], +}; + describe("computeMessageDurationStart", () => { it("returns message createdAt when there is no preceding user message", () => { const result = computeMessageDurationStart([ @@ -254,6 +264,7 @@ describe("deriveMessagesTimelineRows", () => { isWorking: false, activeTurnStartedAt: null, turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), revertTurnCountByUserMessageId: new Map(), }); @@ -313,6 +324,7 @@ describe("deriveMessagesTimelineRows", () => { turnDiffSummaryByAssistantMessageId: new Map([ ["assistant-1" as never, assistantTurnDiffSummary], ]), + assistantTurnStatsByMessageId: new Map([["assistant-1" as never, ASSISTANT_TURN_STATS]]), revertTurnCountByUserMessageId: new Map([["user-1" as never, 1]]), }); @@ -327,6 +339,56 @@ describe("deriveMessagesTimelineRows", () => { expect(userRow?.revertTurnCount).toBe(1); expect(assistantRow?.assistantTurnDiffSummary).toBe(assistantTurnDiffSummary); + expect(assistantRow?.assistantTurnStats).toBe(ASSISTANT_TURN_STATS); + }); + + it("attaches assistant turn stats only to the targeted assistant row", () => { + const rows = deriveMessagesTimelineRows({ + timelineEntries: [ + { + id: "assistant-old-entry", + kind: "message", + createdAt: "2026-01-01T00:00:05Z", + message: { + id: "assistant-old" as never, + role: "assistant", + text: "Older reply", + turnId: "turn-0" as never, + createdAt: "2026-01-01T00:00:05Z", + completedAt: "2026-01-01T00:00:06Z", + streaming: false, + }, + }, + { + id: "assistant-new-entry", + kind: "message", + createdAt: "2026-01-01T00:00:20Z", + message: { + id: "assistant-new" as never, + role: "assistant", + text: "Latest reply", + turnId: "turn-1" as never, + createdAt: "2026-01-01T00:00:20Z", + completedAt: "2026-01-01T00:00:30Z", + streaming: false, + }, + }, + ], + completionDividerBeforeEntryId: null, + isWorking: false, + activeTurnStartedAt: null, + turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map([["assistant-new" as never, ASSISTANT_TURN_STATS]]), + revertTurnCountByUserMessageId: new Map(), + }); + + const assistantRows = rows.filter( + (row): row is Extract<(typeof rows)[number], { kind: "message" }> => + row.kind === "message" && row.message.role === "assistant", + ); + + expect(assistantRows[0]?.assistantTurnStats).toBeUndefined(); + expect(assistantRows[1]?.assistantTurnStats).toBe(ASSISTANT_TURN_STATS); }); }); @@ -368,6 +430,7 @@ describe("computeStableMessagesTimelineRows", () => { isWorking: false, activeTurnStartedAt: null, turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), revertTurnCountByUserMessageId: new Map(), }); @@ -418,6 +481,7 @@ describe("computeStableMessagesTimelineRows", () => { isWorking: false, activeTurnStartedAt: null, turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), revertTurnCountByUserMessageId: new Map(), }); @@ -473,6 +537,7 @@ describe("computeStableMessagesTimelineRows", () => { isWorking: false, activeTurnStartedAt: null, turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), revertTurnCountByUserMessageId: new Map(), }); diff --git a/apps/web/src/components/chat/MessagesTimeline.logic.ts b/apps/web/src/components/chat/MessagesTimeline.logic.ts index c99f0b98a66..b0d9879c8c6 100644 --- a/apps/web/src/components/chat/MessagesTimeline.logic.ts +++ b/apps/web/src/components/chat/MessagesTimeline.logic.ts @@ -2,6 +2,7 @@ import * as Equal from "effect/Equal"; import { type TimelineEntry, type WorkLogEntry } from "../../session-logic"; import { type ChatMessage, type ProposedPlan, type TurnDiffSummary } from "../../types"; import { type MessageId } from "@t3tools/contracts"; +import { type AssistantTurnStats } from "../../lib/turnStats"; export const MAX_VISIBLE_WORK_LOG_ENTRIES = 6; @@ -28,6 +29,7 @@ export type MessagesTimelineRow = showCompletionDivider: boolean; showAssistantCopyButton: boolean; assistantTurnDiffSummary?: TurnDiffSummary | undefined; + assistantTurnStats?: AssistantTurnStats | undefined; revertTurnCount?: number | undefined; } | { @@ -114,6 +116,7 @@ export function deriveMessagesTimelineRows(input: { isWorking: boolean; activeTurnStartedAt: string | null; turnDiffSummaryByAssistantMessageId: ReadonlyMap; + assistantTurnStatsByMessageId: ReadonlyMap; revertTurnCountByUserMessageId: ReadonlyMap; }): MessagesTimelineRow[] { const nextRows: MessagesTimelineRow[] = []; @@ -174,6 +177,10 @@ export function deriveMessagesTimelineRows(input: { timelineEntry.message.role === "assistant" ? input.turnDiffSummaryByAssistantMessageId.get(timelineEntry.message.id) : undefined, + assistantTurnStats: + timelineEntry.message.role === "assistant" + ? input.assistantTurnStatsByMessageId.get(timelineEntry.message.id) + : undefined, revertTurnCount: timelineEntry.message.role === "user" ? input.revertTurnCountByUserMessageId.get(timelineEntry.message.id) @@ -234,6 +241,7 @@ function isRowUnchanged(a: MessagesTimelineRow, b: MessagesTimelineRow): boolean a.showCompletionDivider === bm.showCompletionDivider && a.showAssistantCopyButton === bm.showAssistantCopyButton && a.assistantTurnDiffSummary === bm.assistantTurnDiffSummary && + a.assistantTurnStats === bm.assistantTurnStats && a.revertTurnCount === bm.revertTurnCount ); } diff --git a/apps/web/src/components/chat/MessagesTimeline.test.tsx b/apps/web/src/components/chat/MessagesTimeline.test.tsx index a8a53831a2c..978961700b9 100644 --- a/apps/web/src/components/chat/MessagesTimeline.test.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.test.tsx @@ -84,6 +84,7 @@ function buildProps() { completionDividerBeforeEntryId: null, completionSummary: null, turnDiffSummaryByAssistantMessageId: new Map(), + assistantTurnStatsByMessageId: new Map(), routeThreadKey: "environment-local:thread-1", onOpenTurnDiff: () => {}, revertTurnCountByUserMessageId: new Map(), diff --git a/apps/web/src/components/chat/MessagesTimeline.tsx b/apps/web/src/components/chat/MessagesTimeline.tsx index 2ae24a8c130..1225bc98d08 100644 --- a/apps/web/src/components/chat/MessagesTimeline.tsx +++ b/apps/web/src/components/chat/MessagesTimeline.tsx @@ -19,6 +19,7 @@ import { LegendList, type LegendListRef } from "@legendapp/list/react"; import { deriveTimelineEntries, formatElapsed } from "../../session-logic"; import { type TurnDiffSummary } from "../../types"; import { summarizeTurnDiffStats } from "../../lib/turnDiffTree"; +import { type AssistantTurnStats } from "../../lib/turnStats"; import ChatMarkdown from "../ChatMarkdown"; import { BotIcon, @@ -40,6 +41,7 @@ import { ProposedPlanCard } from "./ProposedPlanCard"; import { ChangedFilesTree } from "./ChangedFilesTree"; import { DiffStatLabel, hasNonZeroStat } from "./DiffStatLabel"; import { MessageCopyButton } from "./MessageCopyButton"; +import { TurnStatsFooter } from "./TurnStatsFooter"; import { computeStableMessagesTimelineRows, MAX_VISIBLE_WORK_LOG_ENTRIES, @@ -101,6 +103,8 @@ const TimelineRowActivityCtx = createContext(null!); const TIMELINE_LIST_HEADER =
; const TIMELINE_LIST_FOOTER =
; const EMPTY_TIMELINE_SKILLS: ReadonlyArray> = []; +const EMPTY_ASSISTANT_TURN_STATS_BY_MESSAGE_ID: ReadonlyMap = + new Map(); // --------------------------------------------------------------------------- // Props (public API) @@ -115,10 +119,11 @@ interface MessagesTimelineProps { timelineEntries: ReturnType; completionDividerBeforeEntryId: string | null; completionSummary: string | null; - turnDiffSummaryByAssistantMessageId: Map; + turnDiffSummaryByAssistantMessageId: ReadonlyMap; + assistantTurnStatsByMessageId?: ReadonlyMap; routeThreadKey: string; onOpenTurnDiff: (turnId: TurnId, filePath?: string) => void; - revertTurnCountByUserMessageId: Map; + revertTurnCountByUserMessageId: ReadonlyMap; onRevertUserMessage: (messageId: MessageId) => void; isRevertingCheckpoint: boolean; onImageExpand: (preview: ExpandedImagePreview) => void; @@ -145,6 +150,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ completionDividerBeforeEntryId, completionSummary, turnDiffSummaryByAssistantMessageId, + assistantTurnStatsByMessageId = EMPTY_ASSISTANT_TURN_STATS_BY_MESSAGE_ID, routeThreadKey, onOpenTurnDiff, revertTurnCountByUserMessageId, @@ -167,6 +173,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ isWorking, activeTurnStartedAt, turnDiffSummaryByAssistantMessageId, + assistantTurnStatsByMessageId, revertTurnCountByUserMessageId, }), [ @@ -175,6 +182,7 @@ export const MessagesTimeline = memo(function MessagesTimeline({ isWorking, activeTurnStartedAt, turnDiffSummaryByAssistantMessageId, + assistantTurnStatsByMessageId, revertTurnCountByUserMessageId, ], ); @@ -444,6 +452,7 @@ function AssistantTimelineRow({ row }: { row: Extract
+ {row.assistantTurnStats ? : null}
); diff --git a/apps/web/src/components/chat/TurnStatsFooter.tsx b/apps/web/src/components/chat/TurnStatsFooter.tsx new file mode 100644 index 00000000000..c1178d452c9 --- /dev/null +++ b/apps/web/src/components/chat/TurnStatsFooter.tsx @@ -0,0 +1,33 @@ +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip"; +import { type AssistantTurnStats } from "../../lib/turnStats"; + +export function TurnStatsFooter(props: { stats: AssistantTurnStats }) { + return ( +
+ {props.stats.items.map((item, index) => ( + + {index > 0 ? : null} + {item.tooltip ? ( + + + {item.label} + + +

{item.tooltip}

+ + + ) : ( + {item.label} + )} + + ))} +
+ ); +} diff --git a/apps/web/src/lib/contextWindow.test.ts b/apps/web/src/lib/contextWindow.test.ts index 70479b3fa43..4155f37c25b 100644 --- a/apps/web/src/lib/contextWindow.test.ts +++ b/apps/web/src/lib/contextWindow.test.ts @@ -1,9 +1,19 @@ import { describe, expect, it } from "vitest"; import { EventId, type OrchestrationThreadActivity, TurnId } from "@t3tools/contracts"; -import { deriveLatestContextWindowSnapshot, formatContextWindowTokens } from "./contextWindow"; +import { + deriveLatestContextWindowSnapshot, + deriveLatestContextWindowSnapshotForTurn, + deriveLatestUnassignedContextWindowSnapshotSince, + formatContextWindowTokens, +} from "./contextWindow"; -function makeActivity(id: string, kind: string, payload: unknown): OrchestrationThreadActivity { +function makeActivity( + id: string, + kind: string, + payload: unknown, + overrides?: Partial, +): OrchestrationThreadActivity { return { id: EventId.make(id), tone: "info", @@ -12,6 +22,7 @@ function makeActivity(id: string, kind: string, payload: unknown): Orchestration payload, turnId: TurnId.make("turn-1"), createdAt: "2026-03-23T00:00:00.000Z", + ...overrides, }; } @@ -64,4 +75,70 @@ describe("contextWindow", () => { expect(snapshot?.usedTokens).toBe(81_659); expect(snapshot?.totalProcessedTokens).toBe(748_126); }); + + it("preserves explicit generation duration when available", () => { + const snapshot = deriveLatestContextWindowSnapshot([ + makeActivity("activity-1", "context-window.updated", { + usedTokens: 81_659, + lastOutputTokens: 1_790, + durationMs: 25_000, + timeToFirstTokenMs: 4_300, + }), + ]); + + expect(snapshot?.lastOutputTokens).toBe(1_790); + expect(snapshot?.durationMs).toBe(25_000); + expect(snapshot?.timeToFirstTokenMs).toBe(4_300); + }); + + it("can select the latest valid context window snapshot for a specific turn", () => { + const snapshot = deriveLatestContextWindowSnapshotForTurn( + [ + { + ...makeActivity("activity-1", "context-window.updated", { + usedTokens: 8_000, + lastOutputTokens: 900, + }), + turnId: TurnId.make("turn-0"), + }, + makeActivity("activity-2", "context-window.updated", { + usedTokens: 4_000, + lastOutputTokens: 321, + }), + ], + TurnId.make("turn-1"), + ); + + expect(snapshot?.usedTokens).toBe(4_000); + expect(snapshot?.lastOutputTokens).toBe(321); + }); + + it("can select the latest unassigned context window snapshot after a turn starts", () => { + const snapshot = deriveLatestUnassignedContextWindowSnapshotSince( + [ + makeActivity( + "activity-1", + "context-window.updated", + { usedTokens: 9_000, lastOutputTokens: 900 }, + { turnId: null, createdAt: "2026-03-22T23:59:59.000Z" }, + ), + makeActivity( + "activity-2", + "context-window.updated", + { usedTokens: 4_000, lastOutputTokens: 321 }, + { turnId: null, createdAt: "2026-03-23T00:00:10.000Z" }, + ), + makeActivity( + "activity-3", + "context-window.updated", + { usedTokens: 5_000, lastOutputTokens: 654 }, + { turnId: TurnId.make("turn-2"), createdAt: "2026-03-23T00:00:11.000Z" }, + ), + ], + "2026-03-23T00:00:00.000Z", + ); + + expect(snapshot?.usedTokens).toBe(4_000); + expect(snapshot?.lastOutputTokens).toBe(321); + }); }); diff --git a/apps/web/src/lib/contextWindow.ts b/apps/web/src/lib/contextWindow.ts index f668135a13a..11782b40475 100644 --- a/apps/web/src/lib/contextWindow.ts +++ b/apps/web/src/lib/contextWindow.ts @@ -27,10 +27,46 @@ export type ContextWindowSnapshot = NullableContextWindowUsage & { export function deriveLatestContextWindowSnapshot( activities: ReadonlyArray, +): ContextWindowSnapshot | null { + return deriveLatestContextWindowSnapshotForTurn(activities); +} + +export function deriveLatestContextWindowSnapshotForTurn( + activities: ReadonlyArray, + turnId?: OrchestrationThreadActivity["turnId"], +): ContextWindowSnapshot | null { + return findLatestContextWindowSnapshot(activities, (activity) => + turnId !== undefined ? activity.turnId === turnId : true, + ); +} + +export function deriveLatestUnassignedContextWindowSnapshotSince( + activities: ReadonlyArray, + startedAt: string | null | undefined, +): ContextWindowSnapshot | null { + if (!startedAt) { + return null; + } + const startedAtMs = Date.parse(startedAt); + if (!Number.isFinite(startedAtMs)) { + return null; + } + return findLatestContextWindowSnapshot(activities, (activity) => { + if (activity.turnId !== null && activity.turnId !== undefined) { + return false; + } + const activityAtMs = Date.parse(activity.createdAt); + return Number.isFinite(activityAtMs) && activityAtMs >= startedAtMs; + }); +} + +function findLatestContextWindowSnapshot( + activities: ReadonlyArray, + matchesActivity: (activity: OrchestrationThreadActivity) => boolean, ): ContextWindowSnapshot | null { for (let index = activities.length - 1; index >= 0; index -= 1) { const activity = activities[index]; - if (!activity || activity.kind !== "context-window.updated") { + if (!activity || activity.kind !== "context-window.updated" || !matchesActivity(activity)) { continue; } @@ -65,6 +101,7 @@ export function deriveLatestContextWindowSnapshot( lastReasoningOutputTokens: asFiniteNumber(payload?.lastReasoningOutputTokens), toolUses: asFiniteNumber(payload?.toolUses), durationMs: asFiniteNumber(payload?.durationMs), + timeToFirstTokenMs: asFiniteNumber(payload?.timeToFirstTokenMs), compactsAutomatically: asBoolean(payload?.compactsAutomatically) ?? false, updatedAt: activity.createdAt, }; diff --git a/apps/web/src/lib/turnStats.test.ts b/apps/web/src/lib/turnStats.test.ts new file mode 100644 index 00000000000..f102c3f7977 --- /dev/null +++ b/apps/web/src/lib/turnStats.test.ts @@ -0,0 +1,456 @@ +import { describe, expect, it } from "vitest"; +import { + EventId, + ProviderInstanceId, + TurnId, + type ModelSelection, + type OrchestrationLatestTurn, + type OrchestrationThreadActivity, +} from "@t3tools/contracts"; +import { createModelSelection } from "@t3tools/shared/model"; + +import type { ChatMessage } from "../types"; +import { + buildLatestAssistantTurnStatsMap, + deriveLatestAssistantTurnStats, + formatAssistantTurnStatTokenCount, +} from "./turnStats"; + +const TURN_ID = TurnId.make("turn-1"); + +function makeLatestTurn(overrides?: Partial): OrchestrationLatestTurn { + return { + turnId: TURN_ID, + state: "completed", + requestedAt: "2026-05-07T23:00:00.000Z", + startedAt: "2026-05-07T23:00:01.000Z", + completedAt: "2026-05-07T23:00:11.000Z", + assistantMessageId: "assistant-1" as never, + ...overrides, + }; +} + +function makeAssistantMessage(overrides?: Partial): ChatMessage { + return { + id: "assistant-1" as never, + role: "assistant", + text: "Done.", + turnId: TURN_ID, + createdAt: "2026-05-07T23:00:03.000Z", + completedAt: "2026-05-07T23:00:11.000Z", + streaming: false, + ...overrides, + }; +} + +function makeActivity(input: { + id: string; + kind: string; + payload: unknown; + turnId?: OrchestrationThreadActivity["turnId"]; + createdAt?: string; +}): OrchestrationThreadActivity { + return { + id: EventId.make(input.id), + tone: input.kind.startsWith("tool.") ? "tool" : "info", + kind: input.kind, + summary: input.kind, + payload: input.payload, + turnId: input.turnId === undefined ? TURN_ID : input.turnId, + createdAt: input.createdAt ?? "2026-05-07T23:00:10.000Z", + }; +} + +function makeModelSelection(options?: ModelSelection["options"]): ModelSelection { + return createModelSelection(ProviderInstanceId.make("codex"), "gpt-5.4", options); +} + +describe("turnStats", () => { + it("derives a compact stats row for the latest completed assistant turn", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "tool-started-1", + kind: "tool.started", + payload: { data: { toolCallId: "tool-1" } }, + }), + makeActivity({ + id: "context-window-1", + kind: "context-window.updated", + payload: { + usedTokens: 9_999, + lastOutputTokens: 3_276, + totalProcessedTokens: 8_192, + durationMs: 5_537, + timeToFirstTokenMs: 2_000, + }, + }), + ], + modelSelection: makeModelSelection([{ id: "reasoningEffort", value: "high" }]), + }); + + expect(stats).not.toBeNull(); + expect(stats?.items.map((item) => item.label)).toEqual([ + "gpt-5.4 (High)", + "10 sec", + "3,276 tokens", + "592 tok/sec", + "Time-to-first: 2 sec", + "1 tool call", + ]); + expect(stats?.items.find((item) => item.id === "throughput")?.tooltip).toContain( + "Approximate throughput", + ); + expect(stats?.items.find((item) => item.id === "ttft")?.tooltip).toContain( + "Approximate time-to-first", + ); + }); + + it("hides unavailable metrics instead of fabricating zeros", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "context-window-1", + kind: "context-window.updated", + payload: { usedTokens: 10 }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.map((item) => item.label)).toEqual(["10 sec"]); + expect(stats?.summaryLabel).not.toContain("0 tok/sec"); + expect(stats?.summaryLabel).not.toContain("0 tool calls"); + }); + + it("does not derive throughput or TTFT from whole-turn elapsed time", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-07T23:00:00.000Z", + completedAt: "2026-05-07T23:04:44.000Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-07T23:04:43.500Z", + completedAt: "2026-05-07T23:04:44.000Z", + }), + activities: [ + makeActivity({ + id: "context-window-1", + kind: "context-window.updated", + payload: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.map((item) => item.label)).toEqual(["4 min 44 sec", "1,790 tokens"]); + expect(stats?.items.find((item) => item.id === "throughput")).toBeUndefined(); + expect(stats?.items.find((item) => item.id === "ttft")).toBeUndefined(); + expect(stats?.summaryLabel).not.toContain("6.3 tok/sec"); + expect(stats?.summaryLabel).not.toContain("Time-to-first"); + }); + + it("derives throughput only from explicit positive generation duration", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-07T23:00:00.000Z", + completedAt: "2026-05-07T23:04:44.000Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-07T23:04:19.000Z", + completedAt: "2026-05-07T23:04:44.000Z", + }), + activities: [ + makeActivity({ + id: "context-window-1", + kind: "context-window.updated", + payload: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 25_000, + timeToFirstTokenMs: 259_000, + }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.map((item) => item.label)).toEqual([ + "4 min 44 sec", + "1,790 tokens", + "71.6 tok/sec", + "Time-to-first: 4 min 19 sec", + ]); + expect(stats?.items.find((item) => item.id === "throughput")?.label).toBe("71.6 tok/sec"); + expect(stats?.summaryLabel).not.toContain("6.3 tok/sec"); + }); + + it("formats throughput from corrected response-boundary duration", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-08T12:00:00.000Z", + completedAt: "2026-05-08T12:00:06.700Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-08T12:00:06.186Z", + completedAt: "2026-05-08T12:00:06.700Z", + }), + activities: [ + makeActivity({ + id: "context-window-tiny", + kind: "context-window.updated", + createdAt: "2026-05-08T12:00:06.700Z", + payload: { + usedTokens: 72_193, + lastOutputTokens: 16, + durationMs: 514, + timeToFirstTokenMs: 6_186, + }, + }), + ], + modelSelection: makeModelSelection([{ id: "reasoningEffort", value: "medium" }]), + }); + + expect(stats?.items.map((item) => item.label)).toEqual([ + "gpt-5.4 (Medium)", + "6.7 sec", + "16 tokens", + "31.1 tok/sec", + "Time-to-first: 6.2 sec", + ]); + expect(stats?.summaryLabel).not.toContain("1,778 tok/sec"); + }); + + it("omits observed bad TTFT examples when explicit first-token timing is missing", () => { + const longTurn = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-08T08:15:13.000Z", + completedAt: "2026-05-08T08:17:30.000Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-08T08:17:30.000Z", + completedAt: "2026-05-08T08:17:30.000Z", + }), + activities: [ + makeActivity({ + id: "context-window-long", + kind: "context-window.updated", + payload: { + usedTokens: 66_957, + lastOutputTokens: 910, + }, + }), + ], + modelSelection: null, + }); + + const shortTurn = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-08T08:17:49.000Z", + completedAt: "2026-05-08T08:18:20.000Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-08T08:18:20.000Z", + completedAt: "2026-05-08T08:18:20.000Z", + }), + activities: [ + makeActivity({ + id: "context-window-short", + kind: "context-window.updated", + payload: { + usedTokens: 72_193, + lastOutputTokens: 895, + }, + }), + ], + modelSelection: null, + }); + + expect(longTurn?.items.find((item) => item.id === "ttft")).toBeUndefined(); + expect(longTurn?.summaryLabel).not.toContain("Time-to-first"); + expect(shortTurn?.items.find((item) => item.id === "ttft")).toBeUndefined(); + expect(shortTurn?.summaryLabel).not.toContain("Time-to-first"); + }); + + it("skips zero token fields and falls back to the first positive token count", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "context-window-1", + kind: "context-window.updated", + payload: { + usedTokens: 10, + lastOutputTokens: 0, + lastUsedTokens: 420, + totalProcessedTokens: 900, + durationMs: 210_000, + }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.find((item) => item.id === "tokens")?.label).toBe("420 tokens"); + expect(stats?.items.find((item) => item.id === "throughput")?.label).toBe("2 tok/sec"); + expect(stats?.items.some((item) => item.id === "tokens" && item.label === "0 tokens")).toBe( + false, + ); + }); + + it("falls back to an unassigned context window snapshot from the completed turn window", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-07T23:00:00.000Z", + completedAt: "2026-05-07T23:03:39.000Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-07T23:03:39.000Z", + completedAt: "2026-05-07T23:03:39.000Z", + }), + activities: [ + makeActivity({ + id: "context-window-old", + kind: "context-window.updated", + turnId: null, + createdAt: "2026-05-07T22:59:59.000Z", + payload: { usedTokens: 20_000, lastOutputTokens: 99_999 }, + }), + makeActivity({ + id: "context-window-unassigned", + kind: "context-window.updated", + turnId: null, + createdAt: "2026-05-07T23:03:40.000Z", + payload: { usedTokens: 4_000, lastOutputTokens: 438 }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.map((item) => item.label)).toEqual(["3 min 39 sec", "438 tokens"]); + expect(stats?.items.find((item) => item.id === "throughput")).toBeUndefined(); + expect(stats?.items.find((item) => item.id === "ttft")).toBeUndefined(); + }); + + it("returns null for incomplete or mismatched assistant turns", () => { + expect( + deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ state: "running", completedAt: null }), + assistantMessage: makeAssistantMessage(), + activities: [], + modelSelection: makeModelSelection(), + }), + ).toBeNull(); + + expect( + deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage({ turnId: TurnId.make("turn-2") }), + activities: [], + modelSelection: makeModelSelection(), + }), + ).toBeNull(); + }); + + it("falls back to completed or updated tool lifecycle events when starts are missing", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "tool-updated-1", + kind: "tool.updated", + payload: { data: { toolCallId: "tool-1" } }, + }), + makeActivity({ + id: "tool-completed-1", + kind: "tool.completed", + payload: { data: { toolCallId: "tool-2" } }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.find((item) => item.id === "tools")?.label).toBe("2 tool calls"); + }); + + it("counts repeated same-title tool starts as distinct tool calls", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "tool-started-1", + kind: "tool.started", + payload: { itemType: "tool_call" }, + }), + makeActivity({ + id: "tool-started-2", + kind: "tool.started", + payload: { itemType: "tool_call" }, + }), + makeActivity({ + id: "tool-started-3", + kind: "tool.started", + payload: { itemType: "tool_call" }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.find((item) => item.id === "tools")?.label).toBe("3 tool calls"); + }); + + it("filters context window snapshots by turn id", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn(), + assistantMessage: makeAssistantMessage(), + activities: [ + makeActivity({ + id: "context-window-old", + kind: "context-window.updated", + turnId: TurnId.make("turn-0"), + payload: { usedTokens: 20_000, lastOutputTokens: 99_999 }, + }), + makeActivity({ + id: "context-window-current", + kind: "context-window.updated", + payload: { usedTokens: 4_000, lastOutputTokens: 321 }, + }), + ], + modelSelection: null, + }); + + expect(stats?.items.find((item) => item.id === "tokens")?.label).toBe("321 tokens"); + }); + + it("builds a message-id keyed stats map only for renderable latest-turn stats", () => { + const assistantMessage = makeAssistantMessage(); + const statsByMessageId = buildLatestAssistantTurnStatsMap({ + latestTurn: makeLatestTurn(), + assistantMessage, + activities: [ + makeActivity({ + id: "context-window-current", + kind: "context-window.updated", + payload: { usedTokens: 4_000, lastOutputTokens: 321 }, + }), + ], + modelSelection: null, + }); + + expect(statsByMessageId.get(assistantMessage.id)?.items[0]?.label).toBe("10 sec"); + }); + + it("formats compact token count labels", () => { + expect(formatAssistantTurnStatTokenCount(1_400)).toBe("1.4k tokens"); + }); +}); diff --git a/apps/web/src/lib/turnStats.ts b/apps/web/src/lib/turnStats.ts new file mode 100644 index 00000000000..f0fca3431dc --- /dev/null +++ b/apps/web/src/lib/turnStats.ts @@ -0,0 +1,279 @@ +import type { + ModelSelection, + OrchestrationLatestTurn, + OrchestrationThreadActivity, +} from "@t3tools/contracts"; +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; + +import type { ChatMessage } from "../types"; +import { + deriveLatestContextWindowSnapshotForTurn, + deriveLatestUnassignedContextWindowSnapshotSince, + formatContextWindowTokens, + type ContextWindowSnapshot, +} from "./contextWindow"; + +export interface AssistantTurnStatItem { + readonly id: string; + readonly label: string; + readonly tooltip?: string | undefined; +} + +export interface AssistantTurnStats { + readonly summaryLabel: string; + readonly items: ReadonlyArray; +} + +type CompletedTurn = Pick< + OrchestrationLatestTurn, + "turnId" | "state" | "startedAt" | "completedAt" +>; + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" ? (value as Record) : null; +} + +function asTrimmedString(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function toPositiveFiniteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; +} + +function countSingularPlural(value: number, singular: string, plural = `${singular}s`): string { + return `${value.toLocaleString()} ${value === 1 ? singular : plural}`; +} + +function formatDurationSeconds(value: number): string { + if (value >= 60) { + const totalSeconds = Math.round(value); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + const minuteLabel = `${minutes.toLocaleString()} min`; + return seconds > 0 ? `${minuteLabel} ${seconds.toLocaleString()} sec` : minuteLabel; + } + if (value >= 10) { + return `${Math.round(value)} sec`; + } + return `${value.toFixed(1).replace(/\.0$/, "")} sec`; +} + +function formatTokensPerSecond(value: number): string { + if (value >= 100) { + return `${Math.round(value).toLocaleString()} tok/sec`; + } + if (value >= 10) { + return `${value.toFixed(1).replace(/\.0$/, "")} tok/sec`; + } + return `${value.toFixed(2).replace(/0+$/, "").replace(/\.$/, "")} tok/sec`; +} + +function formatEffortLabel(value: string | null): string | null { + if (!value) { + return null; + } + const compact = value.replace(/[_-]+/g, " ").trim(); + if (!compact) { + return null; + } + if (/^x?high$/i.test(compact)) { + return compact.length === 5 ? "XHigh" : "High"; + } + return compact + .split(/\s+/) + .map((segment) => `${segment.charAt(0).toUpperCase()}${segment.slice(1)}`) + .join(" "); +} + +function formatModelLabel(modelSelection: ModelSelection | null | undefined): string | null { + const model = asTrimmedString(modelSelection?.model); + if (!model) { + return null; + } + const effort = formatEffortLabel( + getModelSelectionStringOptionValue(modelSelection, "reasoningEffort") ?? + getModelSelectionStringOptionValue(modelSelection, "effort") ?? + null, + ); + return effort ? `${model} (${effort})` : model; +} + +function deriveElapsedMs(latestTurn: CompletedTurn): number | null { + if (!latestTurn.startedAt || !latestTurn.completedAt) { + return null; + } + const startedAt = Date.parse(latestTurn.startedAt); + const completedAt = Date.parse(latestTurn.completedAt); + if (!Number.isFinite(startedAt) || !Number.isFinite(completedAt) || completedAt <= startedAt) { + return null; + } + return completedAt - startedAt; +} + +function deriveDisplayTokenCount(snapshot: ContextWindowSnapshot | null): number | null { + if (!snapshot) { + return null; + } + return ( + toPositiveFiniteNumber(snapshot.lastOutputTokens) ?? + toPositiveFiniteNumber(snapshot.lastUsedTokens) ?? + toPositiveFiniteNumber(snapshot.totalProcessedTokens) + ); +} + +function extractToolIdentity( + activity: OrchestrationThreadActivity, + options?: { includeDisplayFallback?: boolean }, +): string | null { + const payload = asRecord(activity.payload); + const data = asRecord(payload?.data); + return ( + asTrimmedString(data?.toolCallId) ?? + asTrimmedString(payload?.itemId) ?? + asTrimmedString(data?.itemId) ?? + (options?.includeDisplayFallback + ? (asTrimmedString(payload?.title) ?? asTrimmedString(activity.summary)) + : null) ?? + null + ); +} + +function countToolCalls( + activities: ReadonlyArray, + turnId: OrchestrationThreadActivity["turnId"], + snapshot: ContextWindowSnapshot | null, +): number | null { + const relevant = activities.filter((activity) => activity.turnId === turnId); + const startedKeys = new Set(); + for (const activity of relevant) { + if (activity.kind !== "tool.started") { + continue; + } + const key = extractToolIdentity(activity); + startedKeys.add(key ?? activity.id); + } + if (startedKeys.size > 0) { + return startedKeys.size; + } + + const lifecycleKeys = new Set(); + for (const activity of relevant) { + if (activity.kind !== "tool.updated" && activity.kind !== "tool.completed") { + continue; + } + const key = extractToolIdentity(activity); + lifecycleKeys.add(key ?? activity.id); + } + if (lifecycleKeys.size > 0) { + return lifecycleKeys.size; + } + + const toolUses = toPositiveFiniteNumber(snapshot?.toolUses); + return toolUses; +} + +export function deriveLatestAssistantTurnStats(input: { + latestTurn: CompletedTurn | null; + assistantMessage: ChatMessage | null; + activities: ReadonlyArray; + modelSelection: ModelSelection | null | undefined; +}): AssistantTurnStats | null { + const { latestTurn, assistantMessage, activities, modelSelection } = input; + if ( + !latestTurn || + latestTurn.state !== "completed" || + !latestTurn.turnId || + !assistantMessage || + assistantMessage.role !== "assistant" || + assistantMessage.streaming || + assistantMessage.turnId !== latestTurn.turnId + ) { + return null; + } + + const snapshot = + deriveLatestContextWindowSnapshotForTurn(activities, latestTurn.turnId) ?? + deriveLatestUnassignedContextWindowSnapshotSince(activities, latestTurn.startedAt); + const items: AssistantTurnStatItem[] = []; + + const modelLabel = formatModelLabel(modelSelection); + if (modelLabel) { + items.push({ id: "model", label: modelLabel }); + } + + const elapsedMs = deriveElapsedMs(latestTurn); + if (elapsedMs !== null) { + items.push({ + id: "elapsed", + label: formatDurationSeconds(elapsedMs / 1_000), + }); + } + + const tokenCount = deriveDisplayTokenCount(snapshot); + if (tokenCount !== null) { + items.push({ + id: "tokens", + label: countSingularPlural(tokenCount, "token"), + }); + } + + const throughputDurationMs = toPositiveFiniteNumber(snapshot?.durationMs); + if (tokenCount !== null && throughputDurationMs !== null) { + const tokensPerSecond = tokenCount / (throughputDurationMs / 1_000); + if (Number.isFinite(tokensPerSecond) && tokensPerSecond > 0) { + items.push({ + id: "throughput", + label: formatTokensPerSecond(tokensPerSecond), + tooltip: "Approximate throughput based on provider token counts and generation duration.", + }); + } + } + + const timeToFirstTokenMs = toPositiveFiniteNumber(snapshot?.timeToFirstTokenMs); + if (timeToFirstTokenMs !== null) { + items.push({ + id: "ttft", + label: `Time-to-first: ${formatDurationSeconds(timeToFirstTokenMs / 1_000)}`, + tooltip: "Approximate time-to-first based on first assistant output timing.", + }); + } + + const toolCalls = countToolCalls(activities, latestTurn.turnId, snapshot); + if (toolCalls !== null && toolCalls > 0) { + items.push({ + id: "tools", + label: countSingularPlural(toolCalls, "tool call"), + }); + } + + if (items.length === 0) { + return null; + } + + return { + summaryLabel: items.map((item) => item.label).join(". "), + items, + }; +} + +export function buildLatestAssistantTurnStatsMap(input: { + latestTurn: CompletedTurn | null; + assistantMessage: ChatMessage | null; + activities: ReadonlyArray; + modelSelection: ModelSelection | null | undefined; +}): ReadonlyMap { + const stats = deriveLatestAssistantTurnStats(input); + if (!stats || !input.assistantMessage) { + return new Map(); + } + return new Map([[input.assistantMessage.id, stats]]); +} + +export function formatAssistantTurnStatTokenCount(value: number): string { + return `${formatContextWindowTokens(value)} tokens`; +} diff --git a/apps/web/src/proposedPlan.test.ts b/apps/web/src/proposedPlan.test.ts index 63f95d4137b..49827ea0f8a 100644 --- a/apps/web/src/proposedPlan.test.ts +++ b/apps/web/src/proposedPlan.test.ts @@ -102,9 +102,15 @@ describe("buildPlanImplementationThreadTitle", () => { }); describe("buildProposedPlanMarkdownFilename", () => { - it("derives a stable markdown filename from the plan heading", () => { + it("derives a root plan-artifact markdown filename from the plan heading", () => { expect(buildProposedPlanMarkdownFilename("# Integrate Effect RPC Into Server App")).toBe( - "integrate-effect-rpc-into-server-app.md", + "plan-integrate-effect-rpc-into-server-app.md", + ); + }); + + it("does not double-prefix headings that already start with plan", () => { + expect(buildProposedPlanMarkdownFilename("# PLAN: Integrate Effect RPC")).toBe( + "plan-integrate-effect-rpc.md", ); }); diff --git a/apps/web/src/proposedPlan.ts b/apps/web/src/proposedPlan.ts index 48186392e8a..6386b07cfbe 100644 --- a/apps/web/src/proposedPlan.ts +++ b/apps/web/src/proposedPlan.ts @@ -102,7 +102,10 @@ export function buildPlanImplementationThreadTitle(planMarkdown: string): string export function buildProposedPlanMarkdownFilename(planMarkdown: string): string { const title = proposedPlanTitle(planMarkdown); - return `${sanitizePlanFileSegment(title ?? "plan")}.md`; + const segment = sanitizePlanFileSegment(title ?? "plan"); + const planSegment = + segment === "plan" || segment.startsWith("plan-") ? segment : `plan-${segment}`; + return `${planSegment}.md`; } export function normalizePlanMarkdownForExport(planMarkdown: string): string { diff --git a/packages/contracts/src/providerRuntime.test.ts b/packages/contracts/src/providerRuntime.test.ts index 587c1879454..376988a8eda 100644 --- a/packages/contracts/src/providerRuntime.test.ts +++ b/packages/contracts/src/providerRuntime.test.ts @@ -170,6 +170,7 @@ describe("ProviderRuntimeEvent", () => { maxTokens: 200000, toolUses: 25, durationMs: 43567, + timeToFirstTokenMs: 4300, }, }, }); @@ -180,5 +181,6 @@ describe("ProviderRuntimeEvent", () => { } expect(parsed.payload.usage.maxTokens).toBe(200000); expect(parsed.payload.usage.usedTokens).toBe(31251); + expect(parsed.payload.usage.timeToFirstTokenMs).toBe(4300); }); }); diff --git a/packages/contracts/src/providerRuntime.ts b/packages/contracts/src/providerRuntime.ts index 5032dc4eb41..28e03185380 100644 --- a/packages/contracts/src/providerRuntime.ts +++ b/packages/contracts/src/providerRuntime.ts @@ -318,6 +318,7 @@ export const ThreadTokenUsageSnapshot = Schema.Struct({ lastReasoningOutputTokens: Schema.optional(NonNegativeInt), toolUses: Schema.optional(NonNegativeInt), durationMs: Schema.optional(NonNegativeInt), + timeToFirstTokenMs: Schema.optional(NonNegativeInt), compactsAutomatically: Schema.optional(Schema.Boolean), }); export type ThreadTokenUsageSnapshot = typeof ThreadTokenUsageSnapshot.Type; From e4338721de04a8a43a6bc638e3dcdbacdc7eacdb Mon Sep 17 00:00:00 2001 From: Coy Geek <65363919+coygeek@users.noreply.github.com> Date: Fri, 8 May 2026 09:41:05 -0700 Subject: [PATCH 2/4] fix(web): remove unused turn stats formatter --- apps/web/src/lib/turnStats.test.ts | 10 +--------- apps/web/src/lib/turnStats.ts | 5 ----- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/apps/web/src/lib/turnStats.test.ts b/apps/web/src/lib/turnStats.test.ts index f102c3f7977..13cde2d228d 100644 --- a/apps/web/src/lib/turnStats.test.ts +++ b/apps/web/src/lib/turnStats.test.ts @@ -10,11 +10,7 @@ import { import { createModelSelection } from "@t3tools/shared/model"; import type { ChatMessage } from "../types"; -import { - buildLatestAssistantTurnStatsMap, - deriveLatestAssistantTurnStats, - formatAssistantTurnStatTokenCount, -} from "./turnStats"; +import { buildLatestAssistantTurnStatsMap, deriveLatestAssistantTurnStats } from "./turnStats"; const TURN_ID = TurnId.make("turn-1"); @@ -449,8 +445,4 @@ describe("turnStats", () => { expect(statsByMessageId.get(assistantMessage.id)?.items[0]?.label).toBe("10 sec"); }); - - it("formats compact token count labels", () => { - expect(formatAssistantTurnStatTokenCount(1_400)).toBe("1.4k tokens"); - }); }); diff --git a/apps/web/src/lib/turnStats.ts b/apps/web/src/lib/turnStats.ts index f0fca3431dc..66b9a233111 100644 --- a/apps/web/src/lib/turnStats.ts +++ b/apps/web/src/lib/turnStats.ts @@ -9,7 +9,6 @@ import type { ChatMessage } from "../types"; import { deriveLatestContextWindowSnapshotForTurn, deriveLatestUnassignedContextWindowSnapshotSince, - formatContextWindowTokens, type ContextWindowSnapshot, } from "./contextWindow"; @@ -273,7 +272,3 @@ export function buildLatestAssistantTurnStatsMap(input: { } return new Map([[input.assistantMessage.id, stats]]); } - -export function formatAssistantTurnStatTokenCount(value: number): string { - return `${formatContextWindowTokens(value)} tokens`; -} From 69658e238160b728ae00adf4ce82d63fa55c2ad9 Mon Sep 17 00:00:00 2001 From: Coy Geek <65363919+coygeek@users.noreply.github.com> Date: Fri, 8 May 2026 09:50:27 -0700 Subject: [PATCH 3/4] fix(web): normalize rounded minute durations --- apps/web/src/lib/turnStats.test.ts | 18 ++++++++++++++++++ apps/web/src/lib/turnStats.ts | 6 +++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/apps/web/src/lib/turnStats.test.ts b/apps/web/src/lib/turnStats.test.ts index 13cde2d228d..36a95f7762b 100644 --- a/apps/web/src/lib/turnStats.test.ts +++ b/apps/web/src/lib/turnStats.test.ts @@ -123,6 +123,24 @@ describe("turnStats", () => { expect(stats?.summaryLabel).not.toContain("0 tool calls"); }); + it("formats rounded minute-boundary durations as minutes", () => { + const stats = deriveLatestAssistantTurnStats({ + latestTurn: makeLatestTurn({ + startedAt: "2026-05-07T23:00:00.000Z", + completedAt: "2026-05-07T23:00:59.500Z", + }), + assistantMessage: makeAssistantMessage({ + createdAt: "2026-05-07T23:00:59.500Z", + completedAt: "2026-05-07T23:00:59.500Z", + }), + activities: [], + modelSelection: null, + }); + + expect(stats?.items.map((item) => item.label)).toEqual(["1 min"]); + expect(stats?.summaryLabel).not.toContain("60 sec"); + }); + it("does not derive throughput or TTFT from whole-turn elapsed time", () => { const stats = deriveLatestAssistantTurnStats({ latestTurn: makeLatestTurn({ diff --git a/apps/web/src/lib/turnStats.ts b/apps/web/src/lib/turnStats.ts index 66b9a233111..f038c33cf01 100644 --- a/apps/web/src/lib/turnStats.ts +++ b/apps/web/src/lib/turnStats.ts @@ -49,15 +49,15 @@ function countSingularPlural(value: number, singular: string, plural = `${singul } function formatDurationSeconds(value: number): string { - if (value >= 60) { - const totalSeconds = Math.round(value); + const totalSeconds = Math.round(value); + if (totalSeconds >= 60) { const minutes = Math.floor(totalSeconds / 60); const seconds = totalSeconds % 60; const minuteLabel = `${minutes.toLocaleString()} min`; return seconds > 0 ? `${minuteLabel} ${seconds.toLocaleString()} sec` : minuteLabel; } if (value >= 10) { - return `${Math.round(value)} sec`; + return `${totalSeconds} sec`; } return `${value.toFixed(1).replace(/\.0$/, "")} sec`; } From 89169c22c74f8f9a2bfd11a9a7627a97bc589084 Mon Sep 17 00:00:00 2001 From: Coy Geek <65363919+coygeek@users.noreply.github.com> Date: Fri, 8 May 2026 20:27:35 -0700 Subject: [PATCH 4/4] fix(stats): exclude tool wall time from derived TPS Close Codex assistant response timing segments when tool work starts so fallback generation duration does not include command or file-change output time. Add regression coverage for a long command gap and clarify the TPS tooltip. Refs: #2518 --- .../Layers/ProviderRuntimeIngestion.test.ts | 108 ++++++++++++++++++ .../Layers/ProviderRuntimeIngestion.ts | 16 +++ apps/web/src/lib/turnStats.test.ts | 2 +- apps/web/src/lib/turnStats.ts | 3 +- 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts index 9be138e0173..9f2897f83f3 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts @@ -3067,6 +3067,114 @@ describe("ProviderRuntimeIngestion", () => { ).toBe(undefined); }); + it("excludes command execution wall time from fallback assistant duration", async () => { + const harness = await createHarness(); + const turnId = asTurnId("turn-throughput-command-gap"); + + harness.emit({ + type: "turn.started", + eventId: asEventId("evt-throughput-command-gap-turn-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:00.000Z", + threadId: asThreadId("thread-1"), + turnId, + }); + harness.emit({ + type: "content.delta", + eventId: asEventId("evt-throughput-command-gap-delta"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:01.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-command-gap-assistant"), + payload: { + streamKind: "assistant_text", + delta: "I will inspect that.", + }, + }); + harness.emit({ + type: "item.started", + eventId: asEventId("evt-throughput-command-gap-tool-started"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:00:02.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-command-gap-tool"), + payload: { + itemType: "command_execution", + status: "in_progress", + title: "Run command", + detail: "sleep 100", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-throughput-command-gap-tool-completed"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:01:42.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-command-gap-tool"), + payload: { + itemType: "command_execution", + title: "Run command", + detail: "done", + }, + }); + harness.emit({ + type: "item.completed", + eventId: asEventId("evt-throughput-command-gap-assistant-completed"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:01:43.000Z", + threadId: asThreadId("thread-1"), + turnId, + itemId: asItemId("item-throughput-command-gap-assistant"), + payload: { + itemType: "assistant_message", + }, + }); + harness.emit({ + type: "thread.token-usage.updated", + eventId: asEventId("evt-throughput-command-gap-token-usage"), + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-05-08T12:01:43.000Z", + threadId: asThreadId("thread-1"), + turnId, + payload: { + usage: { + usedTokens: 150_000, + lastOutputTokens: 1_790, + }, + }, + }); + + const thread = await waitForThread(harness.readModel, (entry) => + entry.activities.some( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-command-gap-token-usage", + ), + ); + const usageActivity = thread.activities.find( + (activity: ProviderRuntimeTestActivity) => + activity.id === "evt-throughput-command-gap-token-usage", + ); + + expect(usageActivity?.payload).toMatchObject({ + usedTokens: 150_000, + lastOutputTokens: 1_790, + durationMs: 1_000, + timeToFirstTokenMs: 1_000, + }); + expect((usageActivity?.payload as { durationMs?: number } | undefined)?.durationMs).not.toBe( + 102_000, + ); + expect( + thread.activities.some( + (activity: ProviderRuntimeTestActivity) => activity.kind === "tool.started", + ), + ).toBe(true); + }); + it("sums assistant response segments without including tool gaps", async () => { const harness = await createHarness(); const turnId = asTurnId("turn-throughput-tool-gap"); diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index ac77e3e8fd3..390cb0975ed 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -1566,6 +1566,22 @@ const make = Effect.gen(function* () { } } + const toolWorkTurnId = + ((event.type === "item.started" || event.type === "item.updated") && + isToolLifecycleItemType(event.payload.itemType)) || + (event.type === "content.delta" && + (event.payload.streamKind === "command_output" || + event.payload.streamKind === "file_change_output")) + ? toTurnId(event.turnId) + : undefined; + if (toolWorkTurnId) { + yield* closeAssistantResponseSegment({ + threadId: thread.id, + turnId: toolWorkTurnId, + createdAt: now, + }); + } + const pauseForUserTurnId = event.type === "request.opened" || event.type === "user-input.requested" ? toTurnId(event.turnId) diff --git a/apps/web/src/lib/turnStats.test.ts b/apps/web/src/lib/turnStats.test.ts index 36a95f7762b..cf249415c42 100644 --- a/apps/web/src/lib/turnStats.test.ts +++ b/apps/web/src/lib/turnStats.test.ts @@ -97,7 +97,7 @@ describe("turnStats", () => { "1 tool call", ]); expect(stats?.items.find((item) => item.id === "throughput")?.tooltip).toContain( - "Approximate throughput", + "derived timings exclude tool time", ); expect(stats?.items.find((item) => item.id === "ttft")?.tooltip).toContain( "Approximate time-to-first", diff --git a/apps/web/src/lib/turnStats.ts b/apps/web/src/lib/turnStats.ts index f038c33cf01..0072807e6ab 100644 --- a/apps/web/src/lib/turnStats.ts +++ b/apps/web/src/lib/turnStats.ts @@ -228,7 +228,8 @@ export function deriveLatestAssistantTurnStats(input: { items.push({ id: "throughput", label: formatTokensPerSecond(tokensPerSecond), - tooltip: "Approximate throughput based on provider token counts and generation duration.", + tooltip: + "Approximate assistant output throughput based on token counts and provider or derived generation duration; derived timings exclude tool time.", }); } }