diff --git a/apps/gittensory-ui/src/routes/app.operator.tsx b/apps/gittensory-ui/src/routes/app.operator.tsx index 514e9a03..5a394d73 100644 --- a/apps/gittensory-ui/src/routes/app.operator.tsx +++ b/apps/gittensory-ui/src/routes/app.operator.tsx @@ -1,6 +1,6 @@ import { useState } from "react"; import { createFileRoute } from "@tanstack/react-router"; -import { Check, Copy, FileJson } from "lucide-react"; +import { BarChart3, Check, Copy, FileJson } from "lucide-react"; import { toast } from "sonner"; import { @@ -23,6 +23,7 @@ type OperatorDashboardResponse = { metrics: Array<{ label: string; value: string; delta: string }>; noiseReduction: Array<{ label: string; value: number; spark: number[] }>; weeklyReport: string[]; + recommendationQuality?: RecommendationQualityReport; weeklyValueReport?: { freshness: { status: string; latestRollupDay?: string | null }; warnings: string[]; @@ -31,6 +32,43 @@ type OperatorDashboardResponse = { upstreamDrift?: { status?: string } | null; }; +type RecommendationQualityReport = { + windowDays: number; + visibility: "operator_only"; + empty: boolean; + sparse: boolean; + totals: RecommendationQualityTotals; + trends: Array; + failureCategories: Array<{ category: string; label: string; count: number; detail: string }>; + roleSurfaces: Array< + RecommendationQualityTotals & { + role: "miner" | "maintainer" | "owner" | "operator"; + label: string; + topRepos: Array<{ + repoFullName: string; + total: number; + positive: number; + negative: number; + signal: "positive" | "negative" | "mixed"; + }>; + } + >; + warnings: string[]; + publicExport: { available: false; reason: string }; + privateSummary: string; +}; + +type RecommendationQualityTotals = { + total: number; + positive: number; + negative: number; + positiveRate: number; + maintainerLaneTotal: number; + highConfidence: number; + mediumConfidence: number; + lowConfidence: number; +}; + type ReportExportFormat = "markdown" | "json"; function OperatorDashboard() { @@ -40,6 +78,7 @@ function OperatorDashboard() { ); const [copiedExport, setCopiedExport] = useState(null); const data = dashboard.status === "ready" ? dashboard.data : null; + const quality = data?.recommendationQuality; const copyWeeklyReport = async (format: ReportExportFormat) => { if (!data?.weeklyValueReport) return; try { @@ -106,6 +145,151 @@ function OperatorDashboard() { ))} + {quality ? ( +
+
+
+
+ +

+ Recommendation quality +

+
+

+ {quality.privateSummary} +

+
+
+ + {quality.empty ? "empty" : quality.sparse ? "sparse" : "populated"} + + {quality.windowDays}d +
+
+ +
+ + + + +
+ +
+
+

+ Role surfaces +

+
+ {quality.roleSurfaces.length ? ( + quality.roleSurfaces.map((surface) => ( +
+
+
+
+ {surface.label} +
+
+ {surface.positive} positive · {surface.negative} negative +
+
+ + {Math.round(surface.positiveRate * 100)}% + +
+ {surface.topRepos.length ? ( +
    + {surface.topRepos.slice(0, 3).map((repo) => ( +
  • + {repo.repoFullName} + + {repo.positive}/{repo.total} + +
    • + ))} +
+ ) : null} +
+ )) + ) : ( +
+ No role-specific outcomes in this window. +
+ )} +
+
+ +
+

+ Failure categories +

+
+ {quality.failureCategories.length ? ( + quality.failureCategories.map((category) => ( +
+
+ {category.label} + + {category.count} + +
+

+ {category.detail} +

+
+ )) + ) : ( +
+ No failure categories in this window. +
+ )} +
+
+
+ + {quality.trends.length ? ( +
+

+ Trend +

+
+ {quality.trends.map((bucket) => ( +
+ ))} +
+
+ ) : null} + + {quality.warnings.length ? ( +
    + {quality.warnings.slice(0, 3).map((warning) => ( +
  • · {warning}
    • + ))} +
+ ) : null} +
+ ) : null} +

Noise reduction

@@ -200,6 +384,12 @@ function OperatorDashboard() { ); } +function qualityStatus(rate: number): "ready" | "warn" | "stale" { + if (rate >= 0.67) return "ready"; + if (rate >= 0.4) return "stale"; + return "warn"; +} + async function loadWeeklyReportMarkdown(): Promise { const result = await apiFetch( `${getApiOrigin().replace(/\/$/, "")}/v1/app/analytics/weekly-value-report?variant=operator&format=markdown`, diff --git a/src/services/operator-dashboard.ts b/src/services/operator-dashboard.ts index 3c37a2ff..10954d4f 100644 --- a/src/services/operator-dashboard.ts +++ b/src/services/operator-dashboard.ts @@ -27,6 +27,7 @@ import type { } from "../types"; import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset"; import { nowIso } from "../utils/json"; +import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report"; import { buildWeeklyValueReport } from "./weekly-value-report"; export type OperatorDashboardMetric = { @@ -52,6 +53,7 @@ export type OperatorDashboardPayload = { usageRollupStatus: ProductUsageRollupStatus; mcpCompatibilityAdoption: McpCompatibilityAdoptionSummary; commandUsefulness: CommandUsefulnessSummary; + recommendationQuality: RecommendationQualityReport; registry: RegistrySnapshot | null; scoringModel: ScoringModelSnapshotRecord | null; upstreamDrift: UpstreamStatus; @@ -76,6 +78,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise record.status !== "healthy").length), @@ -164,6 +173,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise b.day.localeCompare(a.day))[0] ?? null; + return [...rollups].sort((a, b) => b.day.localeCompare(a.day))[0]!; } function usefulnessDelta(rate: number | null): string { diff --git a/src/services/recommendation-quality-report.ts b/src/services/recommendation-quality-report.ts new file mode 100644 index 00000000..739ca5d1 --- /dev/null +++ b/src/services/recommendation-quality-report.ts @@ -0,0 +1,283 @@ +import { listAgentRecommendationOutcomes } from "../db/repositories"; +import type { AgentActionType, AgentRecommendationOutcomeRecord, AgentRecommendationOutcomeState, JsonValue, ProductUsageRole } from "../types"; +import { nowIso } from "../utils/json"; + +export type RecommendationQualityRole = Extract; + +export type RecommendationQualityTotals = { + total: number; + positive: number; + negative: number; + positiveRate: number; + maintainerLaneTotal: number; + highConfidence: number; + mediumConfidence: number; + lowConfidence: number; +}; + +export type RecommendationQualityTrendBucket = RecommendationQualityTotals & { + periodStart: string; + periodEnd: string; +}; + +export type RecommendationQualityFailureCategory = { + category: "closed_without_merge" | "stale" | "ignored" | "low_confidence" | "maintainer_lane"; + label: string; + count: number; + detail: string; +}; + +export type RecommendationQualityRoleSurface = RecommendationQualityTotals & { + role: RecommendationQualityRole; + label: string; + topRepos: Array<{ + repoFullName: string; + total: number; + positive: number; + negative: number; + signal: "positive" | "negative" | "mixed"; + }>; +}; + +export type RecommendationQualityReport = { + generatedAt: string; + windowDays: number; + visibility: "operator_only"; + empty: boolean; + sparse: boolean; + totals: RecommendationQualityTotals; + trends: RecommendationQualityTrendBucket[]; + failureCategories: RecommendationQualityFailureCategory[]; + roleSurfaces: RecommendationQualityRoleSurface[]; + warnings: string[]; + publicExport: { + available: false; + reason: string; + }; + privateSummary: string; +}; + +const ROLE_ORDER: RecommendationQualityRole[] = ["miner", "maintainer", "owner", "operator"]; +const POSITIVE_STATES: AgentRecommendationOutcomeState[] = ["accepted", "merged", "improved"]; +const NEGATIVE_STATES: AgentRecommendationOutcomeState[] = ["closed", "stale", "ignored"]; + +export async function buildRecommendationQualityReport( + env: Env, + options: { now?: string; windowDays?: number; limit?: number } = {}, +): Promise { + const generatedAt = options.now ?? nowIso(); + const windowDays = clampInteger(options.windowDays ?? 90, 1, 365); + const outcomes = await listAgentRecommendationOutcomes(env, { + windowDays, + now: generatedAt, + limit: options.limit ?? 5000, + }); + return buildRecommendationQualityReportFromOutcomes(outcomes, { generatedAt, windowDays }); +} + +export function buildRecommendationQualityReportFromOutcomes( + outcomes: AgentRecommendationOutcomeRecord[], + options: { generatedAt: string; windowDays: number }, +): RecommendationQualityReport { + const sorted = [...outcomes].sort((left, right) => outcomeTimestamp(left).localeCompare(outcomeTimestamp(right))); + const totals = qualityTotals(sorted); + const roleSurfaces = ROLE_ORDER.map((role) => roleSurface(role, sorted.filter((outcome) => roleForOutcome(outcome) === role))).filter((surface) => surface.total > 0 || surface.maintainerLaneTotal > 0); + const failureCategories = failureCategoryRows(sorted); + const trends = trendBuckets(sorted, options.generatedAt, options.windowDays); + const sparse = totals.total > 0 && totals.total < 5; + const warnings = [ + ...(totals.total === 0 ? ["No recommendation outcomes have been evaluated in this window."] : []), + ...(sparse ? ["Recommendation quality data is sparse; treat trends as directional only."] : []), + ...(roleSurfaces.length === 0 ? ["No role-specific outcome surfaces have enough data to display."] : []), + ]; + const privateSummary = totals.total === 0 + ? `No recommendation quality outcomes are available for the last ${options.windowDays} day(s).` + : `Recommendation quality has ${totals.positive} positive and ${totals.negative} unresolved or negative outcome(s) across ${totals.total} evaluated recommendation(s).`; + return { + generatedAt: options.generatedAt, + windowDays: options.windowDays, + visibility: "operator_only", + empty: totals.total === 0 && totals.maintainerLaneTotal === 0, + sparse, + totals, + trends, + failureCategories, + roleSurfaces, + warnings, + publicExport: { + available: false, + reason: "Recommendation quality reports are available only in the authenticated operator dashboard.", + }, + privateSummary, + }; +} + +function roleSurface(role: RecommendationQualityRole, outcomes: AgentRecommendationOutcomeRecord[]): RecommendationQualityRoleSurface { + const totals = qualityTotals(outcomes); + const byRepo = new Map(); + for (const outcome of outcomes) { + const repoFullName = outcome.outcomeRepoFullName ?? outcome.targetRepoFullName; + if (!repoFullName) continue; + const key = repoFullName.toLowerCase(); + byRepo.set(key, [...(byRepo.get(key) ?? []), outcome]); + } + const topRepos = [...byRepo.values()] + .map((repoOutcomes) => { + const first = repoOutcomes[0]!; + const repoTotals = qualityTotals(repoOutcomes); + return { + repoFullName: first.outcomeRepoFullName ?? first.targetRepoFullName ?? "unknown/repo", + total: repoTotals.total, + positive: repoTotals.positive, + negative: repoTotals.negative, + signal: signalFor(repoTotals), + }; + }) + .sort((left, right) => right.total - left.total || left.repoFullName.localeCompare(right.repoFullName)) + .slice(0, 5); + return { + role, + label: roleLabel(role), + ...totals, + topRepos, + }; +} + +function qualityTotals(outcomes: AgentRecommendationOutcomeRecord[]): RecommendationQualityTotals { + const positive = outcomes.filter((outcome) => POSITIVE_STATES.includes(outcome.outcomeState)).length; + const negative = outcomes.filter((outcome) => NEGATIVE_STATES.includes(outcome.outcomeState)).length; + const total = positive + negative; + return { + total, + positive, + negative, + positiveRate: total > 0 ? roundRate(positive / total) : 0, + maintainerLaneTotal: outcomes.filter((outcome) => outcome.maintainerLane).length, + highConfidence: outcomes.filter((outcome) => outcome.confidence === "high").length, + mediumConfidence: outcomes.filter((outcome) => outcome.confidence === "medium").length, + lowConfidence: outcomes.filter((outcome) => outcome.confidence === "low").length, + }; +} + +function failureCategoryRows(outcomes: AgentRecommendationOutcomeRecord[]): RecommendationQualityFailureCategory[] { + const rows: RecommendationQualityFailureCategory[] = [ + { + category: "closed_without_merge", + label: "Closed without merge", + count: outcomes.filter((outcome) => outcome.outcomeState === "closed").length, + detail: "Recommended work reached a closed terminal state without a merge signal.", + }, + { + category: "stale", + label: "Stale follow-through", + count: outcomes.filter((outcome) => outcome.outcomeState === "stale").length, + detail: "Recommended work remained open without fresh cached activity past the freshness window.", + }, + { + category: "ignored", + label: "No matched activity", + count: outcomes.filter((outcome) => outcome.outcomeState === "ignored").length, + detail: "No cached PR or issue activity matched the recommendation after the action window.", + }, + { + category: "low_confidence", + label: "Low confidence matches", + count: outcomes.filter((outcome) => outcome.confidence === "low").length, + detail: "The recommendation target was too broad or missing for a strong match.", + }, + { + category: "maintainer_lane", + label: "Maintainer-lane separated", + count: outcomes.filter((outcome) => outcome.maintainerLane).length, + detail: "Maintainer-associated outcomes are separated from contributor recommendation quality.", + }, + ]; + return rows.filter((row) => row.count > 0).sort((left, right) => right.count - left.count || left.label.localeCompare(right.label)); +} + +function trendBuckets( + outcomes: AgentRecommendationOutcomeRecord[], + generatedAt: string, + windowDays: number, +): RecommendationQualityTrendBucket[] { + const bucketCount = Math.min(6, Math.max(1, Math.ceil(windowDays / 7))); + const now = Date.parse(generatedAt); + const bucketMs = Math.max(1, Math.ceil((windowDays * 24 * 60 * 60 * 1000) / bucketCount)); + return Array.from({ length: bucketCount }, (_, index) => { + const periodStartMs = now - bucketMs * (bucketCount - index); + const periodEndMs = index === bucketCount - 1 ? now : periodStartMs + bucketMs; + const bucketOutcomes = outcomes.filter((outcome) => { + const timestamp = Date.parse(outcomeTimestamp(outcome)); + return Number.isFinite(timestamp) && timestamp >= periodStartMs && timestamp <= periodEndMs; + }); + return { + periodStart: new Date(periodStartMs).toISOString(), + periodEnd: new Date(periodEndMs).toISOString(), + ...qualityTotals(bucketOutcomes), + }; + }); +} + +function roleForOutcome(outcome: AgentRecommendationOutcomeRecord): RecommendationQualityRole { + const metadataRole = roleFromMetadata(outcome.metadata); + if (metadataRole) return metadataRole; + if (outcome.maintainerLane) return "maintainer"; + return roleFromActionType(outcome.actionType); +} + +function roleFromMetadata(metadata: Record): RecommendationQualityRole | null { + for (const key of ["role", "roles", "actorRole", "actorKind", "audience", "surface"]) { + const role = roleFromJsonValue(metadata[key]); + if (role) return role; + } + return null; +} + +function roleFromJsonValue(value: JsonValue | undefined): RecommendationQualityRole | null { + if (Array.isArray(value)) { + for (const entry of value) { + const role = roleFromJsonValue(entry); + if (role) return role; + } + return null; + } + if (typeof value !== "string") return null; + const normalized = value.toLowerCase().replace(/[_\s-]+/g, "_"); + if (normalized === "miner" || normalized === "contributor") return "miner"; + if (normalized === "maintainer") return "maintainer"; + if (normalized === "owner" || normalized === "repo_owner" || normalized === "repository_owner") return "owner"; + if (normalized === "operator") return "operator"; + return null; +} + +function roleFromActionType(actionType: AgentActionType): RecommendationQualityRole { + if (actionType === "explain_repo_fit") return "owner"; + if (actionType === "monitor_existing_pr" || actionType === "check_duplicate_risk") return "maintainer"; + return "miner"; +} + +function roleLabel(role: RecommendationQualityRole): string { + if (role === "miner") return "Miner guidance"; + if (role === "maintainer") return "Maintainer guidance"; + if (role === "owner") return "Repo-owner guidance"; + return "Operator guidance"; +} + +function signalFor(totals: RecommendationQualityTotals): "positive" | "negative" | "mixed" { + if (totals.positive > totals.negative) return "positive"; + if (totals.negative > totals.positive) return "negative"; + return "mixed"; +} + +function outcomeTimestamp(outcome: AgentRecommendationOutcomeRecord): string { + return outcome.updatedAt ?? outcome.detectedAt ?? outcome.createdAt ?? ""; +} + +function roundRate(value: number): number { + return Math.round(value * 1000) / 1000; +} + +function clampInteger(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min; + return Math.min(max, Math.max(min, Math.round(value))); +} diff --git a/test/integration/api.test.ts b/test/integration/api.test.ts index bb147be7..e1326e9e 100644 --- a/test/integration/api.test.ts +++ b/test/integration/api.test.ts @@ -28,6 +28,8 @@ import { upsertRepositoryFromGitHub, upsertRepositorySettings, createAgentRun, + replaceAgentActions, + upsertAgentRecommendationOutcome, } from "../../src/db/repositories"; import { createApp } from "../../src/api/routes"; import { BURDEN_FORECAST_MAX_AGE_MS } from "../../src/services/burden-forecast"; @@ -1396,6 +1398,12 @@ describe("api routes", () => { expect(emptyOperator.status).toBe(200); await expect(emptyOperator.json()).resolves.toMatchObject({ metrics: expect.arrayContaining([expect.objectContaining({ label: "Registered repos", delta: "registry missing" })]), + recommendationQuality: expect.objectContaining({ + empty: true, + sparse: false, + totals: expect.objectContaining({ total: 0, positive: 0, negative: 0 }), + roleSurfaces: [], + }), }); const driftDigest = await app.request("/v1/app/digest", { headers: apiHeaders(emptyEnv) }, emptyEnv); expect(driftDigest.status).toBe(200); @@ -1841,14 +1849,81 @@ describe("api routes", () => { expect(defaultUsefulness.status).toBe(200); await expect(defaultUsefulness.json()).resolves.toMatchObject({ windowDays: 30 }); + await createAgentRun(env, { + id: "api-quality-run", + objective: "Track recommendation quality", + actorLogin: "quality-user", + surface: "api", + mode: "copilot", + status: "completed", + dataQualityStatus: "complete", + payload: {}, + createdAt: "2026-05-28T00:00:00.000Z", + updatedAt: "2026-05-28T00:00:00.000Z", + }); + await replaceAgentActions(env, "api-quality-run", [ + { + id: "api-quality-action", + runId: "api-quality-run", + actionType: "prepare_pr_packet", + targetRepoFullName: "JSONbored/gittensory", + targetPullNumber: null, + targetIssueNumber: null, + status: "recommended", + recommendation: "pursue", + why: ["Safe aggregate fixture."], + blockedBy: [], + publicSafeSummary: "Safe aggregate fixture.", + approvalRequired: true, + safetyClass: "private", + payload: {}, + createdAt: "2026-05-28T00:00:00.000Z", + }, + ]); + await upsertAgentRecommendationOutcome(env, { + actionId: "api-quality-action", + runId: "api-quality-run", + actorLogin: "quality-user", + actionType: "prepare_pr_packet", + targetRepoFullName: "JSONbored/gittensory", + targetPullNumber: null, + targetIssueNumber: null, + outcomeState: "merged", + outcomeTargetType: "pull_request", + outcomeRepoFullName: "JSONbored/gittensory", + outcomePullNumber: 330, + outcomeIssueNumber: null, + maintainerLane: false, + confidence: "high", + reason: "Safe aggregate fixture.", + sourceUpdatedAt: "2026-05-28T00:00:00.000Z", + detectedAt: "2026-05-28T00:00:00.000Z", + updatedAt: "2026-05-28T00:00:00.000Z", + metadata: { role: "miner" }, + }); + const operator = await app.request("/v1/app/operator-dashboard", { headers: apiHeaders(env) }, env); expect(operator.status).toBe(200); - await expect(operator.json()).resolves.toMatchObject({ + const operatorBody = (await operator.json()) as { + metrics: unknown[]; + noiseReduction: unknown[]; + weeklyReport: string[]; + commandUsefulness: unknown; + recommendationQuality: unknown; + }; + expect(operatorBody).toMatchObject({ metrics: expect.arrayContaining([expect.objectContaining({ label: "Active sessions" }), expect.objectContaining({ label: "Digest subscriptions" }), expect.objectContaining({ label: "Command usefulness", value: "0/1" })]), noiseReduction: expect.any(Array), weeklyReport: expect.arrayContaining([expect.stringContaining("registered repo")]), commandUsefulness: expect.objectContaining({ totals: expect.objectContaining({ feedbackCount: 1 }) }), + recommendationQuality: expect.objectContaining({ + visibility: "operator_only", + publicExport: expect.objectContaining({ available: false }), + totals: expect.objectContaining({ total: 1, positive: 1, negative: 0 }), + roleSurfaces: expect.arrayContaining([expect.objectContaining({ role: "miner", positive: 1 })]), + }), }); + expect(JSON.stringify(operatorBody.recommendationQuality)).not.toMatch(FORBIDDEN_PUBLIC_REPORT_TERMS); const notificationModel = await app.request("/v1/app/notification-model", { headers: apiHeaders(env) }, env); expect(notificationModel.status).toBe(200); diff --git a/test/unit/recommendation-quality-report.test.ts b/test/unit/recommendation-quality-report.test.ts new file mode 100644 index 00000000..79eabdab --- /dev/null +++ b/test/unit/recommendation-quality-report.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import { createAgentRun, replaceAgentActions, upsertAgentRecommendationOutcome } from "../../src/db/repositories"; +import { buildRecommendationQualityReport, buildRecommendationQualityReportFromOutcomes } from "../../src/services/recommendation-quality-report"; +import type { AgentRecommendationOutcomeRecord } from "../../src/types"; +import { createTestEnv } from "../helpers/d1"; + +const FORBIDDEN_REPORT_TERMS = + /wallet|hotkey|raw trust|trust[-\s]?score|payout|reward[-\s]?estimate|farming|private[-\s]?reviewability|public[-\s]?score[-\s]?(?:estimate|prediction)|private[-\s]?scoreability|github_pat|ghp_/i; + +describe("recommendation quality report", () => { + it("builds role-aware totals, trends, and failure categories without leaking private fields", () => { + const report = buildRecommendationQualityReportFromOutcomes( + [ + outcome("miner-merged", "merged", { actionType: "prepare_pr_packet", repo: "owner/good", updatedAt: "2026-05-05T00:00:00.000Z" }), + outcome("miner-closed", "closed", { + actionType: "preflight_branch", + repo: "owner/risky", + updatedAt: "2026-05-12T00:00:00.000Z", + reason: "contains wallet and private scoreability context that must never appear", + metadata: { wallet: "secret-wallet", hotkey: "secret-hotkey", token: "ghp_secret" }, + }), + outcome("maintainer-stale", "stale", { actionType: "monitor_existing_pr", repo: "owner/watch", updatedAt: "2026-05-19T00:00:00.000Z" }), + outcome("owner-improved", "improved", { actionType: "explain_repo_fit", repo: "owner/settings", updatedAt: "2026-05-21T00:00:00.000Z", metadata: { role: "owner" } }), + outcome("operator-ignored", "ignored", { actionType: "choose_next_work", repo: "owner/ops", updatedAt: "2026-05-28T00:00:00.000Z", metadata: { role: "operator" }, confidence: "low" }), + outcome("maintainer-lane", "merged", { actionType: "choose_next_work", repo: "dev/own", updatedAt: "2026-05-29T00:00:00.000Z", maintainerLane: true }), + ], + { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 42 }, + ); + + expect(report).toMatchObject({ + visibility: "operator_only", + empty: false, + sparse: false, + totals: { total: 6, positive: 3, negative: 3, positiveRate: 0.5, maintainerLaneTotal: 1, lowConfidence: 1 }, + publicExport: { available: false }, + }); + expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner", "operator"]); + expect(report.roleSurfaces.find((surface) => surface.role === "miner")).toMatchObject({ total: 2, positive: 1, negative: 1 }); + expect(report.failureCategories.map((category) => category.category)).toEqual( + expect.arrayContaining(["closed_without_merge", "stale", "ignored", "low_confidence", "maintainer_lane"]), + ); + expect(report.trends).toHaveLength(6); + expect(JSON.stringify(report)).not.toMatch(FORBIDDEN_REPORT_TERMS); + }); + + it("reports empty and sparse private states deterministically", () => { + const empty = buildRecommendationQualityReportFromOutcomes([], { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 90 }); + expect(empty).toMatchObject({ + empty: true, + sparse: false, + totals: { total: 0, positive: 0, negative: 0, positiveRate: 0 }, + roleSurfaces: [], + failureCategories: [], + }); + expect(empty.warnings.join(" ")).toMatch(/No recommendation outcomes/); + + const sparse = buildRecommendationQualityReportFromOutcomes( + [outcome("one", "accepted", { updatedAt: "2026-05-30T00:00:00.000Z" })], + { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 14 }, + ); + expect(sparse).toMatchObject({ empty: false, sparse: true, totals: { total: 1, positive: 1, negative: 0 } }); + expect(sparse.warnings.join(" ")).toMatch(/sparse/i); + }); + + it("normalizes role metadata, action fallbacks, repo signals, and timestamp fallbacks", () => { + const report = buildRecommendationQualityReportFromOutcomes( + [ + outcome("metadata-array", "accepted", { metadata: { roles: ["unknown", "repo-owner"] }, repo: "owner/mixed" }), + outcome("metadata-array-negative", "closed", { metadata: { roles: ["owner"] }, repo: "owner/mixed", updatedAt: "2026-05-30T00:00:00.000Z" }), + outcome("metadata-contributor", "ignored", { metadata: { audience: "contributor" }, repo: "owner/miner", confidence: "medium" }), + outcome("metadata-unknown", "merged", { metadata: { role: "reviewability" }, repo: "owner/fallback", actionType: "check_duplicate_risk" }), + outcome("no-repo", "stale", { repo: null, actionType: "explain_repo_fit" }), + ], + { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 7 }, + ); + + expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["miner", "maintainer", "owner"]); + expect(report.roleSurfaces.find((surface) => surface.role === "owner")?.topRepos).toEqual( + expect.arrayContaining([expect.objectContaining({ repoFullName: "owner/mixed", signal: "mixed" })]), + ); + expect(report.roleSurfaces.find((surface) => surface.role === "maintainer")).toMatchObject({ positive: 1, negative: 0 }); + expect(report.roleSurfaces.find((surface) => surface.role === "miner")).toMatchObject({ positive: 0, negative: 1, mediumConfidence: 1 }); + }); + + it("falls back to action type when metadata has no recognized role", () => { + const report = buildRecommendationQualityReportFromOutcomes( + [ + outcome("array-without-role", "merged", { + actionType: "monitor_existing_pr", + metadata: { roles: ["reviewability", "unknown"] }, + repo: "owner/fallback-array", + }), + outcome("non-string-role", "closed", { + actionType: "explain_repo_fit", + metadata: { actorRole: 123 }, + repo: "owner/fallback-number", + }), + ], + { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 7 }, + ); + + expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["maintainer", "owner"]); + expect(report.roleSurfaces.find((surface) => surface.role === "maintainer")).toMatchObject({ positive: 1, negative: 0 }); + expect(report.roleSurfaces.find((surface) => surface.role === "owner")).toMatchObject({ positive: 0, negative: 1 }); + }); + + it("uses metadata role and timestamp fallbacks for report grouping", () => { + const detectedOnly = { + ...outcome("detected-only", "accepted", { + repo: "owner/detected", + metadata: { actorKind: "maintainer" }, + }), + updatedAt: null, + detectedAt: "2026-05-25T00:00:00.000Z", + createdAt: null, + }; + const createdOnly = { + ...outcome("created-only", "closed", { + repo: "owner/detected", + metadata: { surface: "repository-owner" }, + }), + updatedAt: null, + detectedAt: null, + createdAt: "2026-05-26T00:00:00.000Z", + }; + const missingTimestamp = { + ...outcome("missing-time", "ignored", { + repo: "owner/no-time", + metadata: { role: "operator" }, + }), + updatedAt: null, + detectedAt: null, + createdAt: null, + }; + + const report = buildRecommendationQualityReportFromOutcomes( + [detectedOnly, createdOnly, missingTimestamp], + { generatedAt: "2026-06-01T00:00:00.000Z", windowDays: 7 }, + ); + + expect(report.roleSurfaces.map((surface) => surface.role)).toEqual(["maintainer", "owner", "operator"]); + expect(report.roleSurfaces.find((surface) => surface.role === "maintainer")).toMatchObject({ positive: 1, negative: 0 }); + expect(report.roleSurfaces.find((surface) => surface.role === "owner")?.topRepos).toEqual( + expect.arrayContaining([expect.objectContaining({ repoFullName: "owner/detected", signal: "negative" })]), + ); + expect(report.roleSurfaces.find((surface) => surface.role === "operator")).toMatchObject({ positive: 0, negative: 1 }); + expect(report.trends.some((bucket) => bucket.total > 0)).toBe(true); + }); + + it("loads persisted outcomes for the operator report window", async () => { + const env = createTestEnv(); + await createAgentRun(env, { + id: "quality-run", + objective: "Track quality", + actorLogin: "quality-user", + surface: "api", + mode: "copilot", + status: "completed", + dataQualityStatus: "complete", + payload: {}, + createdAt: "2026-05-30T00:00:00.000Z", + updatedAt: "2026-05-30T00:00:00.000Z", + }); + await replaceAgentActions(env, "quality-run", [ + { + id: "quality-action", + runId: "quality-run", + actionType: "choose_next_work", + targetRepoFullName: "owner/repo", + targetPullNumber: null, + targetIssueNumber: null, + status: "recommended", + recommendation: "pursue", + why: ["Safe aggregate fixture."], + blockedBy: [], + publicSafeSummary: "Safe aggregate fixture.", + approvalRequired: true, + safetyClass: "private", + payload: {}, + createdAt: "2026-05-30T00:00:00.000Z", + }, + ]); + await upsertAgentRecommendationOutcome(env, { + ...outcome("persisted", "merged", { repo: "owner/repo", updatedAt: "2026-05-30T00:00:00.000Z" }), + actionId: "quality-action", + runId: "quality-run", + actorLogin: "quality-user", + }); + + await expect(buildRecommendationQualityReport(env, { now: "2026-06-01T00:00:00.000Z", windowDays: 14 })).resolves.toMatchObject({ + totals: { total: 1, positive: 1, negative: 0 }, + roleSurfaces: [expect.objectContaining({ role: "miner", positive: 1 })], + }); + }); + + it("clamps invalid persisted report windows", async () => { + const env = createTestEnv(); + + await expect(buildRecommendationQualityReport(env, { now: "2026-06-01T00:00:00.000Z", windowDays: Number.NaN })).resolves.toMatchObject({ + windowDays: 1, + totals: { total: 0 }, + }); + }); +}); + +function outcome( + id: string, + outcomeState: AgentRecommendationOutcomeRecord["outcomeState"], + options: { + actionType?: AgentRecommendationOutcomeRecord["actionType"]; + repo?: string | null; + updatedAt?: string; + reason?: string; + metadata?: AgentRecommendationOutcomeRecord["metadata"]; + maintainerLane?: boolean; + confidence?: AgentRecommendationOutcomeRecord["confidence"]; + } = {}, +): AgentRecommendationOutcomeRecord { + return { + id: `outcome:${id}`, + actionId: `action:${id}`, + runId: `run:${id}`, + actorLogin: "dev", + actionType: options.actionType ?? "choose_next_work", + targetRepoFullName: options.repo === null ? null : options.repo ?? "owner/repo", + targetPullNumber: null, + targetIssueNumber: null, + outcomeState, + outcomeTargetType: "repository", + outcomeRepoFullName: options.repo === null ? null : options.repo ?? "owner/repo", + outcomePullNumber: null, + outcomeIssueNumber: null, + maintainerLane: options.maintainerLane ?? false, + confidence: options.confidence ?? "high", + reason: options.reason ?? "safe aggregate fixture", + sourceUpdatedAt: options.updatedAt ?? "2026-05-30T00:00:00.000Z", + detectedAt: options.updatedAt ?? "2026-05-30T00:00:00.000Z", + metadata: options.metadata ?? {}, + createdAt: options.updatedAt ?? "2026-05-30T00:00:00.000Z", + updatedAt: options.updatedAt ?? "2026-05-30T00:00:00.000Z", + }; +} diff --git a/test/unit/repo-policy-readiness.test.ts b/test/unit/repo-policy-readiness.test.ts index 8bbddf0f..455fbe47 100644 --- a/test/unit/repo-policy-readiness.test.ts +++ b/test/unit/repo-policy-readiness.test.ts @@ -384,10 +384,9 @@ describe("buildRepoPolicyReadiness", () => { ); }); - it("uses empty owner-context defaults when the focus manifest is omitted", () => { - const report = buildRepoPolicyReadiness(input({ focusManifest: undefined })); - - expect(report).toMatchObject({ + it("handles absent optional policy data and clean policy summaries", () => { + const missing = buildRepoPolicyReadiness(input({ focusManifest: undefined })); + expect(missing).toMatchObject({ present: false, ownerContext: { manifestPresent: false, @@ -400,10 +399,15 @@ describe("buildRepoPolicyReadiness", () => { issueDiscoveryPolicy: "neutral", }, }); - expect(report.publicWarnings).toEqual( + expect(missing.publicWarnings).toEqual( expect.arrayContaining([ expect.objectContaining({ code: "focus_policy_missing" }), ]), ); + + const clean = buildRepoPolicyReadiness(input()); + expect(clean.publicWarnings).toEqual([]); + expect(clean.droppedPublicWarnings).toEqual([]); + expect(clean.summary).toBe("Policy readiness has no public-safe warnings for owner review."); }); });