Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/harness/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,5 @@ Install Chromium once with `pnpm --filter harness exec playwright install chromi
The browser check runs the built renderer and real API/engine on disposable ports, then changes the upstream fixture. It verifies SQLite snapshots, normal GUI polling (without reloading), queue entry/exit, failure preservation, recovery, and absence of duplicate PRs. Browser console errors, runtime exceptions, unexpected external requests and unsuccessful API responses fail the check.

Failures retain a Playwright trace and screenshot in `packages/harness/test-results/`. The HTML report includes API responses, upstream requests, and server logs; open it with `pnpm --filter harness exec playwright show-report`. The same check runs in CI and uploads its report on failure.

Additional phases: `conflicts`, `needs-approval`, `changes-requested`, `empty`, `assigned`, `review-requested`, `unassigned`, and `rate-limited` (20-second reset). Assignment-only and review-request-only fixtures are distinct: the current engine still uses review requests; changing the product inbox query belongs to #84. Tests can use `startScenario({ instances: 2 })` and target the second upstream with `setPhase(phase, 1)` / `syncPrs(1)` to check partial outages.
10 changes: 5 additions & 5 deletions packages/harness/src/board.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ test("polls queue entry, queue failure, and recovery through the real sync engin
return route.continue();
});
try {
await scenario.syncPrs();
await scenario.syncAll();
await page.goto(scenario.url);
const card = page.getByRole("listitem").filter({ hasText: SCENARIO_TITLE });
await expect(card).toHaveCount(1);
Expand All @@ -35,7 +35,7 @@ test("polls queue entry, queue failure, and recovery through the real sync engin
).toBeVisible();

scenario.setPhase("queued");
await scenario.syncPrs();
await scenario.syncAll();
expect(await scenario.persistedPrs()).toMatchObject([
{ number: 101, inMergeQueue: true, ciStatus: "pending" },
]);
Expand All @@ -48,7 +48,7 @@ test("polls queue entry, queue failure, and recovery through the real sync engin
await expect(card).toHaveCount(1);

scenario.setPhase("queue-failed");
await scenario.syncPrs();
await scenario.syncAll();
expect(await scenario.persistedPrs()).toMatchObject([
{ number: 101, inMergeQueue: false, ciStatus: "failure" },
]);
Expand All @@ -59,7 +59,7 @@ test("polls queue entry, queue failure, and recovery through the real sync engin
await expect(card).toHaveCount(1);

scenario.setPhase("offline");
await expect(scenario.syncPrs()).rejects.toThrow("Scenario sync failed");
await expect(scenario.syncAll()).rejects.toThrow("Scenario sync failed");
expect(await scenario.persistedPrs()).toMatchObject([
{ number: 101, ciStatus: "failure" },
]);
Expand All @@ -71,7 +71,7 @@ test("polls queue entry, queue failure, and recovery through the real sync engin
).toBeVisible();

scenario.setPhase("in-flight");
await scenario.syncPrs();
await scenario.syncAll();
await expect(card.getByText("passing", { exact: true })).toBeVisible();
await expect(card).toHaveCount(1);
expect(await scenario.persistedPrs()).toHaveLength(1);
Expand Down
72 changes: 72 additions & 0 deletions packages/harness/src/github.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,3 +86,75 @@ it("rejects unexpected calls instead of pretending they succeeded", async () =>
await github.close();
}
});

it("distinguishes assignment from review requests and supports conflict, approval, empty, and budget states", async () => {
const root = mkdtempSync(join(tmpdir(), "ghd-scenarios-"));
vi.stubEnv("XDG_CACHE_HOME", root);
const github = await startGitHub();
const engine = createSync({ loadInstances: () => [github.instance] });
try {
for (const [phase, expected] of [
["conflicts", { mergeable: false }],
[
"needs-approval",
{
reviewDecision: "REVIEW_REQUIRED",
reviews: { approved: [], changesRequested: [] },
},
],
[
"changes-requested",
{
reviewDecision: "CHANGES_REQUESTED",
reviews: { changesRequested: ["reviewer"] },
},
],
] as const) {
github.setPhase(phase);
await engine.sync();
expect(
engine.listPullRequests(github.instance.id, "prs")[0],
).toMatchObject(expected);
}
github.setPhase("assigned");
await engine.sync();
// The existing product still uses review requests. Assignment-only PRs
// must not leak in until #84 changes that query explicitly.
expect(engine.listPullRequests(github.instance.id, "reviews")).toEqual([]);
const assigned = await fetch(`${github.domain}/api/graphql`, {
method: "POST",
headers: { authorization: `token ${github.instance.token}` },
body: JSON.stringify({
query: "fragment PrFields query",
variables: {
first: 100,
q: "assignee:scenario-user type:pr state:open",
},
}),
});
expect(
(await assigned.json()).data.search.nodes[0].assignees.nodes,
).toEqual([{ login: "scenario-user" }]);
github.setPhase("review-requested");
await engine.sync();
expect(engine.listPullRequests(github.instance.id, "reviews")).toHaveLength(
1,
);
github.setPhase("unassigned");
await engine.sync();
expect(engine.listPullRequests(github.instance.id, "reviews")).toEqual([]);
github.setPhase("empty");
await engine.sync();
expect(engine.listPullRequests(github.instance.id, "prs")).toEqual([]);
github.setPhase("rate-limited");
await engine.sync();
await engine.sync();
expect(
engine.getHealth().find((health) => health.kind === "prs"),
).toMatchObject({ status: "skipped", retryAt: expect.any(String) });
} finally {
await engine.close();
await github.close();
rmSync(root, { recursive: true, force: true });
}
});
74 changes: 65 additions & 9 deletions packages/harness/src/github.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,20 @@
import { createServer } from "node:http";

export type Phase = "in-flight" | "queued" | "queue-failed" | "offline";
export const PHASES = [
"in-flight",
"queued",
"queue-failed",
"offline",
"conflicts",
"needs-approval",
"changes-requested",
"empty",
"assigned",
"review-requested",
"unassigned",
"rate-limited",
] as const;
export type Phase = (typeof PHASES)[number];
export const SCENARIO_TITLE = "Ship the status board";
export const SCENARIO_TOKEN = "scenario-token";
export const SCENARIO_USER = "scenario-user";
Expand Down Expand Up @@ -48,6 +62,7 @@ export async function startGitHub() {
![
`author:${SCENARIO_USER} type:pr state:open`,
`review-requested:${SCENARIO_USER} type:pr state:open`,
`assignee:${SCENARIO_USER} type:pr state:open`,
].includes(variables.q)
) {
send(501, { message: "Unsupported scenario GraphQL request" });
Expand All @@ -56,14 +71,34 @@ export async function startGitHub() {
send(200, {
data: {
search: {
nodes: variables.q.startsWith("author:")
? [pullRequest(phase)]
: [],
nodes:
phase === "empty"
? []
: variables.q.startsWith("author:")
? [pullRequest(phase)]
: (variables.q.startsWith("assignee:") &&
phase === "assigned") ||
(variables.q.startsWith("review-requested:") &&
phase === "review-requested")
? [
{
...pullRequest(phase),
id: "PR_review",
databaseId: 202,
number: 202,
title: "Review the storage change",
author: { login: "colleague", avatarUrl: "" },
},
]
: [],
},
rateLimit: {
cost: 1,
remaining: 5000,
resetAt: "2099-01-01T00:00:00Z",
remaining: phase === "rate-limited" ? 0 : 5000,
resetAt:
phase === "rate-limited"
? new Date(Date.now() + 20_000).toISOString()
: "2099-01-01T00:00:00Z",
},
},
});
Expand Down Expand Up @@ -120,9 +155,14 @@ function pullRequest(phase: Exclude<Phase, "offline">) {
? "2026-01-01T12:01:00Z"
: "2026-01-01T12:02:00Z",
isDraft: false,
mergeable: "MERGEABLE",
mergeable: phase === "conflicts" ? "CONFLICTING" : "MERGEABLE",
mergeStateStatus: "CLEAN",
reviewDecision: "APPROVED",
reviewDecision:
phase === "needs-approval"
? "REVIEW_REQUIRED"
: phase === "changes-requested"
? "CHANGES_REQUESTED"
: "APPROVED",
additions: 12,
deletions: 3,
headRefName: "feature/status-board",
Expand All @@ -148,7 +188,23 @@ function pullRequest(phase: Exclude<Phase, "offline">) {
},
],
},
reviews: { nodes: [{ state: "APPROVED", author: { login: "reviewer" } }] },
assignees: {
nodes: phase === "assigned" ? [{ login: SCENARIO_USER }] : [],
},
reviews: {
nodes:
phase === "needs-approval"
? []
: [
{
state:
phase === "changes-requested"
? "CHANGES_REQUESTED"
: "APPROVED",
author: { login: "reviewer" },
},
],
},
reviewThreads: { totalCount: 0, nodes: [] },
autoMergeRequest: null,
mergeQueueEntry: phase === "queued" ? { id: "queue-entry" } : null,
Expand Down
12 changes: 5 additions & 7 deletions packages/harness/src/run.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createInterface } from "node:readline/promises";
import { startScenario } from "./runtime.js";
import type { Phase } from "./github.js";
import { PHASES, type Phase } from "./github.js";

const scenario = await startScenario();
const terminal = createInterface({
Expand All @@ -14,24 +14,22 @@ const stop = async () => {
process.once("SIGINT", () => void stop());
process.once("SIGTERM", () => void stop());
console.log(`Scenario board: ${scenario.url}`);
console.log("Commands: in-flight, queued, queue-failed, offline, sync, quit");
console.log(`Commands: ${PHASES.join(", ")}, sync, quit`);
console.log(
"Each phase command syncs the real API. The board polls every 10s; reload for an immediate update.",
);
try {
for await (const line of terminal) {
const command = line.trim();
if (command === "quit") break;
if (["in-flight", "queued", "queue-failed", "offline"].includes(command)) {
if ((PHASES as readonly string[]).includes(command)) {
scenario.setPhase(command as Phase);
} else if (command !== "sync") {
console.log(
"Unknown command. Use in-flight, queued, queue-failed, offline, sync, or quit.",
);
console.log(`Unknown command. Use ${PHASES.join(", ")}, sync, or quit.`);
continue;
}
try {
const prs = await scenario.syncPrs();
const prs = await scenario.syncAll();
console.log(
JSON.stringify(
prs.map(
Expand Down
45 changes: 36 additions & 9 deletions packages/harness/src/runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@ import { startGitHub, SCENARIO_TOKEN, type Phase } from "./github.js";
const repo = resolve(import.meta.dirname, "../../..");

/** Runs the real API + built renderer against an isolated upstream and cache. */
export async function startScenario() {
export async function startScenario(options: { instances?: number } = {}) {
const root = mkdtempSync(join(tmpdir(), "ghd-scenario-"));
const upstreams: Awaited<ReturnType<typeof startGitHub>>[] = [];
let github: Awaited<ReturnType<typeof startGitHub>> | undefined;
let child: ChildProcess | undefined;
let output = "";
Expand All @@ -28,18 +29,20 @@ export async function startScenario() {
await exited;
clearTimeout(force);
}
await github?.close();
await Promise.all(upstreams.map((upstream) => upstream.close()));
rmSync(root, { recursive: true, force: true });
})());
try {
github = await startGitHub();
for (let i = 0; i < (options.instances ?? 1); i++)
upstreams.push(await startGitHub());
github = upstreams[0];
const instanceId = github.instance.id;
const configHome = join(root, "config");
const cacheHome = join(root, "cache");
mkdirSync(join(configHome, "github-dashboard"), { recursive: true });
writeFileSync(
join(configHome, "github-dashboard/config.yml"),
`instances:\n - domain: ${github.domain}\n label: Scenario GitHub\n token: ${SCENARIO_TOKEN}\n`,
`instances:\n${upstreams.map((upstream, i) => ` - domain: ${upstream.domain}\n label: Scenario GitHub ${i + 1}\n token: ${SCENARIO_TOKEN}\n`).join("")}`,
{ mode: 0o600 },
);
const port = await freePort();
Expand Down Expand Up @@ -97,15 +100,31 @@ export async function startScenario() {
return {
url,
instanceId,
setPhase(phase: Phase) {
github!.setPhase(phase);
instanceIds: upstreams.map((upstream) => upstream.instance.id),
setPhase(phase: Phase, index = 0) {
upstreams[index].setPhase(phase);
},
async syncPrs() {
const response = await fetch(`${url}/api/${instanceId}/prs?fresh=1`);
async syncPrs(index = 0) {
const response = await fetch(
`${url}/api/${upstreams[index].instance.id}/prs?fresh=1`,
);
if (!response.ok)
throw new Error(`Scenario sync failed (${response.status})`);
return response.json();
},
async syncAll(index = 0) {
const results = await Promise.all(
["prs", "reviews", "notifications"].map(async (kind) => {
const response = await fetch(
`${url}/api/${upstreams[index].instance.id}/${kind}?fresh=1`,
);
if (!response.ok)
throw new Error(`Scenario sync failed (${response.status})`);
return response.json();
}),
);
return results[0];
},
async persistedPrs() {
// Open the actual database through the public engine API. Only its
// synchronous constructor needs the scoped environment override.
Expand All @@ -124,7 +143,15 @@ export async function startScenario() {
await reader.close();
}
},
diagnostics: () => ({ server: output, requests: [...github!.requests] }),
diagnostics: () => ({
server: output,
requests: upstreams.flatMap((upstream) =>
upstream.requests.map((request) => ({
instanceId: upstream.instance.id,
...request,
})),
),
}),
close,
};
} catch (error) {
Expand Down
Loading
Loading