Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/workflows/measure-test-weights.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ jobs:
runtime: [deno, node, bun]
runs-on: ubuntu-latest
timeout-minutes: 165
env:
XMD_TEST_ARTIFACTS: ${{ github.workspace }}/.xmd/test-artifacts
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6

Expand Down Expand Up @@ -94,6 +96,15 @@ jobs:
path: ${{ runner.temp }}/measurements/test-weights-${{ matrix.runtime }}.json
if-no-files-found: warn

- name: Upload the ${{ matrix.runtime }} failure journals
if: ${{ failure() || cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: measure-test-journals-${{ matrix.runtime }}-attempt-${{ github.run_attempt }}
path: ${{ env.XMD_TEST_ARTIFACTS }}
if-no-files-found: ignore
include-hidden-files: true

# Runs only when every runtime's job succeeded: a failed, cancelled or
# timed-out runtime skips it, so no `test-weights` artifact exists for a run
# that did not measure the whole corpus.
Expand Down
8 changes: 5 additions & 3 deletions packages/cli/tests/repl-headless-submit-process.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,11 @@ describe("first SIGINT at the submit host boundary", () => {
expect(call.closed).toBe(false);
expect(call.records.some((record) => record.kind === "finished")).toBe(false);
released.resolve();
expect(
yield* within(call.joined, () => ({ waiting: "joined Stop", ...call.output() })),
).toEqual({ code: 130, signal: null });
const exit = yield* within(call.joined, () => ({ waiting: "joined Stop", ...call.output() }));
if (exit.code !== 130 || exit.signal !== null) {
console.error(JSON.stringify({ ...exit, ...call.output(), records: call.records }));
}
expect(exit).toEqual({ code: 130, signal: null });
const admitted = call.records[0];
const final = call.records.at(-1);
expect(final).toMatchObject({
Expand Down
35 changes: 35 additions & 0 deletions scripts/tests/measure-test-weights.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -442,6 +442,41 @@ describe("the measurement workflow", () => {
);
});

it("retains failure journals separately from measurement records", function* () {
const workflow = yield* document(WORKFLOW);
const jobs = object(workflow.jobs, "workflow.jobs");
const raw = object(jobs.measure, "workflow.jobs.measure");
expect(strings(raw.env, "workflow.jobs.measure.env").XMD_TEST_ARTIFACTS).toEqual(
"${{ github.workspace }}/.xmd/test-artifacts",
);
const { measure: measuring, assemble } = yield* workflowJobs();
const [measurement, journals] = uploads(measuring);
if (measurement === undefined || journals === undefined) {
throw new Error("measurement and journal uploads are required");
}
const measure = stepRunning(measuring, "weights:measure");
expect(measuring.steps.indexOf(measurement)).toBeGreaterThan(measuring.steps.indexOf(measure));
expect(measuring.steps.indexOf(journals)).toBeGreaterThan(measuring.steps.indexOf(measurement));
expect(measurement.with.path).toEqual(
"${{ runner.temp }}/measurements/test-weights-${{ matrix.runtime }}.json",
);
expect(journals.uses).toEqual(
"actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a",
);
expect(journals.if).toEqual("${{ failure() || cancelled() }}");
expect(journals.with).toEqual({
name: "measure-test-journals-${{ matrix.runtime }}-attempt-${{ github.run_attempt }}",
path: "${{ env.XMD_TEST_ARTIFACTS }}",
"if-no-files-found": "ignore",
"include-hidden-files": "true",
});
const download = assemble.steps.find((step) =>
step.uses?.startsWith("actions/download-artifact"),
);
expect(download?.with.pattern).toEqual("test-weights-*-attempt-${{ github.run_attempt }}");
expect(journals.with.name?.startsWith("test-weights-")).toBe(false);
});

it("assembles only after every runtime's job succeeded", function* () {
const { measure: measuring, assemble } = yield* workflowJobs();

Expand Down
7 changes: 7 additions & 0 deletions specs/testing-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -1067,6 +1067,13 @@ runner is outside it. Committing that artifact is a deliberate act, which is
what keeps the provenance true of the run that produced it. Milliseconds are
never edited by hand.

Each measuring job sets `XMD_TEST_ARTIFACTS` to the absolute workspace path
`${{ github.workspace }}/.xmd/test-artifacts`. On failure or cancellation it
uploads the complete saved journal trees, including hidden files, as
`measure-test-journals-<runtime>-attempt-<attempt>`; missing files are ignored.
These artifacts follow [Progress and failure journals](#progress-and-failure-journals)
and remain separate from the measurement records selected for assembly.

Counts are measured rather than chosen. For each runtime the lower bound is
`floor(sum of applicable weights / 420000) + 1`, and the installed count is the
smallest for which five consecutive runs on one fixed head keep every shard
Expand Down
Loading