diff --git a/.github/workflows/measure-test-weights.yml b/.github/workflows/measure-test-weights.yml index 6213bca85..109d5628f 100644 --- a/.github/workflows/measure-test-weights.yml +++ b/.github/workflows/measure-test-weights.yml @@ -36,6 +36,8 @@ jobs: runtime: [deno, node, bun] runs-on: ubuntu-latest timeout-minutes: 165 + env: + XMD_TEST_ARTIFACTS: ${{ github.workspace }}/.xmd/test-artifacts steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -94,6 +96,15 @@ jobs: path: ${{ runner.temp }}/measurements/test-weights-${{ matrix.runtime }}.json if-no-files-found: warn + - name: Upload the ${{ matrix.runtime }} failure journals + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: measure-test-journals-${{ matrix.runtime }}-attempt-${{ github.run_attempt }} + path: ${{ env.XMD_TEST_ARTIFACTS }} + if-no-files-found: ignore + include-hidden-files: true + # Runs only when every runtime's job succeeded: a failed, cancelled or # timed-out runtime skips it, so no `test-weights` artifact exists for a run # that did not measure the whole corpus. diff --git a/packages/cli/tests/repl-headless-submit-process.test.ts b/packages/cli/tests/repl-headless-submit-process.test.ts index 58fa89d09..43cf6a77e 100644 --- a/packages/cli/tests/repl-headless-submit-process.test.ts +++ b/packages/cli/tests/repl-headless-submit-process.test.ts @@ -212,9 +212,11 @@ describe("first SIGINT at the submit host boundary", () => { expect(call.closed).toBe(false); expect(call.records.some((record) => record.kind === "finished")).toBe(false); released.resolve(); - expect( - yield* within(call.joined, () => ({ waiting: "joined Stop", ...call.output() })), - ).toEqual({ code: 130, signal: null }); + const exit = yield* within(call.joined, () => ({ waiting: "joined Stop", ...call.output() })); + if (exit.code !== 130 || exit.signal !== null) { + console.error(JSON.stringify({ ...exit, ...call.output(), records: call.records })); + } + expect(exit).toEqual({ code: 130, signal: null }); const admitted = call.records[0]; const final = call.records.at(-1); expect(final).toMatchObject({ diff --git a/scripts/tests/measure-test-weights.test.ts b/scripts/tests/measure-test-weights.test.ts index c67f06e8d..9ac1dec8c 100644 --- a/scripts/tests/measure-test-weights.test.ts +++ b/scripts/tests/measure-test-weights.test.ts @@ -442,6 +442,41 @@ describe("the measurement workflow", () => { ); }); + it("retains failure journals separately from measurement records", function* () { + const workflow = yield* document(WORKFLOW); + const jobs = object(workflow.jobs, "workflow.jobs"); + const raw = object(jobs.measure, "workflow.jobs.measure"); + expect(strings(raw.env, "workflow.jobs.measure.env").XMD_TEST_ARTIFACTS).toEqual( + "${{ github.workspace }}/.xmd/test-artifacts", + ); + const { measure: measuring, assemble } = yield* workflowJobs(); + const [measurement, journals] = uploads(measuring); + if (measurement === undefined || journals === undefined) { + throw new Error("measurement and journal uploads are required"); + } + const measure = stepRunning(measuring, "weights:measure"); + expect(measuring.steps.indexOf(measurement)).toBeGreaterThan(measuring.steps.indexOf(measure)); + expect(measuring.steps.indexOf(journals)).toBeGreaterThan(measuring.steps.indexOf(measurement)); + expect(measurement.with.path).toEqual( + "${{ runner.temp }}/measurements/test-weights-${{ matrix.runtime }}.json", + ); + expect(journals.uses).toEqual( + "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a", + ); + expect(journals.if).toEqual("${{ failure() || cancelled() }}"); + expect(journals.with).toEqual({ + name: "measure-test-journals-${{ matrix.runtime }}-attempt-${{ github.run_attempt }}", + path: "${{ env.XMD_TEST_ARTIFACTS }}", + "if-no-files-found": "ignore", + "include-hidden-files": "true", + }); + const download = assemble.steps.find((step) => + step.uses?.startsWith("actions/download-artifact"), + ); + expect(download?.with.pattern).toEqual("test-weights-*-attempt-${{ github.run_attempt }}"); + expect(journals.with.name?.startsWith("test-weights-")).toBe(false); + }); + it("assembles only after every runtime's job succeeded", function* () { const { measure: measuring, assemble } = yield* workflowJobs(); diff --git a/specs/testing-spec.md b/specs/testing-spec.md index 8286823dd..af853231d 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -1067,6 +1067,13 @@ runner is outside it. Committing that artifact is a deliberate act, which is what keeps the provenance true of the run that produced it. Milliseconds are never edited by hand. +Each measuring job sets `XMD_TEST_ARTIFACTS` to the absolute workspace path +`${{ github.workspace }}/.xmd/test-artifacts`. On failure or cancellation it +uploads the complete saved journal trees, including hidden files, as +`measure-test-journals--attempt-`; missing files are ignored. +These artifacts follow [Progress and failure journals](#progress-and-failure-journals) +and remain separate from the measurement records selected for assembly. + Counts are measured rather than chosen. For each runtime the lower bound is `floor(sum of applicable weights / 420000) + 1`, and the installed count is the smallest for which five consecutive runs on one fixed head keep every shard