From e0c823cd8675cc040ec59b4c9ac7b30ee9c22c61 Mon Sep 17 00:00:00 2001 From: Dan Hansen Date: Thu, 12 Nov 2026 22:41:36 -0800 Subject: [PATCH] ci: remove the benchmark workflow and the concurrency harness job Delete bench.yml and the concurrency-harness job in ci.yml, along with the bench-x64 runner only bench.yml used. The zig build steps (gsql-bench, analyzer-threads, shared-catalog-threads, grpc-smoke) stay available locally. Co-Authored-By: Claude Fable 5.2 --- .github/runs-on.yml | 10 ----- .github/workflows/bench.yml | 85 ------------------------------------- .github/workflows/ci.yml | 30 ------------- README.md | 3 +- docs/BUILD.md | 5 +-- 5 files changed, 3 insertions(+), 130 deletions(-) delete mode 100644 .github/workflows/bench.yml diff --git a/.github/runs-on.yml b/.github/runs-on.yml index 25a7a9b..d4fe980 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -38,13 +38,3 @@ runners: family: ["c7g", "m7g"] image: ubuntu24-full-arm64 volume: 200gb:gp3:750mbs:4000iops - - # Benchmarks. Pinned to one instance family and spot: false — a run that - # lands on a different CPU, or gets interrupted and retried elsewhere, is - # not comparable with the history it is being charted against. - bench-x64: - cpu: 8 - family: ["c7a"] - image: ubuntu24-full-x64 - volume: 120gb:gp3:500mbs:4000iops - spot: false diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index 8d3af64..0000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Benchmarks - -# Deliberately not on every PR. EC2 timings carry real variance, so charting -# every push to a shared branch keeps the trend meaningful while a `bench` -# label opts an individual PR in when a change is expected to move numbers. -on: - push: - # See the note in ci.yml about server-port. - branches: [master, main, server-port] - pull_request: - types: [labeled] - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -concurrency: - group: bench-${{ github.ref }} - cancel-in-progress: false - -jobs: - shim: - if: >- - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'bench') - name: shim (linux-x64) - uses: ./.github/workflows/shim.yml - with: - platform: linux-x64 - runner: runs-on=${{ github.run_id }}/runner=shim-x64/extras=s3-cache - - gsql-bench: - name: gsql-bench - needs: shim - # bench-x64 is spot:false and pinned to one instance family on purpose — - # a run that lands on different silicon, or gets reclaimed and retried - # elsewhere, is not comparable with the history it is charted against. - runs-on: runs-on=${{ github.run_id }}/runner=bench-x64/extras=s3-cache - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 - with: - egress-policy: audit - - - uses: runs-on/action@284bff869d4a825f778ac7c38e50f1e31ef198fd # v2.6.0 - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - id: shim - uses: ./.github/actions/shim - with: - platform: linux-x64 - - uses: ./.github/actions/zig - - # --json must be the only thing on stdout; the harness prints its - # human table on the other branch of the same switch. - - name: run gsql-bench - run: | - set -euo pipefail - zig build gsql-bench -Drelease=true \ - -Dshim-dylib=${{ steps.shim.outputs.path }} -- --json > bench.json - python3 -c "import json;d=json.load(open('bench.json'));print(len(d),'metrics')" - - - name: readable table for the run summary - run: | - { echo '## gsql-bench'; echo; } >> "$GITHUB_STEP_SUMMARY" - zig build gsql-bench -Drelease=true \ - -Dshim-dylib=${{ steps.shim.outputs.path }} -- --markdown \ - >> "$GITHUB_STEP_SUMMARY" - - # fail-on-alert stays false: on shared cloud hardware a hard gate here - # would fire on noise, not regressions. The alert comment is the signal; - # a real regression is confirmed by re-running or bisecting locally. - - uses: benchmark-action/github-action-benchmark@4322e5726e6334590d251fc4f92bec0efafc45dc # v1.22.2 - with: - name: gsql-bench (goccy/googlesqlite corpus) - tool: customSmallerIsBetter - output-file-path: bench.json - github-token: ${{ secrets.GITHUB_TOKEN }} - gh-pages-branch: benchmarks - benchmark-data-dir-path: dev/bench - auto-push: ${{ github.event_name != 'pull_request' }} - alert-threshold: "150%" - comment-on-alert: true - fail-on-alert: false - summary-always: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cff2428..86b5810 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,36 +108,6 @@ jobs: { echo '## Spec'; echo; echo '```'; head -20 spec.log; echo '```'; } \ >> "$GITHUB_STEP_SUMMARY" - # Not benchmarks: every harness here carries a positive control (GENERATE_UUID - # uniqueness; per-connection catalogs that MUST fail; a gRPC status that must - # come from our handler rather than the transport) and exits nonzero when the - # control does not behave, so a broken build cannot quietly pass them. - concurrency-harness: - name: concurrency harnesses - needs: shim - runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 - with: - egress-policy: audit - - - uses: runs-on/action@284bff869d4a825f778ac7c38e50f1e31ef198fd # v2.6.0 - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - id: shim - uses: ./.github/actions/shim - with: - platform: linux-x64 - - uses: ./.github/actions/zig - - run: zig build analyzer-threads -Drelease=true -Dshim-dylib=${{ steps.shim.outputs.path }} - - run: zig build shared-catalog-threads -Drelease=true -Dshim-dylib=${{ steps.shim.outputs.path }} - # Same contract: the Storage Read transport answers UNIMPLEMENTED, and - # three controls (an unknown service must reach our handler too, a second - # call on one channel must be answered, and the listener must be gone - # after stop()) mean a build that merely links cannot pass. It binds port - # 0, so it cannot collide with anything else on the runner. - - run: zig build grpc-smoke -Drelease=true -Dshim-dylib=${{ steps.shim.outputs.path }} - # The Python client suite ported from Recidiviz/bigquery-emulator, run the # way downstream fixtures consume the emulator: testcontainers against the # Docker image, built here from the same static `dist` binary a release diff --git a/README.md b/README.md index 469ed23..958164a 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,7 @@ Intentional, explicit control over memory allocation improves process stability. | Workflow | Trigger | What it does | |---------------|-----------------------------|---------------------------------------------------------------------------------------------------------------------------| -| `ci.yml` | PR, push | ephor + `zig fmt`, unit tests, the spec corpus under `--strict`, the concurrency harnesses, and a `gen-proto` drift check | -| `bench.yml` | push, `bench` label, manual | `gsql-bench --json` charted by github-action-benchmark on the `benchmarks` branch | +| `ci.yml` | PR, push | ephor + `zig fmt`, unit tests, the spec corpus under `--strict`, and a `gen-proto` drift check | | `release.yml` | `v*.*.*` tag | four platform tarballs, a multi-arch GHCR image, and a GitHub release | Runners are powered by [RunsOn](https://runs-on.com) on EC2 (see `.github/runs-on.yml`). diff --git a/docs/BUILD.md b/docs/BUILD.md index e95076e..5c09cd1 100644 --- a/docs/BUILD.md +++ b/docs/BUILD.md @@ -211,8 +211,7 @@ See `docs/COVERAGE.md` before quoting a number. ### Harnesses with positive controls Not benchmarks, and not part of `test`: each carries a control that must fail, -and exits nonzero when it does not, so a broken build cannot quietly pass. CI -runs all three at `-Drelease=true`. +and exits nonzero when it does not, so a broken build cannot quietly pass. | Step | Arguments | What it checks | |--------------------------|-----------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -245,7 +244,7 @@ are not comparable. | Step | Arguments | What it measures | |----------------|---------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `bench` | `[queries_per_size] [catalog_sizes...]` (default 2000 over 1, 50, 500) | Per-query cost as a function of catalog size, with a fresh `CREATE TABLE` between timed queries so the catalog delta is never empty. | -| `gsql-bench` | `[--time MS] [--iters N] [--filter SUBSTR] [--corpus DIR] [--phases] [--markdown] [--json]` | The googlesqlite corpus vendored under `testdata/bench_corpus`, reported in the same shape as upstream's RESULTS.md. `--json` is what `bench.yml` charts; `--markdown` goes to the run summary. Default budget 1000 ms per query. | +| `gsql-bench` | `[--time MS] [--iters N] [--filter SUBSTR] [--corpus DIR] [--phases] [--markdown] [--json]` | The googlesqlite corpus vendored under `testdata/bench_corpus`, reported in the same shape as upstream's RESULTS.md. Default budget 1000 ms per query. | | `codec-bench` | `[rows]` (default 200000) | Value codec cost with real rows. Needs `-Dcodec-timers=true`. | | `tz-bench` | `[rows]` (default 200000) | Per-row cost of named-timezone resolution, the A/B point for the ICU tzdata integration. | | `load-bench` | none | Whether a delete forces work proportional to catalog size. Run-only, not installed. |