Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 75 additions & 36 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,52 @@ jobs:
- name: ephor
run: zig build --build-file tools/ephor/build.zig run -- --summary --fail-on=note

# The one compile the jobs below share. Each of them links the same gRPC, abseil and
# BoringSSL objects, about ten minutes of C++ on this runner, and a restored Zig cache does
# not spare a second job that work: Zig rejects a cached C or C++ object once its source
# file has a new inode (https://codeberg.org/ziglang/zig/issues/37200). So this job
# compiles and the others receive binaries.
#
# zig-out holds tests/ (debug), bin/ (the release tree `dist` installs) and harness/.
# The key is the commit, so re-running a failed job finds these binaries again.
build:
name: build (linux-x64)
needs: shim
runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3
with:
egress-policy: audit

- uses: runs-on/action@284bff869d4a825f778ac7c38e50f1e31ef198fd # v2.6.0
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- id: shim
uses: ./.github/actions/shim
with:
platform: linux-x64
- id: zig-out
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: zig-out
key: zig-out-linux-x64-${{ github.sha }}
- if: steps.zig-out.outputs.cache-hit != 'true'
uses: ./.github/actions/zig
- name: zig build install-test
if: steps.zig-out.outputs.cache-hit != 'true'
run: zig build install-test -Dshim-dylib=${{ steps.shim.outputs.path }}
- name: zig build dist install-harness
if: steps.zig-out.outputs.cache-hit != 'true'
run: >-
zig build dist install-harness -Drelease=true
-Dshim-dylib=${{ steps.shim.outputs.path }}
-Dshim-static-obj=${{ steps.shim.outputs.static-path }}

# The test binaries and harnesses below load the shim dylib from dist/shim, the path they
# were linked against, so each job that runs them restores it.
test:
name: unit tests
needs: shim
needs: build
runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache
steps:
- name: Harden the runner (Audit all outbound calls)
Expand All @@ -69,13 +112,17 @@ jobs:
uses: ./.github/actions/shim
with:
platform: linux-x64
- uses: ./.github/actions/zig
- run: zig build test -Dshim-dylib=${{ steps.shim.outputs.path }}
- run: zig build test-server -Dshim-dylib=${{ steps.shim.outputs.path }}
- uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: zig-out
key: zig-out-linux-x64-${{ github.sha }}
fail-on-cache-miss: true
- run: ./zig-out/tests/engine-tests
- run: ./zig-out/tests/server-tests

spec:
name: spec (googlesqlite + go-zetasqlite)
needs: shim
needs: build
runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache
steps:
- name: Harden the runner (Audit all outbound calls)
Expand All @@ -89,17 +136,16 @@ jobs:
uses: ./.github/actions/shim
with:
platform: linux-x64
- uses: ./.github/actions/zig
- uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: zig-out
key: zig-out-linux-x64-${{ github.sha }}
fail-on-cache-miss: true

# --strict is what makes this a real gate: without it the runner exits
# nonzero only on a baseline regression, so a case that has never passed
# can start returning wrong answers and still go green. With it, a failing
# case has to be listed in testdata/spec_known_failures.txt.
- name: zig build spec --strict
- name: spec-runner --strict
run: |
set -euo pipefail
zig build spec -Drelease=true \
-Dshim-dylib=${{ steps.shim.outputs.path }} -- --strict | tee spec.log
./zig-out/harness/spec-runner --strict | tee spec.log

# Reuses the run above rather than invoking --markdown, which would
# replay all cases a second time for a per-category table.
Expand All @@ -111,12 +157,12 @@ jobs:

# The Python client suite ported from Recidiviz/bigquery-emulator, run the
# way downstream fixtures consume the emulator: testcontainers against the
# Docker image, built here from the same static `dist` binary a release
# Docker image, built here from the same static `dist` binaries a release
# ships. No other job builds the image before a tag, so this is also what
# catches a Dockerfile or ENTRYPOINT that no longer starts.
python-integration:
name: python integration (Docker image)
needs: shim
needs: build
runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache
steps:
- name: Harden the runner (Audit all outbound calls)
Expand All @@ -126,23 +172,20 @@ jobs:

- uses: runs-on/action@284bff869d4a825f778ac7c38e50f1e31ef198fd # v2.6.0
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- id: shim
uses: ./.github/actions/shim
- uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
platform: linux-x64
- uses: ./.github/actions/zig
path: zig-out
key: zig-out-linux-x64-${{ github.sha }}
fail-on-cache-miss: true

# Staged outside the checkout: the Dockerfile copies dist/bin/, and a
# context rooted at the repo would also carry dist/shim/ (~150 MB of
# cached shim) to the daemon.
# context rooted at the repo would carry the whole tree to the daemon.
- name: build image
run: |
set -euo pipefail
ctx="$RUNNER_TEMP/image"
zig build dist -Drelease=true \
-Dshim-dylib=${{ steps.shim.outputs.path }} \
-Dshim-static-obj=${{ steps.shim.outputs.static-path }} \
--prefix "$ctx/dist"
mkdir -p "$ctx/dist"
cp -r zig-out/bin "$ctx/dist/bin"
cp LICENSE NOTICE THIRD_PARTY_NOTICES "$ctx/dist/"
docker build -f Dockerfile -t zigquery-emulator:ci "$ctx"

Expand All @@ -165,7 +208,7 @@ jobs:
# python-integration's. It covers the client paths downstream Node services take.
node-integration:
name: node integration (Docker image)
needs: shim
needs: build
runs-on: runs-on=${{ github.run_id }}/runner=zig-x64/extras=s3-cache
steps:
- name: Harden the runner (Audit all outbound calls)
Expand All @@ -175,22 +218,18 @@ jobs:

- uses: runs-on/action@284bff869d4a825f778ac7c38e50f1e31ef198fd # v2.6.0
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- id: shim
uses: ./.github/actions/shim
with:
platform: linux-x64
- uses: ./.github/actions/zig
- uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
version: ${{ env.ZIG_VERSION }}
path: zig-out
key: zig-out-linux-x64-${{ github.sha }}
fail-on-cache-miss: true

- name: build image
run: |
set -euo pipefail
ctx="$RUNNER_TEMP/image"
zig build dist -Drelease=true \
-Dshim-dylib=${{ steps.shim.outputs.path }} \
-Dshim-static-obj=${{ steps.shim.outputs.static-path }} \
--prefix "$ctx/dist"
mkdir -p "$ctx/dist"
cp -r zig-out/bin "$ctx/dist/bin"
cp LICENSE NOTICE THIRD_PARTY_NOTICES "$ctx/dist/"
docker build -f Dockerfile -t zigquery-emulator:ci "$ctx"

Expand Down
15 changes: 15 additions & 0 deletions build.zig
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,21 @@ pub fn build(b: *std.Build) void {
dist_step.dependOn(&b.addInstallArtifact(exe, .{}).step);
dist_step.dependOn(&b.addInstallArtifact(server_exe, .{}).step);

// ---- harnesses ----
//
// The executables behind `spec`, `analyzer-threads`, `shared-catalog-threads` and
// `grpc-smoke`, installed without running them, so one machine can build them and
// another run them. They get their own directory because `bin/` is the release tree.
const harness_step = b.step(
"install-harness",
"Install the spec runner and concurrency harnesses to zig-out/harness/",
);
for ([_]*std.Build.Step.Compile{ spec_exe, threads_exe, sc_threads_exe, grpcsmoke_exe }) |h| {
harness_step.dependOn(&b.addInstallArtifact(h, .{
.dest_dir = .{ .override = .{ .custom = "harness" } },
}).step);
}

// ---- coverage ----
//
// kcov (`brew install kcov`) drives a test binary under a debugger and
Expand Down
1 change: 1 addition & 0 deletions docs/BUILD.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ zig build server -- --project=my-project [--dataset=D] [--host=0.0.0.0] [--port=
|----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `install-spec` | Install only `zig-out/bin/spec-runner`. Faster than `install` when you want a debug target for one spec case. |
| `install-test` | Install the `test` and `test-server` binaries to `zig-out/tests/engine-tests` and `zig-out/tests/server-tests`, so a debugger has a stable path to attach to. These are the same artifacts the test steps run, not a second compile. |
| `install-harness` | Install `spec-runner`, `analyzer-threads`, `shared-catalog-threads` and `grpc-smoke` to `zig-out/harness/` without running them. CI builds them once this way and runs them in other jobs. |
| `spec-ui` | Run the corpus and browse the results in a local web UI. `zig build spec-ui -- [--filter SUBSTR] [--dialect D] [--port N]`; the default port is 8091. |

The checked-in ZigBrains run configurations in `.run/` use `install-spec` and
Expand Down
Loading