Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 87 additions & 52 deletions .ai/skills/gha/SKILL.md

Large diffs are not rendered by default.

36 changes: 27 additions & 9 deletions .github/workflows/container-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@ on:

permissions:
contents: read
checks: read
actions: read
packages: write

cache-mode: none

jobs:
build-and-push:
name: "Build and push container images"
Expand Down Expand Up @@ -52,14 +54,17 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
run: |
check_name="clients_TypeScript"
set -euo pipefail
deadline=$((SECONDS + 3600))
while true; do
conclusions=$(gh api "repos/$GITHUB_REPOSITORY/commits/$TARGET_SHA/check-runs?check_name=$check_name" \
--jq '[.check_runs[].conclusion // "pending"] | unique | join(",")')
# Only this workflow's master push/manual runs can authorize publication.
# A successful PR check (including one with the same job name) cannot.
conclusions=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/continuous-integration.yaml/runs?head_sha=$TARGET_SHA&branch=master&per_page=100" |
jq -r --arg repo "$GITHUB_REPOSITORY" --arg sha "$TARGET_SHA" \
'[.workflow_runs[] | select(.head_repository.full_name == $repo and .head_sha == $sha and .head_branch == "master" and (.event == "push" or .event == "workflow_dispatch"))] | sort_by(.id) | last | if . == null then "" else (.conclusion // "pending") end')
case "$conclusions" in
success) echo "Continuous Integration passed"; break ;;
""|*pending*) echo "Continuous Integration pending, waiting 30s..." ;;
""|pending) echo "Continuous Integration pending, waiting 30s..." ;;
*) echo "::error::Continuous Integration concluded with: $conclusions"; exit 1 ;;
esac
if [ "$SECONDS" -ge "$deadline" ]; then
Expand Down Expand Up @@ -274,6 +279,15 @@ jobs:
# the two already-tagged '<base>-amd64' / '<base>-arm64' images by digest.
# No new single-arch manifests are created, so nothing is left untagged.
- name: Compose multi-arch manifests
env:
OGMIOS_BASE: ${{ steps.tags.outputs.ogmios-base }}
OGMIOS_TAGS: ${{ steps.tags.outputs.ogmios }}
MAINNET_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-mainnet-base }}
MAINNET_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-mainnet }}
PREPROD_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-preprod-base }}
PREPROD_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-preprod }}
PREVIEW_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-preview-base }}
PREVIEW_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-preview }}
run: |
set -euo pipefail
compose() {
Expand All @@ -286,7 +300,11 @@ jobs:
docker buildx imagetools create -t "$tag" "$repo@$amd64" "$repo@$arm64"
done
}
compose '${{ steps.tags.outputs.ogmios-base }}' ${{ steps.tags.outputs.ogmios }}
compose '${{ steps.tags.outputs.cardano-node-ogmios-mainnet-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-mainnet }}
compose '${{ steps.tags.outputs.cardano-node-ogmios-preprod-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-preprod }}
compose '${{ steps.tags.outputs.cardano-node-ogmios-preview-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-preview }}
read -r -a tags <<< "$OGMIOS_TAGS"
compose "$OGMIOS_BASE" "${tags[@]}"
read -r -a tags <<< "$MAINNET_TAGS"
compose "$MAINNET_BASE" "${tags[@]}"
read -r -a tags <<< "$PREPROD_TAGS"
compose "$PREPROD_BASE" "${tags[@]}"
read -r -a tags <<< "$PREVIEW_TAGS"
compose "$PREVIEW_BASE" "${tags[@]}"
75 changes: 40 additions & 35 deletions .github/workflows/continuous-integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,41 @@ on:
branches:
- master

permissions:
contents: read

cache-mode: none

env:
CARDANO_NODE_VERSION: "11.1.1"
# cardano-playground has no node-11.1.1-config tag yet, so configs stay
# pinned to the newest tagged set.
CARDANO_CONFIG_TAG: "node-11.0.1-config"
NETWORK: preview
S3_DB_PATH: s3://${{ secrets.AWS_S3_BUCKET }}/db-preview

jobs:
fork-checks:
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
submodules: true
persist-credentials: false

- name: Setup Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: lts/jod

- name: Build and lint TypeScript client
working-directory: clients/TypeScript
run: |
yarn install --frozen-lockfile --non-interactive --loglevel=error
yarn build
yarn lint

# TODO: switch back to this once Hydra reports check-runs to GitHub
# wait-for-hydra:
# runs-on: ubuntu-latest
Expand All @@ -39,6 +65,11 @@ jobs:
# done

clients_TypeScript:
# Repository writers are trusted to run integration tests with AWS access.
# This routes fork jobs; GitHub's fork secret/token restrictions enforce isolation.
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
env:
S3_DB_PATH: s3://${{ secrets.AWS_S3_BUCKET }}/db-preview
# needs:
# - wait-for-hydra
runs-on: ubuntu-24.04
Expand Down Expand Up @@ -120,34 +151,14 @@ jobs:
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}

# Read-only: Network Synchronization is the only writer of this cache, so
# concurrent pull request runs cannot delete each other's objects.
- name: Restore cardano-node DB from S3
shell: bash {0}
env:
RUNNER_TEMP: ${{ runner.temp }}
run: |
mkdir -p "$RUNNER_TEMP/db-preview"
for attempt in 1 2 3; do
echo "S3 sync attempt $attempt..."
nix shell nixpkgs#s5cmd -c \
s5cmd --log error sync "$S3_DB_PATH/*" "$RUNNER_TEMP/db-preview/" > /tmp/s3-sync.log 2>&1 &
pid=$!
while kill -0 "$pid" 2>/dev/null; do
du -sh "$RUNNER_TEMP/db-preview" 2>/dev/null
sleep 15
done
wait "$pid"
rc=$?
echo "S3 sync attempt $attempt finished (exit $rc). Last 30 lines of log:"
tail -30 /tmp/s3-sync.log
if [ "$rc" -eq 0 ]; then
echo "S3 sync succeeded"
break
fi
if [ "$attempt" -eq 3 ]; then
exit 1
fi
echo "Retrying..."
done
nix shell nixpkgs#s5cmd -c \
./.github/workflows/scripts/preview-db-restore.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview"

- name: Setup Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
Expand All @@ -157,7 +168,7 @@ jobs:
- name: Build TypeScript client
working-directory: clients/TypeScript
run: |
yarn install --frozen-lockfile --non-interactive --logevel=error
yarn install --frozen-lockfile --non-interactive --loglevel=error
yarn build
yarn lint

Expand All @@ -182,16 +193,10 @@ jobs:
working-directory: clients/TypeScript
run: yarn test

# A node under load takes well over the 10s default to flush and write its
# clean-shutdown marker; without it the next start revalidates everything.
- name: Stop cardano-node
if: always() && steps.sync.outcome != 'skipped'
run: |
docker stop cardano-node || true
docker stop -t 300 cardano-node || true
docker rm cardano-node || true

- name: Save cardano-node DB to S3
if: always() && steps.sync.outcome == 'success'
env:
RUNNER_TEMP: ${{ runner.temp }}
run: |
nix shell nixpkgs#s5cmd -c \
s5cmd --log error sync "$RUNNER_TEMP/db-preview/" "$S3_DB_PATH/"
82 changes: 30 additions & 52 deletions .github/workflows/network-synchronization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ on:
default: 1
type: number

permissions:
contents: read

cache-mode: none

env:
CARDANO_NODE_VERSION: "11.1.1"
# cardano-playground has no node-11.1.1-config tag yet, so configs stay
Expand All @@ -21,8 +26,17 @@ env:

jobs:
sync_and_cache:
# One writer across scheduled/manual runs and all selected refs.
concurrency:
group: preview-db-cache-writer
cancel-in-progress: false
runs-on: ubuntu-24.04
steps:
- name: Checkout repository
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
persist-credentials: false

- name: Download cardano-node configs
env:
BASE_URL: https://raw.githubusercontent.com/input-output-hk/cardano-playground/refs/tags/${{ env.CARDANO_CONFIG_TAG }}/docs/environments/preview
Expand All @@ -49,75 +63,39 @@ jobs:
substituters = https://cache.iog.io/ https://cache.nixos.org/

- name: Restore cardano-node DB from S3
shell: bash {0}
id: restore
env:
RUNNER_TEMP: ${{ runner.temp }}
run: |
mkdir -p "$RUNNER_TEMP/db-preview"
for attempt in 1 2 3; do
echo "S3 sync attempt $attempt..."
nix shell nixpkgs#s5cmd -c \
s5cmd --log error sync "$S3_DB_PATH/*" "$RUNNER_TEMP/db-preview/" > /tmp/s3-sync.log 2>&1 &
pid=$!
while kill -0 "$pid" 2>/dev/null; do
du -sh "$RUNNER_TEMP/db-preview" 2>/dev/null
sleep 15
done
wait "$pid"
rc=$?
echo "S3 sync attempt $attempt finished (exit $rc). Last 30 lines of log:"
tail -30 /tmp/s3-sync.log
if [ "$rc" -eq 0 ]; then
echo "S3 sync succeeded"
break
fi
if [ "$attempt" -eq 3 ]; then
exit 1
fi
echo "Retrying..."
done
nix shell nixpkgs#s5cmd -c \
./.github/workflows/scripts/preview-db-restore.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview"

- name: Sync node
id: sync
timeout-minutes: 300
env:
CONFDIR: ${{ github.workspace }}/config/preview
RUNNER_TEMP: ${{ runner.temp }}
SYNC_LEVEL: ${{ inputs.synchronization-level || 1 }}
# Stringify without JSON quotes; the nonempty string '0' preserves zero.
SYNC_LEVEL: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.synchronization-level) || '1' }}
run: |
docker run -d --name cardano-node \
-v "$RUNNER_TEMP/db-preview:/db" \
-v "$RUNNER_TEMP/ipc:/ipc" \
-v "$CONFDIR:/config" \
"ghcr.io/intersectmbo/cardano-node:$CARDANO_NODE_VERSION" run --config /config/config.json --database-path /db --socket-path /ipc/node.socket --topology /config/topology.json

echo "Waiting for node socket..."
while [ ! -S "$RUNNER_TEMP/ipc/node.socket" ]; do
sleep 5
done

echo "Waiting for sync to reach $SYNC_LEVEL..."
while true; do
tip=$(docker exec cardano-node cardano-cli query tip --testnet-magic 2 2>/dev/null) || { sleep 5; continue; }
sync=$(echo "$tip" | jq -r '.syncProgress // "0"' | tr -d '%')
pct=$(echo "scale=4; $sync / 100" | bc)
echo "Sync progress: ${sync}%"
if [ "$(echo "$pct >= $SYNC_LEVEL" | bc)" -eq 1 ]; then
echo "Reached target sync level"
break
fi
sleep 30
done
./.github/workflows/scripts/preview-node-sync.sh "$SYNC_LEVEL"

# A node under load takes well over the 10s default to flush and write its
# clean-shutdown marker; without it the next start revalidates everything.
- name: Stop cardano-node
id: stop
if: always() && steps.sync.outcome != 'skipped'
run: |
docker stop cardano-node || true
docker rm cardano-node || true
docker stop -t 300 cardano-node
docker rm cardano-node

# Save partial sync progress only after a successful restore and shutdown.
# The save script also requires the node's clean-shutdown marker.
- name: Save cardano-node DB to S3
if: always() && steps.sync.outcome != 'skipped'
if: always() && steps.restore.outcome == 'success' && steps.sync.outcome != 'skipped' && steps.stop.outcome == 'success'
env:
RUNNER_TEMP: ${{ runner.temp }}
run: |
nix shell nixpkgs#s5cmd -c \
s5cmd --log error sync "$RUNNER_TEMP/db-preview/" "$S3_DB_PATH/"
./.github/workflows/scripts/preview-db-save.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview"
11 changes: 7 additions & 4 deletions .github/workflows/release-upload.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,9 @@ on:
- '**'

permissions:
contents: write
actions: read
contents: read

env:
GH_TOKEN: ${{ github.token }}
cache-mode: none

jobs:
# TODO: switch back to this once Hydra reports check-runs to GitHub
Expand Down Expand Up @@ -119,6 +117,9 @@ jobs:
retention-days: 1

upload-assets:
permissions:
contents: write
actions: read
needs:
- pull
name: "Upload Assets"
Expand All @@ -132,6 +133,8 @@ jobs:
run: sha256sum ogmios-*.tar.gz > ogmios-sha256sums.txt

- name: Release
env:
GH_TOKEN: ${{ github.token }}
uses: input-output-hk/action-gh-release@26994186c0ac3ef5cae75ac16aa32e8153525f77 # v1
with:
draft: true
Expand Down
38 changes: 38 additions & 0 deletions .github/workflows/scripts/preview-db-common.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/usr/bin/env bash

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# Helpers shared by the preview database restore and save scripts.
# Requires s5cmd and jq. An empty prefix is different from a failed listing.

cache_list() {
local prefix="$1" scratch rc=0
scratch=$(mktemp -d)
s5cmd --json ls "$prefix" > "$scratch/objects" 2> "$scratch/errors" || rc=$?
if [ "$rc" -eq 0 ]; then
jq -s '.' "$scratch/objects" || rc=$?
elif jq -e -s 'length > 0 and all(.[]; .operation == "ls" and .error == "no object found")' \
"$scratch/errors" >/dev/null 2>&1 && [ ! -s "$scratch/objects" ]; then
printf '[]\n'
rc=0
else
cat "$scratch/errors" >&2
fi
rm -rf -- "$scratch"
return "$rc"
}

snapshot_names() {
jq -r '.[] | select(.type == "directory") | .key | rtrimstr("/") | split("/") | last
| select(test("^[0-9]+(_[A-Za-z0-9_-]+)?$"))'
}

check_cache_args() {
if [[ ! "$S3_DB_PATH" =~ ^s3://[^/]+/.+ ]] || [[ "$S3_DB_PATH" == *[\*\?\[]* ]] \
|| [ -z "$DB_DIR" ] || [ "$DB_DIR" = / ]; then
echo "Usage: $0 s3://BUCKET/PREFIX DB_DIR" >&2
exit 1
fi
}
Loading
Loading