From a99b59dc8b07edc8e22dc4e457e08069edd3f2e2 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 12 Nov 2026 16:32:29 -0600 Subject: [PATCH 1/5] ai: update gha skill --- .ai/skills/gha/SKILL.md | 139 +++++++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 52 deletions(-) diff --git a/.ai/skills/gha/SKILL.md b/.ai/skills/gha/SKILL.md index 55e32f0d2..936f6e522 100644 --- a/.ai/skills/gha/SKILL.md +++ b/.ai/skills/gha/SKILL.md @@ -1,15 +1,15 @@ --- name: hardening-github-actions -description: Writes and reviews GitHub Actions workflows with security hardening. Use when creating, modifying, or reviewing .github/workflows/*.yml or .github/actions/*/action.yml files. Covers shell injection prevention, trust gating for fork PRs, action pinning, and secrets hygiene. +description: Writes and reviews GitHub Actions workflows with security hardening. Use when creating, modifying, or reviewing .github/workflows/*.yml or .github/actions/*/action.yml files. Covers shell injection prevention, unprivileged fork CI, trusted publishing workflows, action pinning, and secrets hygiene. --- # Hardening GitHub Actions -This guidance assumes a **public repository** threat model where untrusted users can open fork PRs. The fork trust-gating sections are primarily relevant in that context. Shell injection prevention, action pinning, credential hygiene, and checkout persistence apply to all repos. +This guidance assumes a **public repository** threat model where untrusted users can open fork PRs. The fork CI sections are primarily relevant in that context. Shell injection prevention, action pinning, credential hygiene, and checkout persistence apply to all repos. ## Shell injection prevention -**Never interpolate `${{ }}` expressions directly into `run:` blocks.** GitHub expression substitution happens *before* the shell parses the command. Attacker-controlled values (branch names, PR titles, input fields) can inject arbitrary shell commands. +**Never interpolate `${{ }}` expressions directly into `run:` blocks, even for trusted values.** Pass values through `env:` and use quoted shell variables instead. GitHub expression substitution happens *before* the shell parses the command. This uniform rule avoids trust-classification mistakes and remains safe if a value's source changes later. Unsafe values include anything derived from: - `github.event.pull_request.head.ref` (fork branch names allow shell metacharacters) @@ -20,76 +20,109 @@ Unsafe values include anything derived from: ### Fix: use intermediate environment variables -Pass untrusted expressions via `env:` on the step, then reference them as quoted shell variables. Environment variables are set at runtime and are not subject to shell expansion. +Pass expressions via `env:` on the step, then reference them as quoted shell variables. Their contents are not reparsed as shell syntax. Validate values for the receiving command too: quoting alone does not prevent option injection, unsafe paths, or execution through `eval`. ```yaml # BAD - shell injection via branch name - run: git fetch origin ${{ github.event.pull_request.head.ref }} -# GOOD - safe via env +# Prevents shell injection; validate the ref for the intended operation too. - env: HEAD_REF: ${{ github.event.pull_request.head.ref }} run: git fetch origin "$HEAD_REF" ``` -This applies to **all** `${{ }}` references in `run:` blocks that touch event data or action inputs. Expressions used only in `with:`, `if:`, or `env:` values (not shell) are fine. +This applies to **all** `${{ }}` references in `run:` blocks, including SHAs, numeric IDs, constants, and trusted inputs. Other YAML fields do not automatically make input safe: an action may execute a `with:` input (for example, `actions/github-script`'s `script`), and tools may interpret environment variables as executable configuration. + +This is a placement rule, not a requirement to add format validation for every context value. GitHub-generated `github.sha`, `github.run_id`, and `github.run_number` need no extra hex/numeric checks solely for shell-injection prevention. Likewise, do not add username/repository-name regex checks to `github.actor` or `github.repository` merely to pass them through `env:`. Validate attacker-supplied inputs and operation-specific requirements where needed; authorization checks remain separate. Ref: https://docs.github.com/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable -### Safe contexts (no fix needed) +### Distinguish shell injection from authorization + +Expressions remain appropriate in data and condition fields such as `env:`, `if:`, `run-name:`, and `concurrency.group:`. Safe expression placement does not establish authorization: actor identity and the executing repository's name do not prove where the PR's code came from. -- `github.actor` (alphanumeric + hyphens only) -- `github.repository` (org/repo, restricted charset) -- `github.sha`, `github.run_id`, `github.run_number` (hex/numeric) -- Values used only in `if:`, `with:`, `run-name:`, `concurrency.group:`, or other non-shell YAML fields +## Secure fork CI and privileged workflows -## Trust gating for fork PRs +Fork PRs on `pull_request` triggers normally get a read-only `GITHUB_TOKEN` and no repository secrets (the token itself remains available). These platform restrictions are the security boundary; verify repository settings do not grant fork runs write tokens or secrets. The workflow runs in the PR merge context, so a fork can change workflow definitions and checked-out composite actions as well as build code. -Fork PRs on `pull_request` triggers get a read-only `GITHUB_TOKEN` and empty `secrets.*` (platform-enforced). But composite actions, checkout steps, and shell commands still execute. Structure workflows to avoid running untrusted code with privileges. +### Recommended: unprivileged PR CI, separate trusted publishing -### Preferred: two-job structure +Run fork builds and tests with `pull_request`, read-only permissions, no repository secrets, and GitHub-hosted runners. Do not give these jobs access to privileged self-hosted runners or internal services. The following is a standalone PR workflow; adapt the test command to the project: ```yaml +name: PR checks +on: + pull_request: +permissions: + contents: read jobs: - detect: + test: runs-on: ubuntu-latest - outputs: - is_trusted: ${{ steps.check.outputs.is_trusted }} steps: - - uses: actions/checkout@ + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: - ref: ${{ github.event.pull_request.base.sha || github.sha }} - - id: check - run: # ... determine trust level + persist-credentials: false + # Install any required tools with SHA-pinned actions. + - run: ./ci/test.sh +``` + +Put publishing or deployment in a **separate workflow** triggered by a push to a protected branch after review. Require review of workflow changes through branch protection/rulesets, and restrict bypass permissions. Build from that push's commit rather than a PR ref or artifacts from a fork run. For example, adapt `main` and the illustrative build/publish scripts below: - build: - needs: detect - if: needs.detect.outputs.is_trusted == 'true' +```yaml +name: Publish reviewed code +on: + push: + branches: [main] +permissions: + contents: read +cache-mode: none +jobs: + publish: runs-on: ubuntu-latest + permissions: + contents: write # Example: publishing a GitHub release. steps: - # ... privileged steps with secrets + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + ref: ${{ github.sha }} + persist-credentials: false + # Install any required tools with SHA-pinned actions. + - run: ./ci/build-release.sh + - run: ./ci/publish-release.sh + env: + GH_TOKEN: ${{ github.token }} ``` -Key properties: -- **Job 1 checks out trusted code only** (`base.sha` or `github.sha`, never the fork's HEAD) -- **Job 2 never spawns for untrusted forks** -- composite actions, checkout-merge, and all secret-bearing steps are unreachable -- Step-level `if:` guards are fragile (earlier steps still execute); job-level `if:` is definitive +Grant only the permissions the publishing operation needs. If deployment credentials are needed, store them in a GitHub environment restricted to the protected branch, with required reviewers where appropriate. These controls must be configured in repository settings; writing an environment name in YAML does not create those restrictions. + +The boundary is GitHub's restrictions on fork runs plus protected access to the publishing branch and credentials. A fork can rewrite its PR workflow, but cannot use that to obtain the protected publishing credentials. A passing PR check or maintainer approval to run fork CI does not by itself make the fork's code trusted for privileged execution. -### Avoid: step-level early exit +### Job gates are supplementary controls -A step like `if: is_trusted == 'false'` then `exit` leaves all preceding steps exposed to injection. Use this only as a temporary measure. +In a trusted workflow, a detection job followed by a job-level `if:` can keep unwanted events away from every step of a privileged job. Prefer that over an early-exit step, which leaves preceding steps exposed. A gate in a fork-editable `pull_request` workflow is not an independent security boundary: the fork can change or remove it. -### Why `pull_request` not `pull_request_target` +If a gate needs a local composite action, load it from the trusted workflow revision before any PR checkout. On `pull_request`, checking out `base.sha` protects only the checked-out detector; it cannot protect the fork-editable workflow definition. On `pull_request_target`, `github.sha` identifies the default-branch workflow commit; on `workflow_dispatch`, it identifies the selected ref's commit, which must be reviewed and trusted. -Use `pull_request` triggers, not `pull_request_target`. With `pull_request`, GitHub runs the workflow from the **base branch** -- fork PRs cannot modify the workflow or composite actions that execute. With `pull_request_target`, the workflow runs with write token and secrets but can be tricked into checking out and executing fork code. If `pull_request_target` is ever needed, it must never check out PR code. +A gate must authorize the operation and exact source revision. For a same-repository policy, compare `github.event.pull_request.head.repo.full_name` with `github.repository`; checking only `github.repository` or its owner does not exclude forks. Verify current repository permissions for human authorizers. A bot actor, prior contribution, label, or maintainer rerun alone does not authorize the fork's current code. Bind any manual approval to an immutable head SHA. Keep fork tests isolated from privileged jobs even after approval. -### Detect-job must check out base code only +### Privileged PR metadata operations -The detect/gate job must use `ref: ${{ github.event.pull_request.base.sha || github.sha }}` to ensure it only runs trusted composite actions from the base branch. If this were changed to check out the PR head, a fork could replace the detect action itself. +`pull_request_target` runs the base repository's **default-branch** workflow and can access write permissions and secrets. Ordinary fork-run approval settings do not gate this event. Reserve it for operations such as labeling or commenting; prefer API metadata over checking out fork content. Treat titles, paths, filenames, symlinks, configuration, and artifacts as untrusted too: a trusted tool can expose credentials or execute code while processing them. Do not execute PR code, directly or indirectly, in a privileged job. + +GitHub.com changed `pull_request_target` to always use the default branch on December 8, 2025. Older articles describing workflow selection from arbitrary PR target branches are historical; this change did **not** make `pull_request` workflows trusted. Check current documentation and the deployed GitHub Enterprise Server version when relying on platform protections. + +Refs: [current security guidance](https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target), [workflow-source change](https://github.blog/changelog/2025-11-07-actions-pull_request_target-and-environment-branch-protections-changes/). ### `workflow_dispatch` trust assumptions -Manual dispatch with a PR number is treated as trusted because only maintainers/admins can trigger `workflow_dispatch`. Ensure repository settings continue to restrict dispatch permissions -- if collaborators with lower access levels gain `actions: write`, they could dispatch against a fork PR and run its code with secrets. +A maintainer dispatching a workflow does not make a supplied PR number, ref, or artifact trusted. Keep privileged manual workflows on protected, reviewed refs and validate inputs before use. Do not dispatch a secret-bearing build against fork code. Restrict dispatch access and enforce environment branch restrictions and approvals in repository settings when using deployment credentials. + +For unprivileged manual PR tests, require the full reviewed head SHA and verify that it matches the PR before fetching. Resolve the base SHA once, then merge those exact commits; do not fetch moving branch tips after verification. Keep the run read-only and secret-free, and set `cache-mode: none` so fork code cannot populate the dispatch branch's cache scope. Prefer approving the ordinary `pull_request` run when manual checkout is unnecessary. + +`permissions:` limits `GITHUB_TOKEN`, not cache access. Current GitHub.com defaults restrict default-branch cache writes for certain untrusted triggers, but `workflow_dispatch` retains write-capable caching by default. Do not infer cache isolation from a read-only repository token or from the absence of an explicit cache-save step. + +Ref: https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#cache-access-for-low-trust-workflow-triggers ## Action pinning @@ -123,7 +156,7 @@ Validate free-form `workflow_dispatch` inputs before use: ## Checkout credential persistence -Always set `persist-credentials: false` on `actions/checkout`. The default (`true`) writes the `GITHUB_TOKEN` into `.git/config`, making it readable by every subsequent step -- including third-party actions and build tools. With `contents: write` permission, a compromised step could push to the repository. +Always set `persist-credentials: false` on `actions/checkout` in this repository. The default (`true`) persists credentials on disk: older versions use `.git/config`; v6 and later use `$RUNNER_TEMP`. Either location can expose credentials to subsequent code in the job. Disabling persistence reduces exposure but does not isolate untrusted code from the job's token or secrets. ```yaml - uses: actions/checkout@ @@ -131,7 +164,7 @@ Always set `persist-credentials: false` on `actions/checkout`. The default (`tru persist-credentials: false ``` -If git push is needed later, configure credentials explicitly for just that step. +If a trusted job must push, configure credentials explicitly for that step. Step placement reduces exposure but does not isolate credentials from other code in the job. A job gate inside a fork-editable workflow cannot establish trust. ## Secrets hygiene in `run:` blocks @@ -149,32 +182,34 @@ Pass secrets through `env:` rather than inline `${{ secrets.* }}` in shell: ## `GITHUB_OUTPUT` injection -When writing to `$GITHUB_OUTPUT` with `echo "key=$value"`, an attacker who controls `value` can inject newlines to set arbitrary output keys. Use the multiline delimiter format when the value could contain newlines (e.g., PR titles, commit messages, comment bodies): +When writing to `$GITHUB_OUTPUT`, an attacker-controlled value containing newlines can inject additional output keys. A fixed multiline delimiter such as `EOF` is also unsafe: the value can contain that delimiter on its own line. For arbitrary text, use a file or serialize to a single-line JSON string, for example with `jq`: ```bash -{ - echo "title<> "$GITHUB_OUTPUT" +title_json=$(jq -cn --arg value "$PR_TITLE" '$value') +printf 'title_json=%s\n' "$title_json" >> "$GITHUB_OUTPUT" ``` -Values with restricted charsets (git branch names, numeric IDs, booleans computed from restricted inputs) are safe with the simple `echo "key=$value"` format. +Decode with `fromJSON` into a step's `env:`, then use a quoted shell variable; never insert decoded text into executable script source. If using a multiline output, generate a fresh delimiter and verify it cannot occur as a delimiter line in the value. Apply the same framing rules to `$GITHUB_ENV`, and never let untrusted data choose environment-variable names. + +Validated numeric IDs, SHAs, and booleans can use the simple `printf 'key=%s\n' "$value"` format. Safe output framing does not make a value trusted by its consumer. ## `workflow_run` artifact poisoning -If adding `workflow_run` triggers: a fork PR can upload artifacts via `pull_request`, and a `workflow_run` job (which has secrets) may then download and process them. Never execute or trust artifact contents from untrusted runs without validation. +If privileged reporting requires a `workflow_run` follow-up, keep all PR execution in the unprivileged producer. Select artifacts using the triggering run's ID and verify the expected repository, workflow, event, and head SHA; a successful run does not make its artifacts trusted. Parse only the required data with strict validation. Extract away from trusted code and do not let artifacts overwrite scripts, supply tool configuration or environment-variable names, or enter executable expressions. Rebuild releases from reviewed commits rather than publishing fork-produced binaries. + +Refs: [Apache's workflow and artifact guidance](https://cwiki.apache.org/confluence/spaces/BUILDS/pages/321719166/GitHub%2BActions%2BSecurity), [zizmor security rules](https://docs.zizmor.sh/audits/). Cross-check historical platform claims against current GitHub documentation. ## Review checklist When reviewing or writing a workflow: -1. Grep for `${{ ` inside `run:` blocks -- each one is a potential injection point -2. For each match, determine if the value is attacker-controlled; if so, move to `env:` -3. Verify fork PRs cannot reach secret-bearing steps (prefer two-job gate) +1. Search for `${{` inside `run:` blocks, including expressions without a following space +2. Move every match to `env:` and use quoted shell variables, regardless of whether the value is trusted +3. Keep fork CI unprivileged; place publishing in a separate trusted workflow and verify the repository settings that protect its branch and credentials 4. Confirm third-party actions are SHA-pinned 5. Validate any free-form `workflow_dispatch` inputs -6. Check that `pull_request_target` is not used (or if it is, that it never checks out PR code) +6. For `pull_request_target` or `workflow_run`, trace all untrusted metadata, files, and artifacts into privileged operations; checking for PR script execution alone is insufficient 7. Verify no `accept-flake-config = true` or `--accept-flake-config` in Nix steps (prevents flake nixConfig escape) -8. Verify `persist-credentials: false` on all `actions/checkout` steps -9. Verify `workflow_dispatch` permissions remain restricted to maintainers/admins +8. Verify `persist-credentials: false` on every checkout; configure any required push credentials explicitly in a trusted job +9. Verify privileged `workflow_dispatch` runs use reviewed refs and cannot execute fork code through inputs or artifacts +10. Check cache access and runner isolation independently of token permissions; use `cache-mode: none` for manual fork tests and release jobs From 9fa69559b3ee659b986b26e07799c6713970b9ed Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 12 Nov 2026 17:21:50 -0600 Subject: [PATCH 2/5] ci: harden fork checks and release workflows Add secret-free TypeScript checks for fork PRs while retaining AWS-backed integration tests for same-repository PRs and manual runs. Require successful master CI for the exact commit before publishing containers. Scope release write permissions to asset uploads, disable Actions caching, and pass image outputs through environment variables. Document CI behavior and correct the yarn loglevel option. --- .github/workflows/container-image.yaml | 36 ++++++++++++++----- .github/workflows/continuous-integration.yaml | 35 ++++++++++++++++-- .../workflows/network-synchronization.yaml | 5 +++ .github/workflows/release-upload.yaml | 11 +++--- CONTRIBUTING.md | 6 ++++ 5 files changed, 78 insertions(+), 15 deletions(-) diff --git a/.github/workflows/container-image.yaml b/.github/workflows/container-image.yaml index f09c91a5c..cec981081 100644 --- a/.github/workflows/container-image.yaml +++ b/.github/workflows/container-image.yaml @@ -11,9 +11,11 @@ on: permissions: contents: read - checks: read + actions: read packages: write +cache-mode: none + jobs: build-and-push: name: "Build and push container images" @@ -52,14 +54,17 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - check_name="clients_TypeScript" + set -euo pipefail deadline=$((SECONDS + 3600)) while true; do - conclusions=$(gh api "repos/$GITHUB_REPOSITORY/commits/$TARGET_SHA/check-runs?check_name=$check_name" \ - --jq '[.check_runs[].conclusion // "pending"] | unique | join(",")') + # Only this workflow's master push/manual runs can authorize publication. + # A successful PR check (including one with the same job name) cannot. + conclusions=$(gh api "repos/$GITHUB_REPOSITORY/actions/workflows/continuous-integration.yaml/runs?head_sha=$TARGET_SHA&branch=master&per_page=100" | + jq -r --arg repo "$GITHUB_REPOSITORY" --arg sha "$TARGET_SHA" \ + '[.workflow_runs[] | select(.head_repository.full_name == $repo and .head_sha == $sha and .head_branch == "master" and (.event == "push" or .event == "workflow_dispatch"))] | sort_by(.id) | last | if . == null then "" else (.conclusion // "pending") end') case "$conclusions" in success) echo "Continuous Integration passed"; break ;; - ""|*pending*) echo "Continuous Integration pending, waiting 30s..." ;; + ""|pending) echo "Continuous Integration pending, waiting 30s..." ;; *) echo "::error::Continuous Integration concluded with: $conclusions"; exit 1 ;; esac if [ "$SECONDS" -ge "$deadline" ]; then @@ -274,6 +279,15 @@ jobs: # the two already-tagged '-amd64' / '-arm64' images by digest. # No new single-arch manifests are created, so nothing is left untagged. - name: Compose multi-arch manifests + env: + OGMIOS_BASE: ${{ steps.tags.outputs.ogmios-base }} + OGMIOS_TAGS: ${{ steps.tags.outputs.ogmios }} + MAINNET_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-mainnet-base }} + MAINNET_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-mainnet }} + PREPROD_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-preprod-base }} + PREPROD_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-preprod }} + PREVIEW_BASE: ${{ steps.tags.outputs.cardano-node-ogmios-preview-base }} + PREVIEW_TAGS: ${{ steps.tags.outputs.cardano-node-ogmios-preview }} run: | set -euo pipefail compose() { @@ -286,7 +300,11 @@ jobs: docker buildx imagetools create -t "$tag" "$repo@$amd64" "$repo@$arm64" done } - compose '${{ steps.tags.outputs.ogmios-base }}' ${{ steps.tags.outputs.ogmios }} - compose '${{ steps.tags.outputs.cardano-node-ogmios-mainnet-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-mainnet }} - compose '${{ steps.tags.outputs.cardano-node-ogmios-preprod-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-preprod }} - compose '${{ steps.tags.outputs.cardano-node-ogmios-preview-base }}' ${{ steps.tags.outputs.cardano-node-ogmios-preview }} + read -r -a tags <<< "$OGMIOS_TAGS" + compose "$OGMIOS_BASE" "${tags[@]}" + read -r -a tags <<< "$MAINNET_TAGS" + compose "$MAINNET_BASE" "${tags[@]}" + read -r -a tags <<< "$PREPROD_TAGS" + compose "$PREPROD_BASE" "${tags[@]}" + read -r -a tags <<< "$PREVIEW_TAGS" + compose "$PREVIEW_BASE" "${tags[@]}" diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 2159f0bfd..61123a691 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -10,15 +10,41 @@ on: branches: - master +permissions: + contents: read + +cache-mode: none + env: CARDANO_NODE_VERSION: "11.1.1" # cardano-playground has no node-11.1.1-config tag yet, so configs stay # pinned to the newest tagged set. CARDANO_CONFIG_TAG: "node-11.0.1-config" NETWORK: preview - S3_DB_PATH: s3://${{ secrets.AWS_S3_BUCKET }}/db-preview jobs: + fork-checks: + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + submodules: true + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: lts/jod + + - name: Build and lint TypeScript client + working-directory: clients/TypeScript + run: | + yarn install --frozen-lockfile --non-interactive --loglevel=error + yarn build + yarn lint + # TODO: switch back to this once Hydra reports check-runs to GitHub # wait-for-hydra: # runs-on: ubuntu-latest @@ -39,6 +65,11 @@ jobs: # done clients_TypeScript: + # Repository writers are trusted to run integration tests with AWS access. + # This routes fork jobs; GitHub's fork secret/token restrictions enforce isolation. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + env: + S3_DB_PATH: s3://${{ secrets.AWS_S3_BUCKET }}/db-preview # needs: # - wait-for-hydra runs-on: ubuntu-24.04 @@ -157,7 +188,7 @@ jobs: - name: Build TypeScript client working-directory: clients/TypeScript run: | - yarn install --frozen-lockfile --non-interactive --logevel=error + yarn install --frozen-lockfile --non-interactive --loglevel=error yarn build yarn lint diff --git a/.github/workflows/network-synchronization.yaml b/.github/workflows/network-synchronization.yaml index 03d0eee66..3218a94a8 100644 --- a/.github/workflows/network-synchronization.yaml +++ b/.github/workflows/network-synchronization.yaml @@ -12,6 +12,11 @@ on: default: 1 type: number +permissions: + contents: read + +cache-mode: none + env: CARDANO_NODE_VERSION: "11.1.1" # cardano-playground has no node-11.1.1-config tag yet, so configs stay diff --git a/.github/workflows/release-upload.yaml b/.github/workflows/release-upload.yaml index 2248cc0a2..211ed6418 100644 --- a/.github/workflows/release-upload.yaml +++ b/.github/workflows/release-upload.yaml @@ -13,11 +13,9 @@ on: - '**' permissions: - contents: write - actions: read + contents: read -env: - GH_TOKEN: ${{ github.token }} +cache-mode: none jobs: # TODO: switch back to this once Hydra reports check-runs to GitHub @@ -119,6 +117,9 @@ jobs: retention-days: 1 upload-assets: + permissions: + contents: write + actions: read needs: - pull name: "Upload Assets" @@ -132,6 +133,8 @@ jobs: run: sha256sum ogmios-*.tar.gz > ogmios-sha256sums.txt - name: Release + env: + GH_TOKEN: ${{ github.token }} uses: input-output-hk/action-gh-release@26994186c0ac3ef5cae75ac16aa32e8153525f77 # v1 with: draft: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b06681598..e56e02862 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,12 @@ Do not hesitate to upvote discussions or comments to show your interest! Pull requests are welcome, but we do recommend to open an issue to bring any idea to discussion first! +##### GitHub Actions checks + +Fork pull requests build and lint the TypeScript client without repository secrets. Same-repository pull requests, pushes to `master`, and manual Continuous Integration runs also run the AWS-backed integration tests. Manual runs retain the branch/ref selector; select a trusted repository revision because those runs have AWS access. Repository writers are trusted with that access. + +The fork job conditions route checks; GitHub's restrictions on fork tokens and secrets provide the security boundary. Keep those restrictions enabled in repository settings. Container publication requires a successful Continuous Integration run for the exact tagged commit from a push to `master` or a manual run on `master`; a PR check cannot satisfy it. Actions caching is disabled in these workflows; S3 database snapshots and the Nix binary cache remain available. + #### 3. Donation Want to give some financial support? Have a look at the [sponsors page](https://github.com/sponsors/KtorZ/) for more details. From 89c56934351138b4960f4acb05411044e5d40cf8 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 12 Nov 2026 17:48:01 -0600 Subject: [PATCH 3/5] ci: limit preview database cache growth Restore only the two newest ledger snapshots and make integration jobs read-only consumers of the S3 cache. Serialize cache writers, upload retained snapshots before pruning obsolete ones, and mirror volatile files. Require a successful restore and clean shutdown before saving. Handle S3 listing errors explicitly and add cache regression tests. --- .github/workflows/continuous-integration.yaml | 40 +-- .../workflows/network-synchronization.yaml | 49 ++-- .../workflows/scripts/preview-db-common.sh | 38 +++ .../workflows/scripts/preview-db-restore.sh | 101 ++++++++ .github/workflows/scripts/preview-db-save.sh | 84 ++++++ .../workflows/scripts/test-preview-cache.py | 242 ++++++++++++++++++ CONTRIBUTING.md | 6 + 7 files changed, 499 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/scripts/preview-db-common.sh create mode 100755 .github/workflows/scripts/preview-db-restore.sh create mode 100755 .github/workflows/scripts/preview-db-save.sh create mode 100644 .github/workflows/scripts/test-preview-cache.py diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 61123a691..27005fd3d 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -151,34 +151,14 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} + # Read-only: Network Synchronization is the only writer of this cache, so + # concurrent pull request runs cannot delete each other's objects. - name: Restore cardano-node DB from S3 - shell: bash {0} env: RUNNER_TEMP: ${{ runner.temp }} run: | - mkdir -p "$RUNNER_TEMP/db-preview" - for attempt in 1 2 3; do - echo "S3 sync attempt $attempt..." - nix shell nixpkgs#s5cmd -c \ - s5cmd --log error sync "$S3_DB_PATH/*" "$RUNNER_TEMP/db-preview/" > /tmp/s3-sync.log 2>&1 & - pid=$! - while kill -0 "$pid" 2>/dev/null; do - du -sh "$RUNNER_TEMP/db-preview" 2>/dev/null - sleep 15 - done - wait "$pid" - rc=$? - echo "S3 sync attempt $attempt finished (exit $rc). Last 30 lines of log:" - tail -30 /tmp/s3-sync.log - if [ "$rc" -eq 0 ]; then - echo "S3 sync succeeded" - break - fi - if [ "$attempt" -eq 3 ]; then - exit 1 - fi - echo "Retrying..." - done + nix shell nixpkgs#s5cmd -c \ + ./.github/workflows/scripts/preview-db-restore.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview" - name: Setup Node.js uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 @@ -213,16 +193,10 @@ jobs: working-directory: clients/TypeScript run: yarn test + # A node under load takes well over the 10s default to flush and write its + # clean-shutdown marker; without it the next start revalidates everything. - name: Stop cardano-node if: always() && steps.sync.outcome != 'skipped' run: | - docker stop cardano-node || true + docker stop -t 300 cardano-node || true docker rm cardano-node || true - - - name: Save cardano-node DB to S3 - if: always() && steps.sync.outcome == 'success' - env: - RUNNER_TEMP: ${{ runner.temp }} - run: | - nix shell nixpkgs#s5cmd -c \ - s5cmd --log error sync "$RUNNER_TEMP/db-preview/" "$S3_DB_PATH/" diff --git a/.github/workflows/network-synchronization.yaml b/.github/workflows/network-synchronization.yaml index 3218a94a8..37f7b3d9a 100644 --- a/.github/workflows/network-synchronization.yaml +++ b/.github/workflows/network-synchronization.yaml @@ -26,8 +26,17 @@ env: jobs: sync_and_cache: + # One writer across scheduled/manual runs and all selected refs. + concurrency: + group: preview-db-cache-writer + cancel-in-progress: false runs-on: ubuntu-24.04 steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - name: Download cardano-node configs env: BASE_URL: https://raw.githubusercontent.com/input-output-hk/cardano-playground/refs/tags/${{ env.CARDANO_CONFIG_TAG }}/docs/environments/preview @@ -54,33 +63,12 @@ jobs: substituters = https://cache.iog.io/ https://cache.nixos.org/ - name: Restore cardano-node DB from S3 - shell: bash {0} + id: restore env: RUNNER_TEMP: ${{ runner.temp }} run: | - mkdir -p "$RUNNER_TEMP/db-preview" - for attempt in 1 2 3; do - echo "S3 sync attempt $attempt..." - nix shell nixpkgs#s5cmd -c \ - s5cmd --log error sync "$S3_DB_PATH/*" "$RUNNER_TEMP/db-preview/" > /tmp/s3-sync.log 2>&1 & - pid=$! - while kill -0 "$pid" 2>/dev/null; do - du -sh "$RUNNER_TEMP/db-preview" 2>/dev/null - sleep 15 - done - wait "$pid" - rc=$? - echo "S3 sync attempt $attempt finished (exit $rc). Last 30 lines of log:" - tail -30 /tmp/s3-sync.log - if [ "$rc" -eq 0 ]; then - echo "S3 sync succeeded" - break - fi - if [ "$attempt" -eq 3 ]; then - exit 1 - fi - echo "Retrying..." - done + nix shell nixpkgs#s5cmd -c \ + ./.github/workflows/scripts/preview-db-restore.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview" - name: Sync node id: sync @@ -113,16 +101,21 @@ jobs: sleep 30 done + # A node under load takes well over the 10s default to flush and write its + # clean-shutdown marker; without it the next start revalidates everything. - name: Stop cardano-node + id: stop if: always() && steps.sync.outcome != 'skipped' run: | - docker stop cardano-node || true - docker rm cardano-node || true + docker stop -t 300 cardano-node + docker rm cardano-node + # Save partial sync progress only after a successful restore and shutdown. + # The save script also requires the node's clean-shutdown marker. - name: Save cardano-node DB to S3 - if: always() && steps.sync.outcome != 'skipped' + if: always() && steps.restore.outcome == 'success' && steps.sync.outcome != 'skipped' && steps.stop.outcome == 'success' env: RUNNER_TEMP: ${{ runner.temp }} run: | nix shell nixpkgs#s5cmd -c \ - s5cmd --log error sync "$RUNNER_TEMP/db-preview/" "$S3_DB_PATH/" + ./.github/workflows/scripts/preview-db-save.sh "$S3_DB_PATH" "$RUNNER_TEMP/db-preview" diff --git a/.github/workflows/scripts/preview-db-common.sh b/.github/workflows/scripts/preview-db-common.sh new file mode 100644 index 000000000..cd9fb6691 --- /dev/null +++ b/.github/workflows/scripts/preview-db-common.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Helpers shared by the preview database restore and save scripts. +# Requires s5cmd and jq. An empty prefix is different from a failed listing. + +cache_list() { + local prefix="$1" scratch rc=0 + scratch=$(mktemp -d) + s5cmd --json ls "$prefix" > "$scratch/objects" 2> "$scratch/errors" || rc=$? + if [ "$rc" -eq 0 ]; then + jq -s '.' "$scratch/objects" || rc=$? + elif jq -e -s 'length > 0 and all(.[]; .operation == "ls" and .error == "no object found")' \ + "$scratch/errors" >/dev/null 2>&1 && [ ! -s "$scratch/objects" ]; then + printf '[]\n' + rc=0 + else + cat "$scratch/errors" >&2 + fi + rm -rf -- "$scratch" + return "$rc" +} + +snapshot_names() { + jq -r '.[] | select(.type == "directory") | .key | rtrimstr("/") | split("/") | last + | select(test("^[0-9]+(_[A-Za-z0-9_-]+)?$"))' +} + +check_cache_args() { + if [[ ! "$S3_DB_PATH" =~ ^s3://[^/]+/.+ ]] || [[ "$S3_DB_PATH" == *[\*\?\[]* ]] \ + || [ -z "$DB_DIR" ] || [ "$DB_DIR" = / ]; then + echo "Usage: $0 s3://BUCKET/PREFIX DB_DIR" >&2 + exit 1 + fi +} diff --git a/.github/workflows/scripts/preview-db-restore.sh b/.github/workflows/scripts/preview-db-restore.sh new file mode 100755 index 000000000..1e3e9f9ce --- /dev/null +++ b/.github/workflows/scripts/preview-db-restore.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Restore immutable data, the two newest ledger snapshots, and volatile data. +# Ledger downloads stay bounded even when the legacy cache contains old snapshots. +# The save script mirrors volatile data so it stops accumulating after the first save. +# Usage: preview-db-restore.sh s3://BUCKET/PREFIX DB_DIR + +set -euo pipefail +# shellcheck source=.github/workflows/scripts/preview-db-common.sh +source "$(dirname "${BASH_SOURCE[0]}")/preview-db-common.sh" +S3_DB_PATH="${1:-}" +DB_DIR="${2:-}" +check_cache_args +S3_DB_PATH="${S3_DB_PATH%/}" +DB_DIR="${DB_DIR%/}" +BASELINE_FILE="$DB_DIR.restored-chunks" +mkdir -p "$DB_DIR" +rm -f -- "$BASELINE_FILE" +log=$(mktemp) +pid= +cleanup() { + if [ -n "$pid" ]; then kill "$pid" 2>/dev/null || true; wait "$pid" 2>/dev/null || true; fi + rm -f -- "$log" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +restore() { + local objects snapshots snapshot marker markers + objects=$(cache_list "$S3_DB_PATH/") + if [ "$(jq 'length' <<< "$objects")" -eq 0 ]; then + echo "Cache is empty; the node will sync from genesis." + return 0 + fi + # An inaccessible or partial cache must not be mistaken for a cold cache. + if ! jq -e --arg key "$S3_DB_PATH/immutable/" 'any(.[]; .key == $key)' <<< "$objects" >/dev/null; then + echo "::error::Cache contains data but no immutable directory." >&2 + return 1 + fi + echo "Restoring immutable/..." + s5cmd --log error sync "$S3_DB_PATH/immutable/*" "$DB_DIR/immutable/" + + objects=$(cache_list "$S3_DB_PATH/ledger/") + snapshots=$(snapshot_names <<< "$objects" | sort -t_ -k1,1n | tail -2) + if [ -z "$snapshots" ]; then + echo "Cache holds no ledger snapshot; the node will replay the immutable chain." + else + while IFS= read -r snapshot; do + echo "Restoring ledger/$snapshot..." + s5cmd --log error sync "$S3_DB_PATH/ledger/$snapshot/*" "$DB_DIR/ledger/$snapshot/" + done <<< "$snapshots" + fi + + objects=$(cache_list "$S3_DB_PATH/volatile/") + if [ "$(jq 'length' <<< "$objects")" -gt 0 ]; then + echo "Restoring volatile/..." + s5cmd --log error sync "$S3_DB_PATH/volatile/*" "$DB_DIR/volatile/" + fi + + # Only missing objects are optional; permission and network errors must fail. + objects=$(cache_list "$S3_DB_PATH/") + markers=$(jq -r '.[] | select(.type == "file") | .key | split("/") | last' <<< "$objects") + for marker in protocolMagicId clean; do + rm -f -- "$DB_DIR/$marker" + if grep -Fxq "$marker" <<< "$markers"; then + s5cmd --log error cp "$S3_DB_PATH/$marker" "$DB_DIR/$marker" + fi + done +} + +for attempt in 1 2 3; do + echo "Restore attempt $attempt..." + restore > "$log" 2>&1 & + pid=$! + while kill -0 "$pid" 2>/dev/null; do + # s5cmd renames temporary files while du walks the directory. Progress is + # best-effort; only the restore process's exit status determines success. + du -sh "$DB_DIR" 2>/dev/null || true + sleep 15 + done + rc=0 + wait "$pid" || rc=$? + pid= + echo "Restore attempt $attempt finished (exit $rc). Last 30 lines of log:" + tail -30 "$log" + if [ "$rc" -eq 0 ]; then break; fi + if [ "$attempt" -eq 3 ]; then exit 1; fi + echo "Retrying..." +done + +chunks=0 +if [ -d "$DB_DIR/immutable" ]; then + chunks=$(find "$DB_DIR/immutable" -type f -name '*.chunk' | wc -l) +fi +printf '%s\n' "$chunks" > "$BASELINE_FILE" +echo "Restored $chunks immutable chunks, $(du -sh "$DB_DIR" | cut -f1) on disk." diff --git a/.github/workflows/scripts/preview-db-save.sh b/.github/workflows/scripts/preview-db-save.sh new file mode 100755 index 000000000..b99bfb615 --- /dev/null +++ b/.github/workflows/scripts/preview-db-save.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Save a stopped node's database and prune obsolete cached ledger snapshots. +# The workflow serializes writers across refs; CI jobs only restore this cache. +# Usage: preview-db-save.sh s3://BUCKET/PREFIX DB_DIR + +set -euo pipefail +# shellcheck source=.github/workflows/scripts/preview-db-common.sh +source "$(dirname "${BASH_SOURCE[0]}")/preview-db-common.sh" +S3_DB_PATH="${1:-}" +DB_DIR="${2:-}" +check_cache_args +S3_DB_PATH="${S3_DB_PATH%/}" +DB_DIR="${DB_DIR%/}" +BASELINE_FILE="$DB_DIR.restored-chunks" + +# Preserve partial synchronization progress only after an orderly shutdown. +# Neither a failed restore nor a truncated/unclean database may replace the cache. +for required in "$BASELINE_FILE" "$DB_DIR/protocolMagicId" "$DB_DIR/clean"; do + if [ ! -f "$required" ]; then + echo "::error::Missing $required; refusing to overwrite the cache." >&2 + exit 1 + fi +done +baseline=$(cat "$BASELINE_FILE") +[[ "$baseline" =~ ^[[:space:]]*[0-9]+[[:space:]]*$ ]] || { echo "Invalid restore baseline" >&2; exit 1; } +chunks=$(find "$DB_DIR/immutable" -type f -name '*.chunk' | wc -l) +if [ "$chunks" -eq 0 ] || [ "$chunks" -lt "$baseline" ]; then + echo "::error::Immutable chunk count is $chunks ($baseline restored); refusing to overwrite the cache." >&2 + exit 1 +fi + +local_names= +if [ -d "$DB_DIR/ledger" ]; then + local_names=$(find "$DB_DIR/ledger" -mindepth 1 -maxdepth 1 -type d -printf '%f\n') +fi +local_snapshots=() +while IFS= read -r name; do + if [[ "$name" =~ ^[0-9]+(_[A-Za-z0-9_-]+)?$ ]]; then local_snapshots+=("$name"); fi +done <<< "$local_names" +objects=$(cache_list "$S3_DB_PATH/ledger/") +cached_names=$(snapshot_names <<< "$objects") +objects=$(cache_list "$S3_DB_PATH/") + +# Clear the old completion marker before any upload. Restore may run concurrently; +# this shared prefix is not an atomic snapshot, so never promise it is one. +if jq -e --arg key "$S3_DB_PATH/clean" 'any(.[]; .key == $key)' <<< "$objects" >/dev/null; then + s5cmd --log error rm "$S3_DB_PATH/clean" +fi + +echo "Saving $(du -sh "$DB_DIR" | cut -f1) to $S3_DB_PATH..." +# Immutable data is append-only; never delete cached chunks during publication. +s5cmd --log error sync "$DB_DIR/immutable/" "$S3_DB_PATH/immutable/" +for snapshot in "${local_snapshots[@]}"; do + echo "Uploading ledger/$snapshot..." + s5cmd --log error sync "$DB_DIR/ledger/$snapshot/" "$S3_DB_PATH/ledger/$snapshot/" +done +if [ -d "$DB_DIR/volatile" ]; then + s5cmd --log error sync --delete "$DB_DIR/volatile/" "$S3_DB_PATH/volatile/" +fi +s5cmd --log error cp "$DB_DIR/protocolMagicId" "$S3_DB_PATH/protocolMagicId" + +# Upload every retained snapshot before deleting any old one. An empty local +# ledger must not wipe the cached fallback snapshots. +if [ "${#local_snapshots[@]}" -gt 0 ]; then + while IFS= read -r snapshot; do + [ -n "$snapshot" ] || continue + keep=false + for local_snapshot in "${local_snapshots[@]}"; do + if [ "$local_snapshot" = "$snapshot" ]; then keep=true; break; fi + done + if [ "$keep" = false ]; then + echo "Pruning stale ledger/$snapshot..." + s5cmd --log error rm "$S3_DB_PATH/ledger/$snapshot/*" + fi + done <<< "$cached_names" +fi +# Publish the clean marker only after all data transfers and pruning succeed. +s5cmd --log error cp "$DB_DIR/clean" "$S3_DB_PATH/clean" +echo "Save succeeded" diff --git a/.github/workflows/scripts/test-preview-cache.py b/.github/workflows/scripts/test-preview-cache.py new file mode 100644 index 000000000..846973298 --- /dev/null +++ b/.github/workflows/scripts/test-preview-cache.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Exercise preview cache scripts with a filesystem-backed S3 fixture. + +Run: python3 .github/workflows/scripts/test-preview-cache.py +Requires bash and jq; never contacts AWS or starts a container. +""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parent +S3 = "s3://fixture/db-preview" + +S5CMD = r''' +import fnmatch, json, os, shutil, sys +from pathlib import Path +args = sys.argv[1:] +with open(os.environ['CALL_LOG'], 'a') as log: + log.write(json.dumps(args) + '\n') +while args[0].startswith('--'): + flag = args.pop(0) + if flag == '--log': args.pop(0) +cmd = args.pop(0) +if os.environ.get('FAIL_OPERATION') == cmd and os.environ.get('FAIL_CONTAINS', '') in args[0]: + print(json.dumps({'operation':cmd,'error':'AccessDenied'}), file=sys.stderr) + sys.exit(1) +root = Path(os.environ['BUCKET']) +def local(value): + return root / value.removeprefix('s3://fixture/') if value.startswith('s3://') else Path(value) +def copy(src, dst): + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) +if cmd == 'ls': + prefix = args[0].rstrip('/') + '/' + path = local(prefix) + children = sorted(path.iterdir()) if path.is_dir() else [] + if not children: + print(json.dumps({'operation':'ls','error':'no object found'}), file=sys.stderr) + sys.exit(1) + for child in children: + print(json.dumps({'key':prefix + child.name + ('/' if child.is_dir() else ''), + 'type':'directory' if child.is_dir() else 'file'})) +elif cmd == 'sync': + delete = args[0] == '--delete' + if delete: args.pop(0) + src, dst = local(args[0].removesuffix('*')), local(args[1]) + dst.mkdir(parents=True, exist_ok=True) + if not src.is_dir(): sys.exit(1) + if delete: + for path in dst.rglob('*'): + if path.is_file() and not (src / path.relative_to(dst)).exists(): path.unlink() + for path in src.rglob('*'): + if path.is_file(): copy(path, dst / path.relative_to(src)) +elif cmd == 'cp': + copy(local(args[0]), local(args[1])) +elif cmd == 'rm': + value = args[0] + if value.endswith('/*'): + path = local(value[:-2]) + shutil.rmtree(path) + else: local(value).unlink() +else: raise AssertionError(cmd) +''' + +class CacheTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bucket = self.root / 'bucket' + self.cache = self.bucket / 'db-preview' + self.db = self.root / 'db' + self.bin = self.root / 'bin' + self.bin.mkdir() + self.log = self.root / 'calls' + self.env = os.environ | { + 'PATH': str(self.bin) + os.pathsep + os.environ['PATH'], + 'BUCKET': str(self.bucket), 'CALL_LOG': str(self.log), + } + for name, body in [('s5cmd', S5CMD), ('sleep', 'import time; time.sleep(0.01)')]: + path = self.bin / name + path.write_text(f'#!{sys.executable}\n' + body) + path.chmod(0o755) + + def put(self, root, name, content='fixture'): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + def seed(self): + for name in ['immutable/00000.chunk', 'immutable/00001.chunk', 'volatile/blocks-1.dat', + 'protocolMagicId', 'clean', 'lock']: + self.put(self.cache, name) + for slot in ['77', '900', '4000', '12000', '2345678_db-analyser']: + self.put(self.cache, f'ledger/{slot}/state') + + def local_db(self): + for name in ['immutable/00000.chunk', 'immutable/00001.chunk', 'volatile/blocks-9.dat', + 'ledger/50000/state', 'ledger/50100/state', 'protocolMagicId', 'clean']: + self.put(self.db, name) + self.db.with_suffix('.restored-chunks').write_text('2\n') + + def run_script(self, name, ok=True, extra=None, args=None): + result = subprocess.run(['bash', str(SCRIPTS / name), *(args or [S3, str(self.db)])], + env=self.env | (extra or {}), capture_output=True, text=True, timeout=15) + if ok: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + else: + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + return result + + def calls(self): + return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] + + def test_restore_selects_two_newest_and_keeps_volatile_and_markers(self): + self.seed() + self.run_script('preview-db-restore.sh') + self.assertEqual(sorted(p.name for p in (self.db / 'ledger').iterdir()), ['12000', '2345678_db-analyser']) + for name in ['volatile/blocks-1.dat', 'clean', 'protocolMagicId']: + self.assertTrue((self.db / name).exists()) + self.assertFalse((self.db / 'lock').exists()) + self.assertEqual(self.db.with_suffix('.restored-chunks').read_text().strip(), '2') + + def test_cold_cache(self): + self.run_script('preview-db-restore.sh') + self.assertEqual(self.db.with_suffix('.restored-chunks').read_text().strip(), '0') + + def test_progress_error_does_not_override_restore_result(self): + self.seed() + real_du = shutil.which('du') + marker = self.root / 'du-failed' + stub = self.bin / 'du' + stub.write_text(f'''#!{sys.executable} +import os, sys +from pathlib import Path +marker = Path({str(marker)!r}) +if not marker.exists(): + marker.touch() + print("du: cannot access temporary download file: No such file or directory", file=sys.stderr) + sys.exit(1) +os.execv({real_du!r}, [{real_du!r}, *sys.argv[1:]]) +''') + stub.chmod(0o755) + for fail_download in [False, True]: + with self.subTest(fail_download=fail_download): + marker.unlink(missing_ok=True) + extra = {'FAIL_OPERATION':'sync'} if fail_download else {} + result = self.run_script('preview-db-restore.sh', ok=not fail_download, extra=extra) + self.assertTrue(marker.exists(), 'Progress probe must exercise the du failure') + if fail_download: + self.assertIn('Restore attempt 3 finished (exit 1)', result.stdout) + self.assertIn('AccessDenied', result.stdout) + self.assertFalse(self.db.with_suffix('.restored-chunks').exists()) + else: + self.assertTrue((self.db / 'ledger/12000/state').exists()) + self.assertEqual(self.db.with_suffix('.restored-chunks').read_text().strip(), '2') + + def test_listing_failure_is_not_a_cold_cache(self): + result = self.run_script('preview-db-restore.sh', ok=False, extra={'FAIL_OPERATION':'ls'}) + self.assertIn('AccessDenied', result.stdout) + self.assertFalse(self.db.with_suffix('.restored-chunks').exists()) + + def test_ledger_listing_failure_is_not_an_empty_ledger(self): + self.seed() + self.run_script('preview-db-restore.sh', ok=False, + extra={'FAIL_OPERATION':'ls', 'FAIL_CONTAINS':'/ledger/'}) + self.assertFalse(self.db.with_suffix('.restored-chunks').exists()) + + def test_partial_restore_does_not_enable_save(self): + self.seed() + self.run_script('preview-db-restore.sh', ok=False, + extra={'FAIL_OPERATION':'sync', 'FAIL_CONTAINS':'/ledger/12000/'}) + self.assertFalse(self.db.with_suffix('.restored-chunks').exists()) + + def test_missing_clean_marker_is_optional_on_restore(self): + self.seed() + (self.cache / 'clean').unlink() + self.run_script('preview-db-restore.sh') + self.assertFalse((self.db / 'clean').exists()) + + def test_marker_transfer_error_is_not_ignored(self): + self.seed() + self.run_script('preview-db-restore.sh', ok=False, + extra={'FAIL_OPERATION':'cp', 'FAIL_CONTAINS':'protocolMagicId'}) + self.assertFalse(self.db.with_suffix('.restored-chunks').exists()) + + def test_save_uploads_before_pruning_and_publishes_clean_last(self): + self.seed() + self.local_db() + self.run_script('preview-db-save.sh') + self.assertEqual(sorted(p.name for p in (self.cache / 'ledger').iterdir()), ['50000','50100']) + self.assertEqual(sorted(p.name for p in (self.cache / 'volatile').iterdir()), ['blocks-9.dat']) + calls=self.calls() + uploads=[i for i,c in enumerate(calls) if c[-2].startswith(str(self.db / 'ledger'))] + prunes=[i for i,c in enumerate(calls) if 'rm' in c and '/ledger/' in c[-1]] + self.assertLess(max(uploads),min(prunes)) + self.assertEqual(calls[-1][-3:],['cp',str(self.db / 'clean'),S3+'/clean']) + self.assertFalse(any('rm' in c and '/immutable' in c[-1] for c in calls)) + + def test_failed_upload_does_not_prune_or_publish_clean(self): + self.seed() + self.local_db() + self.run_script('preview-db-save.sh',ok=False, + extra={'FAIL_OPERATION':'sync','FAIL_CONTAINS':'/ledger/50100/'}) + self.assertTrue((self.cache / 'ledger/900/state').exists()) + self.assertFalse((self.cache / 'clean').exists()) + + def test_save_requires_successful_restore_and_clean_shutdown(self): + for missing in ['clean','protocolMagicId','baseline']: + with self.subTest(missing=missing): + self.local_db() + path = self.db.with_suffix('.restored-chunks') if missing=='baseline' else self.db / missing + path.unlink() + self.run_script('preview-db-save.sh',ok=False) + self.assertEqual(self.calls(),[]) + + def test_save_rejects_truncated_database(self): + self.local_db() + (self.db / 'immutable/00001.chunk').unlink() + self.run_script('preview-db-save.sh',ok=False) + self.assertEqual(self.calls(),[]) + + def test_save_listing_failure_precedes_all_writes(self): + self.local_db() + self.run_script('preview-db-save.sh',ok=False,extra={'FAIL_OPERATION':'ls'}) + self.assertTrue(all('ls' in c for c in self.calls())) + + def test_empty_local_ledger_preserves_cached_snapshots(self): + self.seed() + self.local_db() + shutil.rmtree(self.db / 'ledger') + self.run_script('preview-db-save.sh') + self.assertTrue((self.cache / 'ledger/900/state').exists()) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e56e02862..1bf5cbd87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,12 @@ Fork pull requests build and lint the TypeScript client without repository secre The fork job conditions route checks; GitHub's restrictions on fork tokens and secrets provide the security boundary. Keep those restrictions enabled in repository settings. Container publication requires a successful Continuous Integration run for the exact tagged commit from a push to `master` or a manual run on `master`; a PR check cannot satisfy it. Actions caching is disabled in these workflows; S3 database snapshots and the Nix binary cache remain available. +Integration tests restore the preview database from S3 but do not upload it. Network Synchronization is the sole cache writer, with scheduled and manual runs serialized across refs. Restore fetches immutable data, the two newest ledger snapshots, and volatile data. After a successful restore and clean node shutdown, the writer uploads retained snapshots before pruning obsolete snapshots and mirrors volatile files so they stop accumulating. Partial synchronization progress can still be saved; an unclean shutdown cannot replace the cache. + +The first successful writer run prunes the legacy ledger accumulation. Legacy volatile data is still restored in full until that run mirrors the node's current files. The shared S3 prefix is not an atomic snapshot; readers can overlap an upload, and restores retry transfer failures. Avoid running older workflows that still upload the entire database alongside the updated writer. If S3 versioning is enabled, deleting current objects does not reclaim their noncurrent versions; bucket lifecycle rules control that storage. + +Run the local cache regression tests with `python3 .github/workflows/scripts/test-preview-cache.py` (requires Bash and jq). They use fixture data and fake S3 commands, without AWS access or a running container. + #### 3. Donation Want to give some financial support? Have a look at the [sponsors page](https://github.com/sponsors/KtorZ/) for more details. From ab951778cf63ffd451572b30cebf5d514bbc3764 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 12 Nov 2026 17:49:53 -0600 Subject: [PATCH 4/5] ci: fix preview node sync queries and bound retries Pass the node socket explicitly to cardano-cli and surface query errors instead of silently retrying until the job times out. Detect stopped containers, bound startup and query retries, and reserve time for shutdown and cache upload. Validate the sync target while preserving an explicit zero. Add sync-loop regression tests and update testing documentation. --- .../workflows/network-synchronization.yaml | 28 ++------ .../workflows/scripts/preview-node-sync.sh | 66 +++++++++++++++++++ .../workflows/scripts/test-preview-cache.py | 59 ++++++++++++++++- CONTRIBUTING.md | 2 +- 4 files changed, 128 insertions(+), 27 deletions(-) create mode 100755 .github/workflows/scripts/preview-node-sync.sh diff --git a/.github/workflows/network-synchronization.yaml b/.github/workflows/network-synchronization.yaml index 37f7b3d9a..273939b86 100644 --- a/.github/workflows/network-synchronization.yaml +++ b/.github/workflows/network-synchronization.yaml @@ -72,34 +72,14 @@ jobs: - name: Sync node id: sync + timeout-minutes: 300 env: CONFDIR: ${{ github.workspace }}/config/preview RUNNER_TEMP: ${{ runner.temp }} - SYNC_LEVEL: ${{ inputs.synchronization-level || 1 }} + # Stringify without JSON quotes; the nonempty string '0' preserves zero. + SYNC_LEVEL: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.synchronization-level) || '1' }} run: | - docker run -d --name cardano-node \ - -v "$RUNNER_TEMP/db-preview:/db" \ - -v "$RUNNER_TEMP/ipc:/ipc" \ - -v "$CONFDIR:/config" \ - "ghcr.io/intersectmbo/cardano-node:$CARDANO_NODE_VERSION" run --config /config/config.json --database-path /db --socket-path /ipc/node.socket --topology /config/topology.json - - echo "Waiting for node socket..." - while [ ! -S "$RUNNER_TEMP/ipc/node.socket" ]; do - sleep 5 - done - - echo "Waiting for sync to reach $SYNC_LEVEL..." - while true; do - tip=$(docker exec cardano-node cardano-cli query tip --testnet-magic 2 2>/dev/null) || { sleep 5; continue; } - sync=$(echo "$tip" | jq -r '.syncProgress // "0"' | tr -d '%') - pct=$(echo "scale=4; $sync / 100" | bc) - echo "Sync progress: ${sync}%" - if [ "$(echo "$pct >= $SYNC_LEVEL" | bc)" -eq 1 ]; then - echo "Reached target sync level" - break - fi - sleep 30 - done + ./.github/workflows/scripts/preview-node-sync.sh "$SYNC_LEVEL" # A node under load takes well over the 10s default to flush and write its # clean-shutdown marker; without it the next start revalidates everything. diff --git a/.github/workflows/scripts/preview-node-sync.sh b/.github/workflows/scripts/preview-node-sync.sh new file mode 100755 index 000000000..20e433569 --- /dev/null +++ b/.github/workflows/scripts/preview-node-sync.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Run the preview node and query its progress until the requested fraction is met. +# CONFDIR, RUNNER_TEMP and CARDANO_NODE_VERSION are provided by the workflow. +set -euo pipefail +SYNC_LEVEL="${1:-1}" +if ! jq -en --arg level "$SYNC_LEVEL" '$level | tonumber | . >= 0 and . <= 1' >/dev/null; then + echo "::error::Synchronization level must be between 0 and 1." >&2 + exit 1 +fi + +docker run -d --name cardano-node \ + -v "$RUNNER_TEMP/db-preview:/db" \ + -v "$RUNNER_TEMP/ipc:/ipc" \ + -v "$CONFDIR:/config" \ + "ghcr.io/intersectmbo/cardano-node:$CARDANO_NODE_VERSION" run --config /config/config.json --database-path /db --socket-path /ipc/node.socket --topology /config/topology.json + +check_running() { + if [ "$(docker inspect --format '{{.State.Running}}' cardano-node)" != true ]; then + echo "::error::cardano-node stopped before reaching the sync target." >&2 + docker logs --tail 100 cardano-node >&2 || true + exit 1 + fi +} + +echo "Waiting for node socket..." +deadline=$((SECONDS + 900)) +while [ ! -S "$RUNNER_TEMP/ipc/node.socket" ]; do + check_running + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::Timed out waiting for node socket." >&2 + docker logs --tail 100 cardano-node >&2 || true + exit 1 + fi + sleep 5 +done + +echo "Waiting for sync to reach $SYNC_LEVEL..." +query_failures=0 +while true; do + check_running + # docker exec bypasses the image's CLI wrapper, so supply the socket explicitly. + if ! tip=$(docker exec cardano-node cardano-cli query tip --testnet-magic 2 --socket-path /ipc/node.socket); then + query_failures=$((query_failures + 1)) + if [ "$query_failures" -ge 12 ]; then + echo "::error::Twelve consecutive tip queries failed." >&2 + docker logs --tail 100 cardano-node >&2 || true + exit 1 + fi + sleep 5 + continue + fi + query_failures=0 + progress=$(jq -er '.syncProgress | tonumber | select(. >= 0 and . <= 100)' <<< "$tip") + echo "Sync progress: ${progress}%" + if jq -en --argjson progress "$progress" --argjson level "$SYNC_LEVEL" \ + '$progress >= ($level * 100)' >/dev/null; then + echo "Reached target sync level" + break + fi + sleep 30 +done diff --git a/.github/workflows/scripts/test-preview-cache.py b/.github/workflows/scripts/test-preview-cache.py index 846973298..112b09095 100644 --- a/.github/workflows/scripts/test-preview-cache.py +++ b/.github/workflows/scripts/test-preview-cache.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Exercise preview cache scripts with a filesystem-backed S3 fixture. +"""Exercise preview cache scripts with a filesystem-backed S3 fixture and fake Docker. Run: python3 .github/workflows/scripts/test-preview-cache.py Requires bash and jq; never contacts AWS or starts a container. @@ -8,6 +8,7 @@ import os from pathlib import Path import shutil +import socket import subprocess import sys import tempfile @@ -67,6 +68,28 @@ def copy(src, dst): else: raise AssertionError(cmd) ''' +DOCKER = r''' +import json, os, sys +from pathlib import Path +args=sys.argv[1:] +with open(os.environ['DOCKER_LOG'],'a') as log: log.write(json.dumps(args)+'\n') +if args[0]=='run': print('fixture-container') +elif args[0]=='inspect': print('false' if os.environ.get('NODE_STOPPED') else 'true') +elif args[0]=='logs': print('fixture node diagnostics',file=sys.stderr) +elif args[0]=='exec': + if args[-2:] != ['--socket-path','/ipc/node.socket']: + print('Missing: --socket-path',file=sys.stderr);sys.exit(1) + count=Path(os.environ['QUERY_COUNT']) + n=int(count.read_text()) if count.exists() else 0 + count.write_text(str(n+1)) + if n < int(os.environ.get('QUERY_FAILURES','0')): + print('fixture query failure',file=sys.stderr);sys.exit(1) + progress=json.loads(os.environ.get('PROGRESS','["100.00"]')) + print(json.dumps({'syncProgress':progress[min(n,len(progress)-1)]})) +else: raise AssertionError(args) +''' + + class CacheTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -81,8 +104,11 @@ def setUp(self): self.env = os.environ | { 'PATH': str(self.bin) + os.pathsep + os.environ['PATH'], 'BUCKET': str(self.bucket), 'CALL_LOG': str(self.log), + 'RUNNER_TEMP': str(self.root), 'CONFDIR': str(self.root / 'config'), + 'CARDANO_NODE_VERSION': '11.1.1', 'DOCKER_LOG': str(self.root / 'docker-log'), + 'QUERY_COUNT': str(self.root / 'query-count'), } - for name, body in [('s5cmd', S5CMD), ('sleep', 'import time; time.sleep(0.01)')]: + for name, body in [('s5cmd', S5CMD), ('docker', DOCKER), ('sleep', 'import time; time.sleep(0.01)')]: path = self.bin / name path.write_text(f'#!{sys.executable}\n' + body) path.chmod(0o755) @@ -237,6 +263,35 @@ def test_empty_local_ledger_preserves_cached_snapshots(self): self.run_script('preview-db-save.sh') self.assertTrue((self.cache / 'ledger/900/state').exists()) + def node_socket(self): + (self.root / 'ipc').mkdir() + sock=socket.socket(socket.AF_UNIX) + sock.bind(str(self.root / 'ipc/node.socket')) + self.addCleanup(sock.close) + + def test_sync_passes_explicit_socket_and_reaches_target(self): + self.node_socket() + result=self.run_script('preview-node-sync.sh',args=['1'],extra={'PROGRESS':'["97.00","100.00"]'}) + self.assertIn('Reached target sync level',result.stdout) + self.assertEqual((self.root/'query-count').read_text(),'2') + + def test_query_failure_is_visible_and_bounded(self): + self.node_socket() + result=self.run_script('preview-node-sync.sh',ok=False,args=['1'],extra={'QUERY_FAILURES':'100'}) + self.assertIn('fixture query failure',result.stderr) + self.assertIn('Twelve consecutive',result.stderr) + self.assertEqual((self.root/'query-count').read_text(),'12') + + def test_sync_detects_stopped_node(self): + result=self.run_script('preview-node-sync.sh',ok=False,args=['1'],extra={'NODE_STOPPED':'1'}) + self.assertIn('stopped before reaching',result.stderr) + + def test_sync_validates_target_and_accepts_zero(self): + for target in ['-1','1.1','garbage']: + self.run_script('preview-node-sync.sh',ok=False,args=[target]) + self.node_socket() + self.run_script('preview-node-sync.sh',args=['0'],extra={'PROGRESS':'["0"]'}) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1bf5cbd87..ca478205b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,7 @@ Integration tests restore the preview database from S3 but do not upload it. Net The first successful writer run prunes the legacy ledger accumulation. Legacy volatile data is still restored in full until that run mirrors the node's current files. The shared S3 prefix is not an atomic snapshot; readers can overlap an upload, and restores retry transfer failures. Avoid running older workflows that still upload the entire database alongside the updated writer. If S3 versioning is enabled, deleting current objects does not reclaim their noncurrent versions; bucket lifecycle rules control that storage. -Run the local cache regression tests with `python3 .github/workflows/scripts/test-preview-cache.py` (requires Bash and jq). They use fixture data and fake S3 commands, without AWS access or a running container. +Run the local cache and sync-loop regression tests with `python3 .github/workflows/scripts/test-preview-cache.py` (requires Bash and jq). They use fixture data and fake S3/Docker commands, without AWS access or a running container. #### 3. Donation From bc3c9d02e4fa148568a97ff0daf73449a17b96cb Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 12 Nov 2026 19:11:08 -0600 Subject: [PATCH 5/5] ci: retry when node sync progress is unavailable Wait for valid progress instead of exiting when a successful tip query omits it. Bound retries by the synchronization deadline and preserve zero-target behavior. Add regression coverage for missing and invalid progress, recovery, and timeout handling. --- .../workflows/scripts/preview-node-sync.sh | 26 ++++++++++++++- .../workflows/scripts/test-preview-cache.py | 33 +++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/.github/workflows/scripts/preview-node-sync.sh b/.github/workflows/scripts/preview-node-sync.sh index 20e433569..421c39cc2 100755 --- a/.github/workflows/scripts/preview-node-sync.sh +++ b/.github/workflows/scripts/preview-node-sync.sh @@ -6,6 +6,7 @@ # Run the preview node and query its progress until the requested fraction is met. # CONFDIR, RUNNER_TEMP and CARDANO_NODE_VERSION are provided by the workflow. +# Leave room for cleanup before the workflow step's 300-minute timeout. set -euo pipefail SYNC_LEVEL="${1:-1}" if ! jq -en --arg level "$SYNC_LEVEL" '$level | tonumber | . >= 0 and . <= 1' >/dev/null; then @@ -13,12 +14,27 @@ if ! jq -en --arg level "$SYNC_LEVEL" '$level | tonumber | . >= 0 and . <= 1' >/ exit 1 fi +SYNC_TIMEOUT_SECONDS="${SYNC_TIMEOUT_SECONDS:-17400}" +if [[ ! "$SYNC_TIMEOUT_SECONDS" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::SYNC_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 +fi +sync_deadline=$((SECONDS + SYNC_TIMEOUT_SECONDS)) + docker run -d --name cardano-node \ -v "$RUNNER_TEMP/db-preview:/db" \ -v "$RUNNER_TEMP/ipc:/ipc" \ -v "$CONFDIR:/config" \ "ghcr.io/intersectmbo/cardano-node:$CARDANO_NODE_VERSION" run --config /config/config.json --database-path /db --socket-path /ipc/node.socket --topology /config/topology.json +check_deadline() { + if [ "$SECONDS" -ge "$sync_deadline" ]; then + echo "::error::Timed out waiting for node to sync." >&2 + docker logs --tail 100 cardano-node >&2 || true + exit 1 + fi +} + check_running() { if [ "$(docker inspect --format '{{.State.Running}}' cardano-node)" != true ]; then echo "::error::cardano-node stopped before reaching the sync target." >&2 @@ -31,6 +47,7 @@ echo "Waiting for node socket..." deadline=$((SECONDS + 900)) while [ ! -S "$RUNNER_TEMP/ipc/node.socket" ]; do check_running + check_deadline if [ "$SECONDS" -ge "$deadline" ]; then echo "::error::Timed out waiting for node socket." >&2 docker logs --tail 100 cardano-node >&2 || true @@ -43,6 +60,7 @@ echo "Waiting for sync to reach $SYNC_LEVEL..." query_failures=0 while true; do check_running + check_deadline # docker exec bypasses the image's CLI wrapper, so supply the socket explicitly. if ! tip=$(docker exec cardano-node cardano-cli query tip --testnet-magic 2 --socket-path /ipc/node.socket); then query_failures=$((query_failures + 1)) @@ -55,7 +73,13 @@ while true; do continue fi query_failures=0 - progress=$(jq -er '.syncProgress | tonumber | select(. >= 0 and . <= 100)' <<< "$tip") + # A successful query can omit progress while the node is initializing. + # Missing progress is not zero, even when the requested target is zero. + if ! progress=$(jq -er '.syncProgress // empty | tonumber | select(. >= 0 and . <= 100)' <<< "$tip"); then + echo "Waiting for node to report sync progress..." >&2 + sleep 5 + continue + fi echo "Sync progress: ${progress}%" if jq -en --argjson progress "$progress" --argjson level "$SYNC_LEVEL" \ '$progress >= ($level * 100)' >/dev/null; then diff --git a/.github/workflows/scripts/test-preview-cache.py b/.github/workflows/scripts/test-preview-cache.py index 112b09095..2b6baea4d 100644 --- a/.github/workflows/scripts/test-preview-cache.py +++ b/.github/workflows/scripts/test-preview-cache.py @@ -84,8 +84,12 @@ def copy(src, dst): count.write_text(str(n+1)) if n < int(os.environ.get('QUERY_FAILURES','0')): print('fixture query failure',file=sys.stderr);sys.exit(1) - progress=json.loads(os.environ.get('PROGRESS','["100.00"]')) - print(json.dumps({'syncProgress':progress[min(n,len(progress)-1)]})) + if 'TIP_RESPONSES' in os.environ: + tips=json.loads(os.environ['TIP_RESPONSES']) + print(json.dumps(tips[min(n,len(tips)-1)])) + else: + progress=json.loads(os.environ.get('PROGRESS','["100.00"]')) + print(json.dumps({'syncProgress':progress[min(n,len(progress)-1)]})) else: raise AssertionError(args) ''' @@ -293,5 +297,30 @@ def test_sync_validates_target_and_accepts_zero(self): self.run_script('preview-node-sync.sh',args=['0'],extra={'PROGRESS':'["0"]'}) + def test_sync_waits_for_missing_progress_even_at_zero_target(self): + self.node_socket() + tips = [{}] * 15 + [{'syncProgress':'0'}] + result = self.run_script('preview-node-sync.sh', args=['0'], + extra={'TIP_RESPONSES':json.dumps(tips)}) + self.assertIn('Waiting for node to report sync progress', result.stderr) + self.assertIn('Reached target sync level', result.stdout) + self.assertEqual((self.root / 'query-count').read_text(), '16') + + def test_sync_retries_unusable_progress_then_reaches_target(self): + self.node_socket() + tips = [{'syncProgress':value} for value in [None, 'unknown', -1, 101, '100.00']] + result = self.run_script('preview-node-sync.sh', args=['1'], + extra={'TIP_RESPONSES':json.dumps(tips)}) + self.assertIn('Reached target sync level', result.stdout) + self.assertEqual((self.root / 'query-count').read_text(), '5') + + def test_missing_progress_respects_sync_deadline(self): + self.node_socket() + result = self.run_script('preview-node-sync.sh', ok=False, args=['1'], + extra={'TIP_RESPONSES':'[{}]', 'SYNC_TIMEOUT_SECONDS':'1'}) + self.assertIn('Waiting for node to report sync progress', result.stderr) + self.assertIn('Timed out waiting for node to sync', result.stderr) + + if __name__ == '__main__': unittest.main(verbosity=2)