From 6ea1ac941f7175b92924bba312779762f032881a Mon Sep 17 00:00:00 2001 From: Ilan Harel Date: Mon, 9 Nov 2026 13:43:20 +0200 Subject: [PATCH] ci: openshell-next branch automation that tracks OpenShell releases Closes #79 Signed-off-by: Ilan Harel --- .github/rulesets/protect-openshell-next.json | 11 ++ .github/workflows/ci.yml | 2 + .github/workflows/openshell-next.yml | 193 +++++++++++++++++++ .github/workflows/upstream-compat.yml | 23 ++- AGENTS.md | 5 +- CHANGELOG.md | 2 + CONTRIBUTING.md | 8 +- docs/DEVELOPING.md | 69 ++++++- docs/RELEASING.md | 20 ++ scripts/labels.sh | 1 + scripts/openshell-next.sh | 83 ++++++++ scripts/openshell-tags.sh | 59 ++++++ scripts/upstream-compat.sh | 41 ++-- 13 files changed, 475 insertions(+), 42 deletions(-) create mode 100644 .github/rulesets/protect-openshell-next.json create mode 100644 .github/workflows/openshell-next.yml create mode 100755 scripts/openshell-next.sh create mode 100755 scripts/openshell-tags.sh diff --git a/.github/rulesets/protect-openshell-next.json b/.github/rulesets/protect-openshell-next.json new file mode 100644 index 0000000..11cde38 --- /dev/null +++ b/.github/rulesets/protect-openshell-next.json @@ -0,0 +1,11 @@ +{ + "name": "protect-openshell-next", + "target": "branch", + "enforcement": "active", + "conditions": { "ref_name": { "include": ["refs/heads/openshell-next"], "exclude": [] } }, + "rules": [ + { "type": "deletion" }, + { "type": "non_fast_forward" } + ], + "bypass_actors": [] +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c04d24..bd9497a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,8 @@ on: push: branches: [main] pull_request: + # Lets .github/workflows/openshell-next.yml run CI on the commits it pushes (GITHUB_TOKEN pushes trigger nothing else). + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/openshell-next.yml b/.github/workflows/openshell-next.yml new file mode 100644 index 0000000..977e7b9 --- /dev/null +++ b/.github/workflows/openshell-next.yml @@ -0,0 +1,193 @@ +name: openshell-next + +# Keeps the long-lived `openshell-next` branch building against the newest NVIDIA/OpenShell tag, prereleases included, +# so the next OpenShell release is integrated (and any breakage fixed) before it ships as stable. +# Daily: merge main into openshell-next, bump the OpenShell pin when upstream has a newer tag, run the tests, push, and +# report on the tracking issue (label `openshell-next`). See docs/DEVELOPING.md ("Tracking the next OpenShell release"). +# +# Pushes made with GITHUB_TOKEN do not trigger other workflows, so this job runs the test suite itself and then +# dispatches ci.yml on the pushed commit (workflow_dispatch is the one event GITHUB_TOKEN may start), which puts the +# required `check` / `audit` results on the tracking PR's head commit. +# Local equivalent: scripts/openshell-next.sh sync && scripts/openshell-next.sh update + +on: + schedule: + - cron: "17 4 * * *" # daily 04:17 UTC + workflow_dispatch: + inputs: + tag: + description: "OpenShell tag to pin (default: the newest upstream tag). Applied only when newer than the current pin." + required: false + default: "" + report: + description: "Comment on the tracking issue even when nothing changed" + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: openshell-next + cancel-in-progress: false + +env: + CARGO_TERM_COLOR: always + BRANCH: openshell-next + UPSTREAM: https://github.com/NVIDIA/OpenShell + +jobs: + track: + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write # push the main merge / pin bump to openshell-next (never to main) + issues: write # comment on the tracking issue + actions: write # dispatch ci.yml on the pushed commit + pull-requests: read # find the tracking PR + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Check out openshell-next (created from main if missing) + id: branch + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null; then + git checkout -B "$BRANCH" "origin/$BRANCH" + echo "created=false" >> "$GITHUB_OUTPUT" + else + git checkout -B "$BRANCH" origin/main + echo "created=true" >> "$GITHUB_OUTPUT" + fi + echo "before=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Install Rust + run: rustup toolchain install 1.94 --profile minimal --component rustfmt,clippy && rustup default 1.94 + + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-openshell-next-${{ hashFiles('Cargo.lock') }} + restore-keys: ${{ runner.os }}-openshell-next- + + - name: Merge main into openshell-next + id: sync + run: scripts/openshell-next.sh sync origin/main 2>&1 | tee "$RUNNER_TEMP/sync.log" + + - name: Bump to the newest OpenShell tag + id: bump + env: + WANT: ${{ inputs.tag }} + run: | + scripts/openshell-next.sh update "$WANT" | tee "$RUNNER_TEMP/update.out" + if ! git diff --quiet; then + tag=$(sed -n 's/^tag=//p' "$RUNNER_TEMP/update.out") + prev=$(sed -n 's/^previous=//p' "$RUNNER_TEMP/update.out") + git commit -a --signoff -m "chore(deps): track OpenShell $tag on openshell-next" \ + -m "Automated by .github/workflows/openshell-next.yml. Upstream changes: $UPSTREAM/compare/$prev...$tag" + fi + + - name: Test (fmt, clippy, test, docs) + id: test + continue-on-error: true + run: | + set -o pipefail + { + cargo fmt --check && + cargo clippy --all-targets --locked -- -D warnings && + cargo test --locked && + scripts/gen-docs.sh --check + } 2>&1 | tee "$RUNNER_TEMP/test.log" + + # The branch mirrors upstream, so a bump is pushed even when the tests fail: the fix is then made on top of it. + - name: Push openshell-next + id: push + env: + BEFORE: ${{ steps.branch.outputs.before }} + CREATED: ${{ steps.branch.outputs.created }} + run: | + if [ "$CREATED" = true ] || [ "$(git rev-parse HEAD)" != "$BEFORE" ]; then + git push origin "HEAD:refs/heads/$BRANCH" + echo "pushed=true" >> "$GITHUB_OUTPUT" + else + echo "nothing new to push" + echo "pushed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Run CI on the pushed commit + if: steps.push.outputs.pushed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run ci.yml --ref "$BRANCH" + + - name: Report on the tracking issue + if: always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SYNC: ${{ steps.sync.outcome }} + TEST: ${{ steps.test.outcome }} + CHANGED: ${{ steps.bump.outputs.changed }} + PREV: ${{ steps.bump.outputs.previous }} + TAG: ${{ steps.bump.outputs.tag }} + STABLE: ${{ steps.bump.outputs.stable }} + CREATED: ${{ steps.branch.outputs.created }} + PUSHED: ${{ steps.push.outputs.pushed }} + FORCE_REPORT: ${{ inputs.report }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + run: | + icon() { case "$1" in success) echo "✅ passed" ;; failure) echo "❌ failed" ;; *) echo "⏭️ skipped" ;; esac; } + if [ "$CHANGED" != true ] && [ "$SYNC" = success ] && [ "$TEST" = success ] && [ "$CREATED" != true ] && [ "$FORCE_REPORT" != true ]; then + echo "pin unchanged (${TAG:-?}) and everything passed: nothing to report" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + sha=$(git rev-parse HEAD) + pr=$(gh pr list --head "$BRANCH" --base main --state open --json url --jq '.[0].url // empty') + { + if [ "$CHANGED" = true ]; then echo "### OpenShell \`$TAG\` is now pinned on \`$BRANCH\`"; else echo "### \`$BRANCH\` check (pin \`${TAG:-unknown}\`)"; fi + echo + echo "| | |" + echo "|---|---|" + if [ "$CHANGED" = true ]; then + echo "| Pin | \`$PREV\` → \`$TAG\` |" + echo "| Upstream | [release]($UPSTREAM/releases/tag/$TAG) · [changes since \`$PREV\`]($UPSTREAM/compare/$PREV...$TAG) |" + fi + echo "| Merge of main | $(icon "$SYNC") |" + echo "| Tests (fmt, clippy, test, docs) | $(icon "$TEST") |" + echo "| Branch | [\`$BRANCH\` @ \`${sha:0:12}\`]($REPO_URL/commit/$sha)$([ "$PUSHED" = true ] && echo ", pushed; ci.yml dispatched") |" + echo "| Tracking PR | ${pr:-none open: [open one]($REPO_URL/compare/main...$BRANCH?expand=1) as a draft} |" + echo "| Run | $RUN_URL |" + if [ "$SYNC" = failure ]; then + echo; echo "Merging main hit conflicts outside Cargo.toml/Cargo.lock; nothing was pushed. Resolve them by hand:" + echo '```'; tail -n 20 "$RUNNER_TEMP/sync.log" 2>/dev/null; echo '```' + fi + if [ "$TEST" = failure ]; then + echo; echo "Last lines of the test log (fix on \`$BRANCH\`; fixes that also hold on main's pin go to main first):" + echo '```'; tail -n 40 "$RUNNER_TEMP/test.log" 2>/dev/null; echo '```' + fi + if [ "$CHANGED" = true ] && [ "$STABLE" = true ]; then + echo + echo "**\`$TAG\` is a stable OpenShell release.** Promote \`$BRANCH\` to main as described in" + echo "[docs/RELEASING.md]($REPO_URL/blob/main/docs/RELEASING.md#promoting-openshell-next-when-openshell-ships-stable):" + echo "update the version strings and CHANGELOG on \`$BRANCH\`, mark the tracking PR ready for review, and squash-merge it." + fi + } > "$RUNNER_TEMP/body.md" + cat "$RUNNER_TEMP/body.md" >> "$GITHUB_STEP_SUMMARY" + gh label create openshell-next --color 1D76DB --description "Tracking the next OpenShell release" 2>/dev/null || true + issue=$(gh issue list --label openshell-next --state open --json number --jq '.[0].number // empty') + if [ -n "$issue" ]; then + gh issue comment "$issue" --body-file "$RUNNER_TEMP/body.md" + else + gh issue create --label openshell-next --title "Track the next OpenShell release on openshell-next" \ + --body-file "$RUNNER_TEMP/body.md" + fi + + - name: Fail the run when the merge or the tests failed + if: steps.sync.outcome == 'failure' || steps.test.outcome == 'failure' + run: exit 1 diff --git a/.github/workflows/upstream-compat.yml b/.github/workflows/upstream-compat.yml index 4b3d2ae..c6e1218 100644 --- a/.github/workflows/upstream-compat.yml +++ b/.github/workflows/upstream-compat.yml @@ -1,19 +1,28 @@ name: Upstream compatibility -# Builds and tests openshell-annotate against NVIDIA/OpenShell's newest release, newest prerelease and its main branch, -# so changes to the policy schema show up here before they bite users. -# Informational only: this is NOT a required check (see .github/rulesets/protect-main.json). +# Builds and tests main against NVIDIA/OpenShell's newest release, newest prerelease and its main branch (both policy +# crates repointed to the same upstream commit), so upstream changes show up here before they bite users. +# Informational only: this is NOT a required check (see .github/rulesets/protect-main.json), and only scheduled and +# manual runs can fail. The `openshell-next` branch (.github/workflows/openshell-next.yml) is where a newer pin is +# actually integrated and fixed. # Run the same thing locally with: scripts/upstream-compat.sh on: schedule: - cron: "0 5 * * 1" # Mondays 05:00 UTC workflow_dispatch: + # Every merge to main, so a change that breaks the next OpenShell is visible right away (non-blocking). + push: + branches: [main] + paths-ignore: + - "**.md" + - "docs/**" pull_request: paths: - Cargo.toml - Cargo.lock - scripts/upstream-compat.sh + - scripts/openshell-tags.sh - .github/workflows/upstream-compat.yml permissions: @@ -30,8 +39,8 @@ jobs: compat: name: compat (${{ matrix.target }}) runs-on: ubuntu-latest - # Never turns a normal PR red; scheduled and manual runs report real failures. - continue-on-error: ${{ github.event_name == 'pull_request' }} + # Never turns a PR or a push to main red; scheduled and manual runs report real failures. + continue-on-error: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} permissions: contents: read issues: write # only used by the "report" step on scheduled runs @@ -40,10 +49,10 @@ jobs: matrix: target: [latest-release, latest-prerelease, main] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust run: rustup toolchain install 1.94 --profile minimal && rustup default 1.94 - - uses: actions/cache@v4 + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ~/.cargo/registry diff --git a/AGENTS.md b/AGENTS.md index bc06846..6b58d02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,9 @@ subprocesses; add directives with reasons; run `scan`, `generate`, `lint`, `drif ## When changing this tool - Build/test: `cargo build && cargo test` (Rust 1.94+). The schema crate is a git dependency pinned to the OpenShell `v0.1.7` tag (see `Cargo.toml`, `Cargo.lock` pins the commit). Update the tag deliberately and re-run - the golden tests. + the golden tests. `openshell-next` builds against the newest upstream tag, prereleases included, and is moved by + `.github/workflows/openshell-next.yml`; fixes that need the newer OpenShell go there, fixes valid on the current pin + go to `main` (see `docs/DEVELOPING.md`, "Tracking the next OpenShell release"). - Never emit a field the pinned `openshell-policy-schema` rejects. `generate` re-parses its own output with `parse_policy`; tests assert this. - Generated YAML is written as text (comments with `file:line`) because `serialize_policy` drops comments. @@ -37,6 +39,7 @@ A change is not done until the docs that describe it match. Update them in the s | Directives or attributes (`src/directive.rs`, `src/spec.rs`, `src/generate.rs`) | `scripts/gen-docs.sh` (regenerates `docs/DIRECTIVES.md`) and `skills/annotate-openshell-policy/SKILL.md` if authoring guidance changes | | Release process (`scripts/release.sh`, `release.yml`, `install.sh`, packaging) | `docs/RELEASING.md` | | Pinned OpenShell version (`Cargo.toml`) | every doc that names the version (`git grep -n 'v0\.1\.[0-9]'`) | +| OpenShell tracking (`scripts/openshell-*.sh`, `scripts/upstream-compat.sh`, `openshell-next.yml`, `upstream-compat.yml`) | `docs/DEVELOPING.md` (section 4) and the promotion steps in `docs/RELEASING.md` | | Branch, issue or CI rules | `CONTRIBUTING.md`, `docs/DEVELOPING.md`, `.github/pull_request_template.md` | | Anything user-visible | a line under `## [Unreleased]` in `CHANGELOG.md` | diff --git a/CHANGELOG.md b/CHANGELOG.md index 9537e65..005a687 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ version section and adds the squash-merged PR titles since the previous tag. Ver ## [Unreleased] +- ci: `openshell-next` branch automation: a daily workflow merges main into `openshell-next`, moves its OpenShell pin to the newest upstream tag (prereleases included), runs the tests and reports on the tracking issue; `scripts/openshell-tags.sh` and `scripts/openshell-next.sh` do the same by hand +- fix(ci): `scripts/upstream-compat.sh` repoints both OpenShell crates (it left `openshell-policy` on the old tag) and runs on macOS; `upstream-compat.yml` also runs, non-blocking, on every push to main - chore(audit): ignore RUSTSEC-2026-0459 (`libsqlite3-sys` 0.37.0, pinned by OpenShell v0.1.7 through `sqlx` 0.9.0) in `.cargo/audit.toml` until OpenShell upgrades it (#76) - chore(deps): build against OpenShell v0.1.7 (`openshell-policy-schema`, `openshell-policy`); docs and generated policy headers now name v0.1.7 - docs(actions): `docs/GITHUB_ACTIONS.md` is now a setup and usage guide (versions, recipes, troubleshooting); its inputs/outputs tables are generated from `action.yml` by `scripts/gen-actions-docs.sh` and checked in CI diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b72d75..43d32c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,11 @@ required (0 approvals, conversations resolved), status checks `check` and `audit history, no force pushes, no deletion, and no bypass for admins. GitHub only enforces rulesets on private repositories with a paid plan; until then the rules are followed by convention. +The one exception to "branch from `main`" is `openshell-next`, a long-lived integration branch that builds against the +newest OpenShell prerelease and is updated daily by `.github/workflows/openshell-next.yml`. Fixes that only apply to the +new OpenShell version go in a branch off `openshell-next` with a PR into it; everything else goes to `main` first. +Never force-push or delete it. Details: `docs/DEVELOPING.md`, "Tracking the next OpenShell release". + ## Issues GitHub issues are how we track all work: bugs, features and follow-ups. @@ -30,7 +35,8 @@ GitHub issues are how we track all work: bugs, features and follow-ups. disabled; security problems go through `SECURITY.md`, not an issue). 2. Labels: `bug` or `enhancement`, plus an area (`scan`, `generate`, `lint`, `drift`, `diff-logs`, `providers`, `languages`, `docs`). `upstream-openshell` marks behaviour of OpenShell itself that we only track locally; - `upstream-compat` is used by the weekly compatibility job. Create or refresh the labels with `scripts/labels.sh`. + `upstream-compat` is used by the weekly compatibility job and `openshell-next` by the tracking issue for the next + OpenShell release. Create or refresh the labels with `scripts/labels.sh`. 3. Work on a branch (see above) and put `Closes #` in the pull request description, one line per issue, so the issue closes automatically when the PR is squash-merged. Use `Refs #` when the issue should stay open. 4. Keep one pull request per issue, or a small group of closely related issues. diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 29a5abe..f80ee52 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -65,25 +65,71 @@ The inputs/outputs tables in `docs/GITHUB_ACTIONS.md` are generated from `action ### Upstream compatibility (OpenShell schema changes) -`Cargo.toml` pins `openshell-policy-schema` to an OpenShell tag. To see whether the code still builds and passes its -tests (including the golden/fixture tests) against newer OpenShell code, without touching `Cargo.toml` or `Cargo.lock`: +`Cargo.toml` pins both OpenShell crates (`openshell-policy-schema` and the gateway validator `openshell-policy`) to one +OpenShell tag. To see whether the code still builds and passes its tests (including the golden/fixture tests) against +newer OpenShell code, without touching `Cargo.toml` or `Cargo.lock`: ``` scripts/upstream-compat.sh latest-release # newest stable vX.Y.Z tag -scripts/upstream-compat.sh latest-prerelease # newest vX.Y.Z-pre.N tag if newer than stable, else latest-release +scripts/upstream-compat.sh latest-prerelease # newest tag, prereleases included (what openshell-next tracks) scripts/upstream-compat.sh main # OpenShell HEAD scripts/upstream-compat.sh v0.1.7-pre.3 # any tag, branch or commit SHA ``` -It builds a throwaway copy with the dependency repointed to the resolved commit, then runs `cargo build --all-targets`, +It builds a throwaway copy with both crates repointed to the resolved commit, then runs `cargo build --all-targets`, `cargo test` and the docs check, and prints `upstream-compat: @ : compatible|BROKEN` (exit 1 when broken). Set `CARGO_TARGET_DIR` to reuse a build cache. -`.github/workflows/upstream-compat.yml` runs `latest-release`, `latest-prerelease` and `main` weekly (Mondays 05:00 UTC), on manual dispatch, and on pull -requests that touch `Cargo.toml`, `Cargo.lock` or the workflow itself. It is informational, not a required check: it never -turns a normal PR red. A failed *scheduled* run opens (or comments on) a single issue labelled `upstream-compat` with the -target, the upstream commit and a link to the log. The resolved upstream commit is in each run's job summary. -When a target breaks, fix it on a branch (or bump the pin when `latest-release` moves on). +`.github/workflows/upstream-compat.yml` runs `latest-release`, `latest-prerelease` and `main` weekly (Mondays 05:00 UTC), +on manual dispatch, on every push to `main` (except docs-only changes), and on pull requests that touch `Cargo.toml`, +`Cargo.lock`, the scripts or the workflow itself. It is informational, not a required check: pushes and pull requests +never turn red because of it. A failed *scheduled* run opens (or comments on) a single issue labelled `upstream-compat` +with the target, the upstream commit and a link to the log. The resolved upstream commit is in each run's job summary. +When a target breaks, fix it on `openshell-next` (below), or bump the pin when `latest-release` moves on. + +Tag order comes from `scripts/openshell-tags.sh`: only `vX.Y.Z` and `vX.Y.Z-pre.N` tags count, and a stable `vX.Y.Z` +sorts after its own prereleases (`latest`, `latest-stable`, `list`, `pinned`, `newer A B`, `is-stable A`). + +### Tracking the next OpenShell release (`openshell-next`) + +`openshell-next` is the one long-lived branch besides `main`. It is `main` plus the OpenShell pin moved to the newest +upstream tag, prereleases included (for example `v0.1.8-pre.3` while `main` is on `v0.1.7`), plus any fix that only +makes sense on that version. It exists so the next OpenShell is integrated, and its breakage fixed, before it ships as +stable. A draft pull request `openshell-next` → `main` and an issue labelled `openshell-next` track it; the issue gets a +comment for every new upstream tag. + +`.github/workflows/openshell-next.yml` runs daily (04:17 UTC) and on manual dispatch (optional `tag` input to pin a +specific tag, `report` to comment even when nothing changed). It: + +1. checks out `openshell-next` (recreating it from `main` if it was deleted) and merges `origin/main` into it with a + signed-off merge commit. If both sides changed the pin it keeps the newer tag; any other conflict stops the run + without pushing and is reported on the issue for a human to resolve; +2. moves the pin when upstream has a newer tag (`scripts/openshell-next.sh update`) and commits it as + `github-actions[bot]` with a `Signed-off-by` line; +3. runs `cargo fmt --check`, clippy, `cargo test --locked` and `scripts/gen-docs.sh --check` in the same job; +4. pushes the branch, even when the tests fail (the branch mirrors upstream; the fix is made on top), then dispatches + `ci.yml` on it. Pushes with `GITHUB_TOKEN` start no other workflow, so this is what puts `check` and `audit` on the + tracking PR's head commit; +5. comments on the tracking issue when the pin moved, the branch was created, or the merge or tests failed: the old and + new tag, links to the upstream release and compare view, the test result and log tail, and the run. A **stable** tag + adds a note to promote the branch (see "Promoting `openshell-next`" in `docs/RELEASING.md`). + +It uses only `GITHUB_TOKEN` (`contents: write` for `openshell-next`, `issues: write`, `actions: write` to dispatch CI, +`pull-requests: read`). It never pushes to `main` or opens pull requests. + +Working on the branch by hand: + +``` +git fetch origin && git switch openshell-next && git pull --ff-only +scripts/openshell-next.sh sync # merge origin/main (what the workflow does first) +scripts/openshell-next.sh update # move the pin if upstream has a newer tag (edits files only) +scripts/openshell-next.sh bump v0.1.8-pre.3 # or pin an exact tag +TOOLCHAIN=1.94 scripts/ci-local.sh +``` + +Fixes for breakage on the new OpenShell go in a `fix/` branch off `openshell-next` with a PR into +`openshell-next`. A fix that is also correct on `main`'s pin goes to `main` first (its own issue and PR) and reaches +`openshell-next` with the next sync. Never force-push or rebase `openshell-next`: it only moves forward by merges. `cargo audit` ignores are listed in `.cargo/audit.toml`, each with its advisory and the condition for removing it. Today that is `RUSTSEC-2026-0459` (`libsqlite3-sys` pinned by OpenShell's `sqlx`, #76): whenever you bump the OpenShell @@ -176,6 +222,7 @@ It is not needed to contribute; `cargo test` covers the generator. ## 8. Branch workflow `main` is protected: no direct commits or pushes, only branches and pull requests (full rules in `CONTRIBUTING.md`). +The one long-lived branch is `openshell-next`, which the `openshell-next` workflow updates (see section 4). ``` git switch main && git pull --ff-only @@ -194,7 +241,9 @@ Missing sign-off: `git commit --amend -s` (last commit) or `git rebase --signoff want public; GitHub's noreply address works. The protection ruleset lives in `.github/rulesets/protect-main.json`; apply it with `scripts/protect-main.sh` (needs a -public repo or a paid plan). +public repo or a paid plan). `.github/rulesets/protect-openshell-next.json` blocks force pushes to and deletion of +`openshell-next` (the workflow only fast-forwards it); apply it once with +`gh api -X POST repos/ilanh/openshell-annotate/rulesets --input .github/rulesets/protect-openshell-next.json`. ## 9. Troubleshooting diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 0899da9..e26bd66 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -59,6 +59,26 @@ before a tag exists. - Actions must be allowed to create attestations (the default for public repositories). If you restrict allowed actions, permit `actions/*`, `docker/*` and `anchore/sbom-action`. All actions are pinned to a commit SHA. +## Promoting `openshell-next` when OpenShell ships stable + +`openshell-next` (see `docs/DEVELOPING.md`, "Tracking the next OpenShell release") builds against OpenShell +prereleases. When upstream tags the stable `vX.Y.Z`, the daily workflow moves the branch to it and adds a "stable +release" note to the tracking issue. To bring it to `main`: + +1. Read the upstream release notes and the compare link in that comment; tick the tracking PR's checklist. +2. On `openshell-next` (via a `chore/openshell-vX.Y.Z` PR into it, or directly if you maintain the branch), update + everything that names the old version: `git grep -n 'v\?0\.1\.[0-9]'` (README, AGENTS.md, docs, `Dockerfile`, the strings + in `src/`), regenerate the goldens (`UPDATE_GOLDEN=1 cargo test`, then review), and add a CHANGELOG line under + `## [Unreleased]`. The workflow only moves the pin, so these strings still name the old version until then. +3. Retitle the tracking PR to `chore(deps): bump OpenShell crates to vX.Y.Z`, put `Closes #` in it, + mark it ready for review, and squash-merge it once `check` and `audit` pass. `openshell-next` itself survives the + merge (the `protect-openshell-next` ruleset blocks the automatic branch deletion) and keeps tracking from `main`. +4. Open a new tracking issue (label `openshell-next`) and a new draft PR for the following release, or let the + workflow create the issue at the next upstream tag; it links a compare view for opening the PR. +5. Cut a release of openshell-annotate as usual if users should get the new pin. + +Never promote a prerelease pin to `main`: stable openshell-annotate releases build against stable OpenShell only. + ## Installing ``` diff --git a/scripts/labels.sh b/scripts/labels.sh index cbc666e..c7f605b 100755 --- a/scripts/labels.sh +++ b/scripts/labels.sh @@ -19,6 +19,7 @@ LABELS=( "good first issue|7057ff|Good for newcomers" "upstream-openshell|b60205|Behaviour of OpenShell itself; tracked locally, not filed upstream" "upstream-compat|e99695|Breakage against a newer OpenShell (opened by the weekly job)" + "openshell-next|1d76db|Tracking the next OpenShell release on the openshell-next branch" ) for entry in "${LABELS[@]}"; do diff --git a/scripts/openshell-next.sh b/scripts/openshell-next.sh new file mode 100755 index 0000000..fa3b2a1 --- /dev/null +++ b/scripts/openshell-next.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# Maintain the `openshell-next` integration branch, which builds against the newest NVIDIA/OpenShell tag +# (prereleases included). Used by .github/workflows/openshell-next.yml; safe to run by hand on a local checkout. +# +# scripts/openshell-next.sh bump # pin both OpenShell crates to in Cargo.toml and re-resolve Cargo.lock +# scripts/openshell-next.sh update [] # bump to (default: newest upstream tag) only if it is newer than the pin +# scripts/openshell-next.sh sync [] # merge (default origin/main) into the current branch +# +# `bump` and `update` edit files only; they never commit. `sync` creates a signed-off merge commit. When main and the +# branch both changed the pin, `sync` keeps the newer of the two tags and re-resolves Cargo.lock; any other conflict +# aborts the merge (exit 3) so a human can resolve it. +# `update` prints `previous=`, `tag=`, `changed=` and `stable=` lines (also appended to $GITHUB_OUTPUT when set). +# Exit: 0 ok, 2 usage / bad tag, 3 merge conflict that needs a human. +set -euo pipefail +cd "$(dirname "$0")/.." +TAGS=scripts/openshell-tags.sh +CRATES=(openshell-policy-schema openshell-policy) + +bump() { + local tag="$1" tmp + [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$ ]] || { echo "not a release tag: $tag" >&2; exit 2; } + tmp=$(mktemp) + # Repoint both crates (tag = or rev =) and the comment above them; portable sed (no -i). + sed -E \ + -e "/^openshell-policy(-schema)?[[:space:]]*=/ s/(tag|rev)[[:space:]]*=[[:space:]]*\"[^\"]+\"/tag = \"$tag\"/" \ + -e "s/^# Pinned to the OpenShell [^ ]+ tag;/# Pinned to the OpenShell $tag tag;/" \ + Cargo.toml > "$tmp" + cat "$tmp" > Cargo.toml; rm -f "$tmp" + for c in "${CRATES[@]}"; do + grep -Eq "^${c}[[:space:]]*=.*tag = \"$tag\"" Cargo.toml || { echo "failed to pin $c to $tag in Cargo.toml" >&2; exit 2; } + done + cargo update -p openshell-policy-schema -p openshell-policy --quiet + echo "pinned ${CRATES[*]} to $tag" +} + +out() { echo "$1"; if [ -n "${GITHUB_OUTPUT:-}" ]; then echo "$1" >> "$GITHUB_OUTPUT"; fi; } + +update() { + local want="${1:-}" have + have=$("$TAGS" pinned) + [ -n "$want" ] || want=$("$TAGS" latest) + out "previous=$have" + if "$TAGS" newer "$want" "$have"; then bump "$want" >&2; out "tag=$want"; out "changed=true" + else out "tag=$have"; out "changed=false"; want="$have" + fi + if "$TAGS" is-stable "$want"; then out "stable=true"; else out "stable=false"; fi +} + +sync() { + local ref="${1:-origin/main}" before after bad msg + before=$("$TAGS" pinned) + msg="Merge $ref into $(git rev-parse --abbrev-ref HEAD)" + if git merge --no-ff --no-edit --signoff -m "$msg" "$ref"; then + echo "merged $ref cleanly" + else + bad=$(git diff --name-only --diff-filter=U | grep -vxE 'Cargo\.toml|Cargo\.lock' || true) + if [ -n "$bad" ]; then + echo "conflicts outside Cargo.toml/Cargo.lock need a human:" >&2; echo "$bad" >&2 + git merge --abort; exit 3 + fi + # Both sides moved the pin: start from main's files and re-apply the newer tag. + git checkout --theirs -- Cargo.toml Cargo.lock 2>/dev/null || git checkout "$ref" -- Cargo.toml Cargo.lock + after=$("$TAGS" pinned) + if "$TAGS" newer "$before" "$after"; then bump "$before"; fi + git add Cargo.toml Cargo.lock + git commit --signoff -m "$msg" -m "Both sides changed the OpenShell pin; kept the newer tag." + echo "merged $ref; resolved the pin conflict to $("$TAGS" pinned)" + fi + # A clean merge can still leave Cargo.lock stale (e.g. main added a dependency): re-resolve minimally. + if ! cargo metadata --locked --format-version 1 >/dev/null 2>&1; then + cargo metadata --format-version 1 >/dev/null + git add Cargo.lock + git commit --signoff -m "chore(deps): refresh Cargo.lock after merging $ref" + fi +} + +cmd="${1:-}"; shift || true +case "$cmd" in + bump) [ $# -eq 1 ] || { echo "usage: $0 bump " >&2; exit 2; }; bump "$1" ;; + update) update "${1:-}" ;; + sync) sync "${1:-}" ;; + *) sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;; +esac diff --git a/scripts/openshell-tags.sh b/scripts/openshell-tags.sh new file mode 100755 index 0000000..c5ad8dd --- /dev/null +++ b/scripts/openshell-tags.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# List and compare NVIDIA/OpenShell release tags in release order (a stable vX.Y.Z sorts after its vX.Y.Z-pre.N). +# +# scripts/openshell-tags.sh latest # newest tag, prereleases included (what `openshell-next` tracks) +# scripts/openshell-tags.sh latest-stable # newest vX.Y.Z +# scripts/openshell-tags.sh list # every release tag, oldest first +# scripts/openshell-tags.sh pinned # the tag Cargo.toml pins openshell-policy-schema to +# scripts/openshell-tags.sh newer A B # exit 0 when tag A is newer than tag B, else 1 +# scripts/openshell-tags.sh is-stable A # exit 0 when A is vX.Y.Z (no prerelease suffix) +# +# Only vX.Y.Z and vX.Y.Z-pre.N tags count; anything else upstream (`dev`, `vm-runtime`, ...) is ignored. +# Env: UPSTREAM_REPO (default https://github.com/NVIDIA/OpenShell); OPENSHELL_TAGS (newline-separated tag list to use +# instead of asking the remote, for tests and offline runs). +# Exit: 0 ok / true, 1 false, 2 usage or no tags found. +set -euo pipefail +cd "$(dirname "$0")/.." + +REPO="${UPSTREAM_REPO:-https://github.com/NVIDIA/OpenShell}" +RE='^v[0-9]+\.[0-9]+\.[0-9]+(-pre\.[0-9]+)?$' + +usage() { sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } + +valid() { [[ "$1" =~ $RE ]]; } + +# "v0.1.8-pre.3" -> "0 1 8 3 v0.1.8-pre.3"; a stable tag gets 999999 so it sorts after its prereleases. +key() { + local t="${1#v}" pre=999999 + case "$t" in *-pre.*) pre="${t##*-pre.}"; t="${t%-pre.*}" ;; esac + printf '%s %s %s %s %s\n' "${t%%.*}" "$(echo "$t" | cut -d. -f2)" "${t##*.}" "$pre" "$1" +} + +sorted() { while read -r t; do valid "$t" && key "$t"; done | sort -k1,1n -k2,2n -k3,3n -k4,4n | awk '{print $5}'; } + +tags() { + if [ -n "${OPENSHELL_TAGS:-}" ]; then printf '%s\n' "$OPENSHELL_TAGS" + else git ls-remote --tags --refs "$REPO" | sed 's#.*refs/tags/##' + fi | sorted +} + +need() { [ -n "$1" ] || { echo "no matching release tag in $REPO" >&2; exit 2; }; echo "$1"; } + +cmd="${1:-}"; [ -n "$cmd" ] || usage; shift +case "$cmd" in + list) tags ;; + latest) need "$(tags | tail -1)" ;; + latest-stable) need "$(tags | grep -v -- '-pre\.' | tail -1)" ;; + pinned) + t=$(sed -n -E 's/^openshell-policy-schema[[:space:]]*=.*tag[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' Cargo.toml) + [ -n "$t" ] || { echo "Cargo.toml does not pin openshell-policy-schema to a tag" >&2; exit 2; } + echo "$t" ;; + newer) + if [ $# -ne 2 ] || ! valid "$1" || ! valid "$2"; then echo "newer: need two release tags, got: $*" >&2; exit 2; fi + [ "$1" != "$2" ] && [ "$(printf '%s\n%s\n' "$1" "$2" | sorted | tail -1)" = "$1" ] ;; + is-stable) + if [ $# -ne 1 ] || ! valid "$1"; then echo "is-stable: need one release tag" >&2; exit 2; fi + [[ "$1" != *-pre.* ]] ;; + -h|--help|help) usage ;; + *) echo "unknown command: $cmd" >&2; usage ;; +esac diff --git a/scripts/upstream-compat.sh b/scripts/upstream-compat.sh index b54731e..ab02173 100755 --- a/scripts/upstream-compat.sh +++ b/scripts/upstream-compat.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# Build and test openshell-annotate against a different version of OpenShell's policy-schema crate, +# Build and test openshell-annotate against a different version of OpenShell's policy crates +# (openshell-policy-schema and openshell-policy, both repointed to the same upstream commit), # WITHOUT touching this working tree (Cargo.toml / Cargo.lock stay pinned). Works on a temporary copy. # # scripts/upstream-compat.sh latest-release # newest stable vX.Y.Z tag of NVIDIA/OpenShell -# scripts/upstream-compat.sh latest-prerelease # newest vX.Y.Z-pre.N tag, if newer than latest stable (else latest-release) +# scripts/upstream-compat.sh latest-prerelease # newest tag incl. vX.Y.Z-pre.N (what openshell-next tracks; = latest-release when no newer prerelease) # scripts/upstream-compat.sh main # OpenShell HEAD # scripts/upstream-compat.sh v0.1.7-pre.3 # any tag, branch or commit SHA # @@ -18,27 +19,19 @@ TARGET="${1:-}" [ -n "$TARGET" ] || { echo "usage: $0 " >&2; exit 2; } # --- resolve the target to a commit SHA --------------------------------------------------------- +tag_sha() { git ls-remote "$REPO" "refs/tags/$1^{}" "refs/tags/$1" | awk 'NR==1{print $1}'; } case "$TARGET" in latest-prerelease) - # Newest vX.Y.Z-pre.N tag by version order, used only if it is newer than the newest stable tag. - TAGS=$(git ls-remote --tags --refs "$REPO" | sed "s#.*refs/tags/##") || true - STABLE=$(printf "%s\n" "$TAGS" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -V | tail -1) || true - PRE=$(printf "%s\n" "$TAGS" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-pre\.[0-9]+$" | sort -V | tail -1) || true - # A prerelease vX.Y.Z-pre.N precedes vX.Y.Z, so compare its base version against the stable tag. - if [ -n "${PRE:-}" ] && { [ -z "${STABLE:-}" ] || { [ "${PRE%-pre.*}" != "$STABLE" ] && [ "$(printf "%s\n%s\n" "$STABLE" "${PRE%-pre.*}" | sort -V | tail -1)" = "${PRE%-pre.*}" ]; }; }; then - LABEL="$PRE" - else - echo "note: no prerelease newer than ${STABLE:-}; falling back to latest-release" >&2 - LABEL="$STABLE" - fi - [ -n "${LABEL:-}" ] || { echo "could not find a release tag in $REPO" >&2; exit 2; } - SHA=$(git ls-remote "$REPO" "refs/tags/$LABEL^{}" "refs/tags/$LABEL" | awk "NR==1{print \$1}") + # Newest tag in release order (scripts/openshell-tags.sh): a vX.Y.Z-pre.N only when it is newer than the newest + # stable vX.Y.Z, otherwise that stable tag. This is the tag the `openshell-next` branch tracks. + LABEL=$(UPSTREAM_REPO="$REPO" scripts/openshell-tags.sh latest) || exit 2 + scripts/openshell-tags.sh is-stable "$LABEL" && echo "note: no prerelease newer than $LABEL; using it" >&2 + SHA=$(tag_sha "$LABEL") ;; latest-release) # Newest stable release tag (vX.Y.Z, no -pre suffix). ls-remote avoids API rate limits. - LABEL=$(git ls-remote --tags --refs "$REPO" | sed 's#.*refs/tags/##' | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -1) || true - [ -n "${LABEL:-}" ] || { echo "could not find a stable release tag in $REPO" >&2; exit 2; } - SHA=$(git ls-remote "$REPO" "refs/tags/$LABEL^{}" "refs/tags/$LABEL" | awk 'NR==1{print $1}') + LABEL=$(UPSTREAM_REPO="$REPO" scripts/openshell-tags.sh latest-stable) || exit 2 + SHA=$(tag_sha "$LABEL") ;; main) LABEL="main" @@ -53,7 +46,7 @@ case "$TARGET" in esac [ -n "${SHA:-}" ] || { echo "could not resolve '$TARGET' in $REPO" >&2; exit 2; } -PINNED=$(grep -E '^openshell-policy-schema' Cargo.toml | sed -E 's/.*tag = "([^"]+)".*/\1/') +PINNED=$(scripts/openshell-tags.sh pinned) echo "==> upstream target: $TARGET ($LABEL) -> $SHA (pinned in Cargo.toml: $PINNED)" if [ -n "${GITHUB_OUTPUT:-}" ]; then @@ -66,14 +59,16 @@ WORK="${XDG_CACHE_HOME:-$HOME/.cache}/openshell-annotate-upstream-compat-src" rm -rf "$WORK"; mkdir -p "$WORK" tar --exclude=./target --exclude=./.git -cf - . | tar -xf - -C "$WORK" cd "$WORK" -# tag = "vX" -> rev = "" on the openshell-policy-schema line only; the lockfile is re-resolved. -sed -i -E "/^openshell-policy-schema/ s/tag = \"[^\"]+\"/rev = \"$SHA\"/" Cargo.toml -grep -q "rev = \"$SHA\"" Cargo.toml || { echo "failed to repoint the dependency in Cargo.toml" >&2; exit 2; } +# tag = "vX" -> rev = "" on BOTH OpenShell crates (schema and gateway validator), so they build from the same +# upstream commit; the lockfile is re-resolved. Portable sed (no -i) so this also runs on macOS. +sed -E "/^openshell-policy(-schema)?[[:space:]]*=/ s/tag = \"[^\"]+\"/rev = \"$SHA\"/" Cargo.toml > Cargo.toml.new && mv Cargo.toml.new Cargo.toml +[ "$(grep -cE "^openshell-policy(-schema)?[[:space:]]*=.*rev = \"$SHA\"" Cargo.toml)" = 2 ] || + { echo "failed to repoint the OpenShell dependencies in Cargo.toml" >&2; exit 2; } [ -n "${CARGO_TARGET_DIR:-}" ] || export CARGO_TARGET_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/openshell-annotate-upstream-compat" run() { printf '\n==> %s\n' "$*"; "$@" || { echo "FAILED: $*"; return 1; }; } steps() { - run cargo update -p openshell-policy-schema && + run cargo update -p openshell-policy-schema -p openshell-policy && run cargo build --all-targets && run cargo test && run scripts/gen-docs.sh --check