From 89ede760fb073c9cf72bb09fbe6fde871b6310d7 Mon Sep 17 00:00:00 2001 From: Julian Vance Date: Mon, 9 Nov 2026 09:33:48 -0600 Subject: [PATCH] feat(publish): push to staging// and sync the served name Pins grc-store-protocol v0.28.1 and uses its staging package to name the upload repository. Catalog and EvaluationLog publishes push the bundle and its signature to staging// and then sync /; the hub copies an accepted bundle into the served repository, and nothing pushes a served repository directly. There is no fallback. The hub client timeout rises from 60 seconds to 3 minutes, above the hub's 120-second catalog sync deadline, so a slow sync ends with the hub's answer rather than a client timeout. A registry 401/403 on the staging push names the staging repository and the namespace to own; a refused sync shows the hub's error code and says whether to publish again. Breaking: a grcli from before this change gets 401 UNAUTHORIZED at its first blob push against a hub that only accepts staging pushes, and this release cannot publish to a hub from before the cutover. Signed-off-by: Julian Vance --- .github/workflows/publish-gemara.yml | 6 +++ CHANGELOG.md | 16 +++++++ CLAUDE.md | 3 +- README.md | 12 ++++++ cmd/publish.go | 55 +++++++++++++++++++----- cmd/publish_test.go | 36 ++++++++++++++++ go.mod | 2 +- go.sum | 4 +- internal/hub/hub.go | 64 +++++++++++++++++++++++++--- internal/hub/hub_test.go | 48 +++++++++++++++++++++ internal/refs/refs.go | 2 +- 11 files changed, 227 insertions(+), 21 deletions(-) diff --git a/.github/workflows/publish-gemara.yml b/.github/workflows/publish-gemara.yml index 8defca5..9e4b2b2 100644 --- a/.github/workflows/publish-gemara.yml +++ b/.github/workflows/publish-gemara.yml @@ -15,6 +15,12 @@ # Trusted CI publisher for the target namespace. Without that binding the hub # returns 403; adding a GitHub secret will not fix it. # +# Staging: grcli pushes each bundle to staging// +# and the hub copies it into / when the sync accepts it. A +# grcli-version from before the staging cutover gets 401 UNAUTHORIZED at its +# first blob push against a cut-over hub; set grcli-version to a release that +# pushes to staging. +# # --- Call it from a catalog repo --- # name: Publish catalog # on: diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aff3db..169716d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,22 @@ Notable changes to `grcli`. This project is pre-1.0; while on `v0.x`, a breaking change bumps the minor version. +## Unreleased + +### Changed + +- **Breaking:** `publish` pushes every bundle and its signature to the staging + repository `staging//` and then syncs `/` + (the `staging` package of grc-store-protocol v0.28.1 names the repository). There is no fallback to pushing + the served repository, so this release cannot publish to a hub from before + the staging cutover, and older releases cannot publish to a hub after it + (their first blob push gets `401 UNAUTHORIZED`). +- The hub client's timeout rises from 60 seconds to 3 minutes, above the + hub's 120-second catalog sync deadline. +- A refused push names the staging repository and the namespace you need to + own, and a refused sync shows the hub's error code and says whether to + publish again. + ## [0.1.0] - Unreleased First release of `grcli` under `github.com/gemaraproj/grcli`, published as a diff --git a/CLAUDE.md b/CLAUDE.md index 997b840..d688851 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,8 @@ a user command.) `hub` (`/v1/bundles/sync`, `/v2/token`, discovery) · `registry` (OCI packing via `oras-go`) · `sign` (cosign shell-out, Sigstore bundle) · `provenance` (SLSA v1.0 predicate) · `source` (load/merge/verify YAML) · `refs` · `digest` (SHA256) · `cache` (immutable-tag disk cache — v2 stores the whole bundle: files + -`bundle.json`). Imports `grc-store-protocol` (discovery, syncapi, registrytoken, spdx). +`bundle.json`). Imports `grc-store-protocol` (discovery, syncapi, registrytoken, spdx, staging, apierror). +`publish` pushes to `staging.Repository(/)` and syncs the served name, with no fallback. **Auth is no longer here**: `internal/auth` (device flow, credential store, token resolution, GHA OIDC) was deleted in favour of `github.com/gemaraproj/grc-store-clientkit/auth`, shared with privateer-sdk. `cmd.grcliApp` (`cmd/root.go`) is the per-tool identity that keeps grcli's own diff --git a/README.md b/README.md index 5328ec9..2045927 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,18 @@ single `-`, trims leading/trailing `-`, `_`, and `.`, and lowercases the result. So a bundle with `author.id: TAG-SC` and `id: cnsc` publishes to `tag-sc/cnsc` — regardless of which namespace you are a member of. +**The bytes go to staging first.** `publish` always pushes the bundle and its +signature to the staging repository `staging//`, then calls +`POST /v1/bundles/sync` naming `/`. The hub checks the staged +bundle and copies an accepted one into `/`; nobody pushes a +served repository directly. The hub deletes the staged upload +once it decides, and staging removes anything left there after about 24 hours, +so a failed publish is fixed by running `grcli publish` again. There is no +fallback: a hub from before the staging cutover gives no push grant on +`staging/...`, and the push fails with `401` naming the staging repository. +Older grcli releases push the served repository directly, which a cut-over hub +refuses with a `401 UNAUTHORIZED` at the first blob push; upgrade grcli. + **This is the usual cause of a 403 on publish.** Authorization is per *namespace* (the part before the `/`), so you must own — or hold a trusted publisher binding for — the namespace the metadata names, not the one you diff --git a/cmd/publish.go b/cmd/publish.go index 215c82c..aa692d2 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "net/http" "os" "regexp" "strings" @@ -16,7 +17,10 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" + "oras.land/oras-go/v2/registry/remote/errcode" + "github.com/revanite-io/grc-store-protocol/spdx" + "github.com/revanite-io/grc-store-protocol/staging" "github.com/gemaraproj/grc-store-clientkit/auth" "github.com/gemaraproj/grcli/internal/digest" @@ -49,9 +53,10 @@ func newPublishCmd(v *viper.Viper) *cobra.Command { Use: "publish [file...]", Short: "Bundle one Gemara artifact with provenance and push it to grc.store", Long: `Loads the file(s) describing a single artifact, attaches a SLSA-shaped -provenance record, packs an OCI bundle, pushes it to the configured -registry, optionally signs with cosign, and notifies the hub via -POST /v1/bundles/sync. +provenance record, packs an OCI bundle, pushes it to the staging +repository staging// on the hub's registry, optionally +signs it, and asks the hub to check it and publish it to +/ via POST /v1/bundles/sync. Files can be supplied as positional arguments (grcli publish a.yaml b.yaml) or via -f / --file. The two forms are mutually exclusive — @@ -106,7 +111,8 @@ need to set a secret.`, // and artifact metadata defaults are merged. type publishTarget struct { registryHost string - repository string + repository string // the served repository: provenance, version check, sync + pushRepo string // staging.Repository(repository): where the bytes go tag string dryRun bool output string @@ -175,7 +181,7 @@ func runPublish(cmd *cobra.Command, v *viper.Viper, positional []string) error { // The registry rejects unauthenticated writes. Mint a repo-scoped // push token from the hub and export it so both the oras push and // the cosign signature push authenticate. - if err := authenticatePush(ctx, v, target.repository); err != nil { + if err := authenticatePush(ctx, v, target.pushRepo); err != nil { return err } } @@ -212,7 +218,8 @@ func runPublish(cmd *cobra.Command, v *viper.Viper, positional []string) error { } return signAndNotify(ctx, v, signContext{ - repository: target.repository, + repository: target.pushRepo, + served: target.repository, tag: target.tag, reference: result.Reference, registryHost: target.registryHost, @@ -223,7 +230,8 @@ func runPublish(cmd *cobra.Command, v *viper.Viper, positional []string) error { // signContext carries the push coordinates the sign + notify step needs. type signContext struct { - repository string + repository string // the staging repository the bundle and signature go to + served string // the served repository the sync names tag string reference string // /:, bare host registryHost string // scheme-prefixed oras dial target @@ -284,9 +292,21 @@ func resolveTarget(ctx context.Context, v *viper.Viper, loaded *source.Loaded) ( registryHost = d.RegistryURL } + // Every publish goes to the staging form of the served repository and + // the sync names the served one; there is no fallback to pushing the + // served repository. + var pushRepo string + if !dryRun { + var ok bool + if pushRepo, ok = staging.Repository(repository); !ok { + return publishTarget{}, fmt.Errorf("repository %q must be / with no empty segment, and %q is not a namespace you can publish to", repository, staging.Segment) + } + } + target := publishTarget{ registryHost: registryHost, repository: repository, + pushRepo: pushRepo, tag: tag, dryRun: dryRun, output: v.GetString(flagOutput), @@ -311,14 +331,29 @@ func pushBundle(ctx context.Context, target publishTarget, in registry.PackInput result.Reference, result.ManifestDigest, result.BodyDigest, artifactType, artifactID) return result, nil } - result, err := registry.PushRemote(ctx, target.registryHost, target.repository, target.tag, in) + result, err := registry.PushRemote(ctx, target.registryHost, target.pushRepo, target.tag, in) if err != nil { - return nil, fmt.Errorf("push: %w", err) + return nil, pushError(target, err) } fmt.Fprintf(out, "pushed %s\n manifest digest: %s\n", result.Reference, result.ManifestDigest) return result, nil } +// pushError explains a registry refusal. The registry token grants push on +// a staging repository only to the namespace's owner, and only on a hub past +// the staging cutover; anything else gets pull only and zot answers the +// first blob push with 401. +func pushError(target publishTarget, err error) error { + var resp *errcode.ErrorResponse + if errors.As(err, &resp) && (resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden) { + ns, _, _ := strings.Cut(target.repository, "/") + return fmt.Errorf("push: the registry refused the push to staging repository %s (%d): "+ + "you need to own namespace %q (or be its trusted CI publisher), and the hub must run the staging cutover; "+ + "a hub that predates it does not accept staging pushes: %w", target.pushRepo, resp.StatusCode, ns, err) + } + return fmt.Errorf("push to %s: %w", target.pushRepo, err) +} + // signAndNotify runs the optional cosign step and the hub sync call. // Status lines go to os.Stdout rather than a passed-in writer because // the cosign subprocess inside sign.Sign writes to os.Stdout/os.Stderr @@ -352,7 +387,7 @@ func signAndNotify(ctx context.Context, v *viper.Viper, sc signContext) error { if err != nil { return err } - syncResp, err := hub.New(hubURL, token).Sync(ctx, sc.repository, sc.tag) + syncResp, err := hub.New(hubURL, token).Sync(ctx, sc.served, sc.tag) if err != nil { return fmt.Errorf("hub sync: %w", err) } diff --git a/cmd/publish_test.go b/cmd/publish_test.go index d1ede39..d929630 100644 --- a/cmd/publish_test.go +++ b/cmd/publish_test.go @@ -5,14 +5,18 @@ package cmd import ( "bytes" "context" + "errors" + "fmt" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "github.com/gemaraproj/grc-store-clientkit/auth" "github.com/spf13/viper" "github.com/stretchr/testify/require" + "oras.land/oras-go/v2/registry/remote/errcode" "github.com/gemaraproj/grcli/internal/registry" "github.com/gemaraproj/grcli/internal/source" @@ -54,6 +58,7 @@ func TestResolveTarget(t *testing.T) { wantTarget: publishTarget{ registryHost: "registry.example", repository: "my-team/my-policy", + pushRepo: "staging/my-team/my-policy", tag: "1.2.3", output: "grcli-out", }, @@ -68,6 +73,7 @@ func TestResolveTarget(t *testing.T) { wantTarget: publishTarget{ registryHost: "registry.example", repository: "custom/repo", + pushRepo: "staging/custom/repo", tag: "1.2.3", output: "grcli-out", }, @@ -87,6 +93,19 @@ func TestResolveTarget(t *testing.T) { output: "/tmp/out", }, }, + { + // A staging name is not a served repository: name the served one. + name: "repository-in-staging-refused", + flags: map[string]any{flagURL: srv.URL, flagRepository: "staging/my-team/my-policy"}, + loaded: loadedFull, + wantErrSub: "is not a namespace you can publish to", + }, + { + name: "single-segment-repository-refused", + flags: map[string]any{flagURL: srv.URL, flagRepository: "solo"}, + loaded: loadedFull, + wantErrSub: `repository "solo" must be /`, + }, { name: "missing-url-when-not-dry-run", flags: map[string]any{}, @@ -304,3 +323,20 @@ func TestApplyVersionFlag(t *testing.T) { }) } } + +// TestPushError checks that a registry 401/403 on the staging push names the +// staging repository, the namespace and the cutover, and that other errors +// still name where the push went. +func TestPushError(t *testing.T) { + target := publishTarget{repository: "acme/ctl", pushRepo: "staging/acme/ctl"} + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + err := pushError(target, fmt.Errorf("packing bundle: %w", &errcode.ErrorResponse{Method: http.MethodPost, StatusCode: status})) + for _, want := range []string{"staging/acme/ctl", `"acme"`, "staging cutover", strconv.Itoa(status)} { + require.Contains(t, err.Error(), want) + } + var resp *errcode.ErrorResponse + require.ErrorAs(t, err, &resp, "the registry error stays wrapped") + } + err := pushError(target, errors.New("connection refused")) + require.Equal(t, "push to staging/acme/ctl: connection refused", err.Error()) +} diff --git a/go.mod b/go.mod index e1009af..0e09603 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/gemaraproj/go-gemara v0.9.2 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 - github.com/revanite-io/grc-store-protocol v0.8.0 + github.com/revanite-io/grc-store-protocol v0.28.1 github.com/sigstore/sigstore-go v1.3.0 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.21.0 diff --git a/go.sum b/go.sum index c92d91e..70c1f72 100644 --- a/go.sum +++ b/go.sum @@ -278,8 +278,8 @@ github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTU github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= -github.com/revanite-io/grc-store-protocol v0.8.0 h1:ZlAm78+yzAHte23FsNNfLpuf0KiVeprwQr5OepfmE7o= -github.com/revanite-io/grc-store-protocol v0.8.0/go.mod h1:naeWSRGJ9dtyzndHdsA4jrENlSmSlIgLQDAKRSpPhwU= +github.com/revanite-io/grc-store-protocol v0.28.1 h1:qgQCQQIXbAyOh1A0SLrhBrDEpYDZmguPKRdKbV81J1w= +github.com/revanite-io/grc-store-protocol v0.28.1/go.mod h1:lsuyx03NQdd71b9FiXa8Auvz6Go8Mzde3gsOGUGJd9g= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 0be72a6..368fb42 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -18,6 +18,8 @@ import ( "strings" "time" + "github.com/revanite-io/grc-store-protocol/apierror" + "github.com/revanite-io/grc-store-protocol/staging" "github.com/revanite-io/grc-store-protocol/syncapi" ) @@ -35,12 +37,17 @@ type Client struct { HTTP *http.Client } +// requestTimeout bounds every hub call. It sits above catalog sync's +// 120-second write deadline on the hub, so a slow sync ends with the hub's +// answer rather than a client timeout. +const requestTimeout = 3 * time.Minute + // New returns a Client with sensible defaults. func New(baseURL, token string) *Client { return &Client{ BaseURL: strings.TrimRight(baseURL, "/"), Token: token, - HTTP: &http.Client{Timeout: 60 * time.Second}, + HTTP: &http.Client{Timeout: requestTimeout}, } } @@ -300,11 +307,7 @@ func (c *Client) Sync(ctx context.Context, repository, tag string) (*SyncRespons return nil, fmt.Errorf("reading sync response from %s: %w", url, err) } if resp.StatusCode/100 != 2 { - // Same shape as VersionExists/GetCatalog default branches (URL + - // status + body snippet) so an operator chasing a 5xx on any hub - // endpoint sees the same diagnostic surface. - return nil, fmt.Errorf("hub sync %s returned %d: %s", - url, resp.StatusCode, strings.TrimSpace(string(rb))) + return nil, newSyncError(url, repository, tag, resp.StatusCode, rb) } out := &SyncResponse{} if err := json.Unmarshal(rb, out); err != nil { @@ -312,3 +315,52 @@ func (c *Client) Sync(ctx context.Context, repository, tag string) (*SyncRespons } return out, nil } + +// SyncError is a refused POST /v1/bundles/sync. Code and Detail come from +// the hub's apierror envelope; Code is empty when the body was not one. +type SyncError struct { + URL string // the sync endpoint, for an operator chasing a 5xx + Repository, Tag string + Status int + Code, Detail string +} + +func newSyncError(url, repository, tag string, status int, body []byte) *SyncError { + e := &SyncError{URL: url, Repository: repository, Tag: tag, Status: status} + var env apierror.Envelope + if json.Unmarshal(body, &env) == nil && env.Error != "" { + e.Code, e.Detail = env.Error, env.Detail + } else { + e.Detail = strings.TrimSpace(string(body)) + } + return e +} + +func (e *SyncError) Error() string { + what := fmt.Sprintf("%s refused %s:%s (%d", e.URL, e.Repository, e.Tag, e.Status) + if e.Code != "" { + what += " " + e.Code + } + what += ")" + if e.Detail != "" { + what += ": " + e.Detail + } + return what + ". " + e.hint() +} + +// hint says what to do next. The hub deletes the staged upload on success +// and on every 4xx but a 429, and keeps it on a 429 or 5xx, so a refusal +// means publishing again, not syncing again. +func (e *SyncError) hint() string { + stagingRepo, _ := staging.Repository(e.Repository) + switch { + case e.Status == http.StatusUnauthorized: + return "Your hub login is missing or expired: run grcli login (in CI, check the job has id-token: write)." + case e.Code == "fetch_failed": + return fmt.Sprintf("The hub could not read the bundle from %s: the push did not finish, or it was pushed more than 24 hours ago and staging removed it. Run grcli publish again.", stagingRepo) + case e.Status == http.StatusTooManyRequests || e.Status >= 500: + return "Nothing was published; run grcli publish again in a moment." + default: + return "Nothing was published and the hub discarded the staged upload; fix the cause and run grcli publish again." + } +} diff --git a/internal/hub/hub_test.go b/internal/hub/hub_test.go index 8f00590..c01ba3c 100644 --- a/internal/hub/hub_test.go +++ b/internal/hub/hub_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) // TestVersionExists_StatusMapping covers the typed status returns — @@ -205,3 +206,50 @@ func TestReleaseFor_License(t *testing.T) { t.Error("ReleaseFor(3.0.0) should be nil") } } + +// TestSync_Refusal checks that a refusal reads the hub's apierror envelope +// and says whether to publish again, naming the staging repository when the +// hub could not read the staged bundle. +func TestSync_Refusal(t *testing.T) { + cases := []struct { + status int + body string + want []string + }{ + {http.StatusBadRequest, `{"error":"fetch_failed","detail":"manifest unknown"}`, + []string{"(400 fetch_failed): manifest unknown", "staging/a/b", "24 hours", "grcli publish again"}}, + {http.StatusUnprocessableEntity, `{"error":"gemara_version_unsupported","detail":"declare 1.5.0 or later"}`, + []string{"(422 gemara_version_unsupported): declare 1.5.0 or later", "discarded the staged upload"}}, + {http.StatusForbidden, `{"error":"forbidden","detail":"bundle sync requires ownership of namespace \"a\""}`, + []string{"(403 forbidden)", "ownership of namespace"}}, + {http.StatusUnauthorized, `{"error":"unauthorized","detail":"token expired"}`, + []string{"grcli login"}}, + {http.StatusServiceUnavailable, `{"error":"schema_check_unavailable","detail":"busy"}`, + []string{"(503 schema_check_unavailable)", "in a moment"}}, + } + for _, tc := range cases { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + _, err := New(srv.URL, "test-token").Sync(context.Background(), "a/b", "1.0.0") + srv.Close() + var se *SyncError + if !errors.As(err, &se) { + t.Fatalf("%d: want *SyncError, got %v", tc.status, err) + } + for _, want := range tc.want { + if !strings.Contains(err.Error(), want) { + t.Errorf("%d: error %q must contain %q", tc.status, err, want) + } + } + } +} + +// TestNew_TimeoutAboveSyncDeadline holds the client timeout above the hub's +// 120-second catalog sync write deadline. +func TestNew_TimeoutAboveSyncDeadline(t *testing.T) { + if got := New("http://hub", "").HTTP.Timeout; got != 3*time.Minute { + t.Errorf("timeout = %s, want 3m", got) + } +} diff --git a/internal/refs/refs.go b/internal/refs/refs.go index 6e2336d..15e133a 100644 --- a/internal/refs/refs.go +++ b/internal/refs/refs.go @@ -164,7 +164,7 @@ func (a *Artifact) Select(mode Mode) []Selected { // - any other host is not resolvable here. // // The path is read by the shared grc-store-protocol/refurl rule — the same -// one the hub index (ADR-0040) and the web UI follow — so a reference grcli +// one the hub index and the web UI follow — so a reference grcli // pulls is exactly one the hub indexed, slugified segments included. // // ok=false carries a human reason for the skip report; it is never an error —