diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 30399e8fa1..8e50421844 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -86,6 +86,7 @@ See `values.yaml` for the full set; the important keys: | `.resources` | see `values.yaml` | Requests and limits for each container, e.g. `ateApi.resources` and `atenetRouter.agentgateway.resources` | | `atenetRouter.requestParking.budget` | `60s` | Total budget for actor resolution and resume retries | | `atenetRouter.requestTimeout` | `60s` | Timeout for each Substrate ingress API call, including actor resolution | +| `atenetEgress.extraCredentialProviders` | `[]` | Credential providers you deploy, added after the bundled `k8s.io` provider, each `{name, address}`. `name` is the host of the `ate-secret://` URIs it serves; `address` is the `host:port` the egress gateway dials over mTLS. Its serving certificate must chain to the servicedns CA. See [egress credential injection](../../docs/guides/egress-credential-injection.md#the-plugin-api) | | `atenetEgress.agentgateway.logLevel` | `info` | Egress proxy `RUST_LOG` filter, e.g. `debug` or `info,agentgateway=debug` when troubleshooting | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs:///` with bundled RustFS | @@ -94,6 +95,7 @@ See `values.yaml` for the full set; the important keys: | `atelet.imageCredentialProviderConfig` | `""` | Host path to the kubelet credential provider config; set together with the bin directory | | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path | | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only | +| `credentialProvider.enabled` | `true` | Deploy the Kubernetes Secret provider and route `k8s.io` credential URIs to it. With no provider, every credential injection fails closed | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.actorJWTIssuer` | `https://..svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties | | `objectStorePlugin.extraEnv` | `[]` | Object storage credentials/settings shared by both object-store plugins | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 0b15b6bf0f..35d435d350 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -259,3 +259,45 @@ definition so the fallback cannot drift between pods. {{ toYaml . }} {{- end }} {{- end }} + +{{/* +The egress gateway's credentialProviders: the bundled k8s.io provider when +credentialProvider.enabled, then atenetEgress.extraCredentialProviders. Renders +nothing when there are none, so every injection fails closed. +*/}} +{{- define "substrate.credentialProviders" -}} +{{- $providers := list -}} +{{- if .Values.credentialProvider.enabled -}} +{{- $providers = append $providers (dict "name" "k8s.io" "address" (printf "%s.%s.svc:50051" (include "substrate.fullname" (list "k8s-credential-provider" .)) .Release.Namespace)) -}} +{{- end -}} +{{- $seen := dict -}} +{{- range .Values.atenetEgress.extraCredentialProviders -}} +{{- if not .name -}} +{{- fail "atenetEgress.extraCredentialProviders: each provider needs a name" -}} +{{- end -}} +{{- if eq .name "k8s.io" -}} +{{- fail "atenetEgress.extraCredentialProviders: k8s.io is the bundled provider; set credentialProvider.enabled instead" -}} +{{- end -}} +{{- if hasKey $seen .name -}} +{{- fail (printf "atenetEgress.extraCredentialProviders: duplicate name %q" .name) -}} +{{- end -}} +{{- if not (regexMatch "^[^:]+:[0-9]+$" (.address | default "")) -}} +{{- fail (printf "atenetEgress.extraCredentialProviders: %s needs an address as host:port" .name) -}} +{{- end -}} +{{- $_ := set $seen .name true -}} +{{- $providers = append $providers (dict "name" .name "address" .address) -}} +{{- end -}} +{{- if $providers -}} +credentialProviders: +{{- range $providers }} +- uriAuthority: {{ .name | quote }} + target: + host: {{ .address | quote }} + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- end }} +{{- end }} +{{- end -}} diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 52c1c6bc60..304b211184 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -80,15 +80,7 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - credentialProviders: - - uriAuthority: k8s.io - target: - host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- include "substrate.credentialProviders" . | nindent 14 }} - protocol: HTTP routes: - backends: @@ -102,15 +94,7 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - credentialProviders: - - uriAuthority: k8s.io - target: - host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 - policies: - backendTLS: - cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem - key: /run/podidentity.podcert.ate.dev/credential-bundle.pem - root: /run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- include "substrate.credentialProviders" . | nindent 14 }} - protocol: TLS tcpRoutes: - backends: diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml index 73b5bd2574..b2cd5290bb 100644 --- a/charts/substrate/templates/k8s-credential-provider.yaml +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.credentialProvider.enabled }} # The credential provider: a gRPC service that resolves ate-secret:// URIs # of the k8s.io class to Kubernetes Secret values. It is the ONLY @@ -197,3 +198,4 @@ spec: ports: - protocol: TCP port: 50051 +{{- end }} diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index a96f173555..788caa11ea 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -511,3 +511,68 @@ tests: content: name: RUST_LOG value: "info,agentgateway=debug" +- it: routes k8s.io credentials to the bundled provider on both listeners + template: atenet-egress.yaml + documentSelector: + path: kind + value: ConfigMap + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: '(?s)credentialProviders:\n\s+- uriAuthority: "?k8s.io"?\n\s+target:\n\s+host: "?custom-k8s-credential-provider.custom-system.svc:50051"?.*credentialProviders:\n\s+- uriAuthority: "?k8s.io"?' +- it: routes credentials only to extra providers when the bundled one is off + templates: + - atenet-egress.yaml + - k8s-credential-provider.yaml + set: + credentialProvider.enabled: false + atenetEgress.extraCredentialProviders: + - name: vault.example.com + address: vault-provider.vault.svc:50051 + asserts: + - hasDocuments: + count: 0 + template: k8s-credential-provider.yaml + - notMatchRegex: + path: data["config.yaml"] + pattern: 'k8s.io' + template: atenet-egress.yaml + documentSelector: + path: kind + value: ConfigMap + - matchRegex: + path: data["config.yaml"] + pattern: 'uriAuthority: "vault.example.com"\n\s+target:\n\s+host: "vault-provider.vault.svc:50051"' + template: atenet-egress.yaml + documentSelector: + path: kind + value: ConfigMap +- it: configures no credential providers when every provider is off + template: atenet-egress.yaml + set: + credentialProvider.enabled: false + documentSelector: + path: kind + value: ConfigMap + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: credentialProviders +- it: rejects an extra provider named k8s.io + template: atenet-egress.yaml + set: + atenetEgress.extraCredentialProviders: + - name: k8s.io + address: a.b.svc:50051 + asserts: + - failedTemplate: + errorMessage: "atenetEgress.extraCredentialProviders: k8s.io is the bundled provider; set credentialProvider.enabled instead" +- it: rejects an extra provider without a host:port address + template: atenet-egress.yaml + set: + atenetEgress.extraCredentialProviders: + - name: vault.example.com + address: vault-provider.vault.svc + asserts: + - failedTemplate: + errorMessage: "atenetEgress.extraCredentialProviders: vault.example.com needs an address as host:port" diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 4def6e0a61..e9638015b5 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -165,6 +165,14 @@ atenetRouter: memory: 1Gi atenetEgress: + # Credential providers you deploy, added after the bundled k8s.io provider + # when credentialProvider.enabled. name is the host of the ate-secret:// URIs + # the provider serves, and address is the host:port the gateway dials over + # mTLS. The provider's serving certificate must chain to the servicedns CA + # and name the address host. + extraCredentialProviders: [] + # - name: vault.example.com + # address: vault-provider.vault.svc:50051 agentgateway: logLevel: info resources: @@ -178,6 +186,8 @@ atenetEgress: # Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. # Includes get-only Secret RBAC. HTTPS requires egress-mitm-ca-pool and actor trust. credentialProvider: + # Deploy the Kubernetes Secret provider and route k8s.io credential URIs to it. + enabled: true namespacePolicies: [] # - atespace: team-a # allowedNamespaces: [team-a-secrets] diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go index 1d1a7f5b2a..2b21932eec 100644 --- a/cmd/kubectl-ate/internal/cmd/install.go +++ b/cmd/kubectl-ate/internal/cmd/install.go @@ -28,8 +28,8 @@ import ( "github.com/spf13/cobra" "golang.org/x/term" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/install" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/install/tui" + "go.ate.dev/substrate/pkg/kagent/install" + "go.ate.dev/substrate/pkg/kagent/install/tui" ) var ( @@ -37,14 +37,24 @@ var ( chartDir string noTUI bool upgradeDryRun bool - uninstallOpts install.Options + uninstallOpts install.UninstallOptions uninstallYes bool ) const configHelp = `--config takes a YAML file, or - for stdin: - postgres: bundled # or external - objectStore: rustfs # or s3, gcs + postgres: + type: bundled # or external + objectStore: + type: bundled # or s3, gcs + bucket: my-snapshots # for s3 or gcs + s3: # all optional + region: us-east-1 + endpoint: https://s3.example.com + pathStyle: true + credentialsSecret: s3-creds # in the system namespace + gcs: + transferServiceAccount: transfer@my-project.iam.gserviceaccount.com microvm: enabled: true assetBucket: my-assets # for s3 or gcs @@ -79,10 +89,9 @@ a RustFS for snapshots. On a terminal it asks for these choices first and prints the matching config. For anything but evaluation, create the database connection Secrets -described in charts/substrate/README.md and set postgres: external. To store -snapshots in a bucket, set objectStore to s3 or gcs and set the charts' -objectStore.*, ateApi.snapshotLocation and, for GCS, -objectStorePlugin.transferServiceAccount values. +described in charts/substrate/README.md and set postgres.type to external. +To store snapshots in a bucket, set objectStore.type to s3 or gcs and set +objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount. microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each node architecture, caches them under the user cache directory, uploads them to @@ -132,9 +141,14 @@ releases, and CRD changes that would strand stored objects. return err } defer closeTerminal(tty) - opts.DryRun, opts.Out = upgradeDryRun, cmd.OutOrStdout() - return runInstaller(cmd, opts, tty, "upgrade", (*install.Installer).Upgrade, - "Agent Substrate is upgraded.") + up := install.UpgradeOptions{DryRun: upgradeDryRun, Out: cmd.OutOrStdout()} + // The diff goes to stdout, so it can't share the terminal with the TUI. + if up.DryRun { + tty = nil + } + return runInstaller(cmd, opts, tty, "upgrade", func(in *install.Installer, ctx context.Context) error { + return in.Upgrade(ctx, up) + }, "Agent Substrate is upgraded.") }, } @@ -157,12 +171,13 @@ data.`, return err } defer closeTerminal(tty) - opts.IncludeCRDs, opts.Purge = uninstallOpts.IncludeCRDs, uninstallOpts.Purge + un := uninstallOpts if !uninstallYes { - opts.Confirm = confirmFunc(cmd, tty) + un.Confirm = confirmFunc(cmd, tty) } - return runInstaller(cmd, opts, tty, "uninstall", (*install.Installer).Uninstall, - "Agent Substrate is uninstalled.") + return runInstaller(cmd, opts, tty, "uninstall", func(in *install.Installer, ctx context.Context) error { + return in.Uninstall(ctx, un) + }, "Agent Substrate is uninstalled.") }, } @@ -170,7 +185,10 @@ data.`, // from. Without --config the config is install.DefaultConfig. The terminal is // nil when there is none. Close it with closeTerminal. func options(cmd *cobra.Command) (install.Options, *os.File, error) { - opts := install.Options{ChartDir: chartDir} + var opts install.Options + if chartDir != "" { + opts.Charts = os.DirFS(chartDir) + } var err error switch configPath { case "": @@ -282,7 +300,7 @@ func newInstallerWith(cmd *cobra.Command, opts install.Options, r install.Report // lines otherwise. The TUI shows the error itself, so the returned error is // marked as reported. func runInstaller(cmd *cobra.Command, opts install.Options, tty *os.File, title string, fn func(*install.Installer, context.Context) error, summary string) error { - if !useTUI(cmd, tty) || opts.DryRun { + if !useTUI(cmd, tty) { in, err := newInstaller(cmd, opts) if err != nil { return err diff --git a/docs/dev/releasing.md b/docs/dev/releasing.md index bf1b74d61a..d77f44d2a7 100644 --- a/docs/dev/releasing.md +++ b/docs/dev/releasing.md @@ -18,7 +18,7 @@ Releases should always be made from a release branch, named `release-X.Y`. 1. Tag the tested commit on the intended release branch and push the tag. Both tag pushes and manual runs of the [`release`](../../.github/workflows/release.yaml) workflow check out that tag: 1. The `images` job runs `ate-setup publish release-images`, which builds every release image for `linux/amd64` and `linux/arm64`, pushes it to `ghcr.io///:`, and then checks that every tag names the image just pushed, for both platforms. The ko images and `envoy-dataplane`, built from its Dockerfile with `docker buildx`, all cross-compile for arm64 without emulation. - 2. In `kagent-dev/substrate`, the `charts` job publishes all three Helm charts after the images succeed. See [Fork Helm charts](#fork-helm-charts). + 2. In `kagent-dev/substrate`, the `charts` job publishes all four Helm charts after the images succeed. See [Fork Helm charts](#fork-helm-charts). 3. Once artifact publishing succeeds, the `draft` job builds the `kubectl-ate` binaries with `ate-setup publish release-binaries`, then runs [`hack/release/draft-release.sh`](../../hack/release/draft-release.sh), which creates a draft release, and attaches the binaries to it. The notes cover every change since the previous `vX.Y.Z` release, say where the release images are published and link to the list below, give the command that installs them, give the steps to check and install `kubectl-ate`, and end with a list of committers. A release candidate is marked as a prerelease. If images, charts, or the binary build fails, no new draft is created. If attaching the binaries fails, the draft exists without them. Either way, fix the cause and run the workflow again from the Actions tab with the tag as input. @@ -112,7 +112,8 @@ VERSION=v0.3.0 go run ./cmd/ate-setup publish release-binaries ## Fork Helm charts -The fork publishes `substrate-crds`, `substrate`, and `substrate-podcert` to +The fork publishes `substrate-crds`, `substrate`, `substrate-podcert`, and +`substrate-atelet` to `oci://ghcr.io/kagent-dev/substrate/helm`. Their chart version is the release tag without `v`; their `appVersion` is the full tag, which selects the matching component images. All charts are packaged before any chart is pushed. A failed diff --git a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl index f2986910c3..7361c398a5 100644 --- a/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/ate-install/sandboxconfig-microvm.yaml.tmpl @@ -32,7 +32,7 @@ # (virtio-fs RO lower + guest-tmpfs upper), so it fetches virtiofsd too. # # cmd/ate-setup/internal/steps/microvm.go and -# cmd/kubectl-ate/internal/microvmassets read the per-arch sha256 values +# pkg/kagent/microvmassets read the per-arch sha256 values # below to verify the assembled binaries and stages them at these digest-keyed # paths so newer releases do not overwrite kept versions. diff --git a/cmd/kubectl-ate/internal/install/bootstrap.go b/pkg/kagent/install/bootstrap.go similarity index 90% rename from cmd/kubectl-ate/internal/install/bootstrap.go rename to pkg/kagent/install/bootstrap.go index ae86dd50ce..f4cb5e810a 100644 --- a/cmd/kubectl-ate/internal/install/bootstrap.go +++ b/pkg/kagent/install/bootstrap.go @@ -345,17 +345,22 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) ( if err := ensureNamespace(ctx, in.kube, SystemNamespace); err != nil { return "", err } - if in.opts.ObjectStore != ObjectStoreRustFS { - // A resumed install keeps the values the release already has. - vals := in.user[substrateRelease.name] - rel, err := in.helm.get(substrateRelease) - if err != nil { - return "", err - } - if rel != nil { - vals = mergeValues(rel.Config, in.user[substrateRelease.name]) + if in.opts.ObjectStore.Type != ObjectStoreBundled { + // A resumed install keeps the values the releases already have. + atelet, substrate := in.externalStoreValues() + vals := map[release]map[string]any{} + for r, typed := range map[release]map[string]any{ateletRelease: atelet, substrateRelease: substrate} { + rel, err := in.helm.get(r) + if err != nil { + return "", err + } + var stored map[string]any + if rel != nil { + stored = rel.Config + } + vals[r] = mergeValues(stored, in.user[r.name], typed) } - return in.checkExternalObjectStore(ctx, vals) + return in.checkExternalObjectStore(ctx, vals[substrateRelease], vals[ateletRelease]) } if ok, err := secretExists(ctx, in.kube, SystemNamespace, rustfsCredentialsSecret); ok || err != nil { return "already present", err @@ -377,18 +382,19 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) ( } // checkExternalObjectStore fails before any release changes when a value the -// charts need for an external store is missing from vals. -func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[string]any) (string, error) { - location, _ := lookupString(vals, "ateApi", "snapshotLocation") - if location == "" && !previewEnabled(vals, "PerTemplateSnapshotLocation") { - return "", errors.New("set ateApi.snapshotLocation to the bucket URI for snapshots, such as gs://my-bucket/") - } - if in.opts.ObjectStore == ObjectStoreGCS { - if sa, _ := lookupString(vals, "objectStorePlugin", "transferServiceAccount"); sa == "" { - return "", errors.New("set objectStorePlugin.transferServiceAccount to the Google service account for snapshot transfers") +// charts need for an external store is missing from the substrate or atelet +// release values. +func (in *Installer) checkExternalObjectStore(ctx context.Context, substrate, atelet map[string]any) (string, error) { + location, _ := lookupString(substrate, "ateApi", "snapshotLocation") + if location == "" && !previewEnabled(substrate, "PerTemplateSnapshotLocation") { + return "", errors.New("set objectStore.bucket to the snapshot bucket") + } + if in.opts.ObjectStore.Type == ObjectStoreGCS { + if sa, _ := lookupString(substrate, "objectStorePlugin", "transferServiceAccount"); sa == "" { + return "", errors.New("set objectStore.gcs.transferServiceAccount to the Google service account for snapshot transfers") } } - if name, _ := lookupString(vals, "objectStore", "credentialsSecret"); name != "" { + if name, _ := lookupString(atelet, "objectStore", "credentialsSecret"); name != "" { ok, err := secretExists(ctx, in.kube, SystemNamespace, name) if err != nil { return "", err @@ -398,9 +404,9 @@ func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[stri } } if location == "" { - return string(in.opts.ObjectStore) + ", per-template locations", nil + return string(in.opts.ObjectStore.Type) + ", per-template locations", nil } - return string(in.opts.ObjectStore) + " " + location, nil + return string(in.opts.ObjectStore.Type) + " " + location, nil } // lookup returns the value at path in vals, or nil. diff --git a/cmd/kubectl-ate/internal/install/config.go b/pkg/kagent/install/config.go similarity index 50% rename from cmd/kubectl-ate/internal/install/config.go rename to pkg/kagent/install/config.go index 2dcfba4013..d45387c29f 100644 --- a/cmd/kubectl-ate/internal/install/config.go +++ b/pkg/kagent/install/config.go @@ -16,10 +16,12 @@ package install import ( "bytes" + "encoding/json" "errors" "fmt" "io" "path/filepath" + "strings" "time" "helm.sh/helm/v4/pkg/cli/values" @@ -30,9 +32,9 @@ import ( // Config describes an installation. Install, upgrade and uninstall read it // from the file passed to --config. type Config struct { - Postgres PostgresMode `json:"postgres,omitempty"` - ObjectStore ObjectStore `json:"objectStore,omitempty"` - MicroVM MicroVM `json:"microvm,omitzero"` + Postgres Postgres `json:"postgres,omitzero"` + ObjectStore ObjectStore `json:"objectStore,omitzero"` + MicroVM MicroVM `json:"microvm,omitzero"` // Timeout bounds each wait for workloads to become ready. Timeout metav1.Duration `json:"timeout,omitzero"` // Releases holds Helm values for the releases. @@ -55,6 +57,111 @@ type ReleaseValues struct { Values map[string]any `json:"values,omitempty"` } +// Postgres configures the control plane's database. External uses the +// connection Secrets described in charts/substrate/README.md. +type Postgres struct { + Type PostgresType `json:"type,omitempty"` +} + +// UnmarshalJSON also accepts the type alone, as in postgres: external. +func (p *Postgres) UnmarshalJSON(b []byte) error { + var t PostgresType + if json.Unmarshal(b, &t) == nil { + *p = Postgres{Type: t} + return nil + } + type plain Postgres + return decodeStrict(b, (*plain)(p)) +} + +// ObjectStore configures snapshot storage. The s3 and gcs settings fill the +// charts' storage values on install, over the values in Releases. +type ObjectStore struct { + Type ObjectStoreType `json:"type,omitempty"` + // Bucket holds snapshots with an s3 or gcs store. It sets + // ateApi.snapshotLocation to gs:///, the URI both backends use. + // Leave it empty to set snapshot locations per template with the + // PerTemplateSnapshotLocation preview gate. + Bucket string `json:"bucket,omitempty"` + S3 S3ObjectStore `json:"s3,omitzero"` + GCS GCSObjectStore `json:"gcs,omitzero"` +} + +// S3ObjectStore configures an s3 store. Empty fields use the AWS SDK defaults. +type S3ObjectStore struct { + Region string `json:"region,omitempty"` + // Endpoint is the URL of an S3-compatible service. + Endpoint string `json:"endpoint,omitempty"` + PathStyle bool `json:"pathStyle,omitempty"` + // CredentialsSecret names a Secret in SystemNamespace with + // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. Empty uses the + // pods' ambient credentials. + CredentialsSecret string `json:"credentialsSecret,omitempty"` +} + +// GCSObjectStore configures a gcs store. +type GCSObjectStore struct { + // TransferServiceAccount is the Google service account the snapshot + // exchange impersonates for actor-scoped transfers. + TransferServiceAccount string `json:"transferServiceAccount,omitempty"` +} + +// UnmarshalJSON also accepts the type alone, as in objectStore: s3. +func (o *ObjectStore) UnmarshalJSON(b []byte) error { + var t ObjectStoreType + if json.Unmarshal(b, &t) == nil { + *o = ObjectStore{Type: t} + return nil + } + type plain ObjectStore + return decodeStrict(b, (*plain)(o)) +} + +// UnmarshalJSON reads rustfs, the v1.0.0 name, as bundled. +func (t *ObjectStoreType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + if s == "rustfs" { + s = string(ObjectStoreBundled) + } + *t = ObjectStoreType(s) + return nil +} + +// decodeStrict decodes b into v, keeping the fields b doesn't set. +func decodeStrict(b []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + return dec.Decode(v) +} + +func (o ObjectStore) validate() error { + switch o.Type { + case ObjectStoreBundled: + if o != (ObjectStore{Type: ObjectStoreBundled}) { + return errors.New("objectStore bundled takes no other settings") + } + case ObjectStoreS3, ObjectStoreGCS: + default: + return fmt.Errorf("invalid objectStore.type %q: must be %s, %s or %s", o.Type, ObjectStoreBundled, ObjectStoreS3, ObjectStoreGCS) + } + if strings.ContainsAny(o.Bucket, ":/") { + return fmt.Errorf("objectStore.bucket %q must be a bucket name, not a URI", o.Bucket) + } + if o.S3 != (S3ObjectStore{}) && o.Type != ObjectStoreS3 { + return errors.New("objectStore.s3 needs type s3") + } + if o.GCS != (GCSObjectStore{}) && o.Type != ObjectStoreGCS { + return errors.New("objectStore.gcs needs type gcs") + } + if e := o.S3.Endpoint; e != "" && !strings.HasPrefix(e, "http://") && !strings.HasPrefix(e, "https://") { + return fmt.Errorf("objectStore.s3.endpoint %q must be an http or https URL", e) + } + return nil +} + // MicroVM configures the microvm sandbox class. type MicroVM struct { // Enabled stages the microVM guest assets and applies the microvm @@ -68,8 +175,8 @@ type MicroVM struct { // DefaultConfig is an evaluation install: bundled PostgreSQL and RustFS. func DefaultConfig() Config { return Config{ - Postgres: PostgresBundled, - ObjectStore: ObjectStoreRustFS, + Postgres: Postgres{Type: PostgresBundled}, + ObjectStore: ObjectStore{Type: ObjectStoreBundled}, Timeout: metav1.Duration{Duration: 10 * time.Minute}, } } @@ -99,17 +206,15 @@ func ReadConfig(r io.Reader, dir string) (Config, error) { } func (c Config) validate() error { - switch c.Postgres { + switch c.Postgres.Type { case PostgresBundled, PostgresExternal: default: - return fmt.Errorf("invalid postgres %q: must be %s or %s", c.Postgres, PostgresBundled, PostgresExternal) + return fmt.Errorf("invalid postgres.type %q: must be %s or %s", c.Postgres.Type, PostgresBundled, PostgresExternal) } - switch c.ObjectStore { - case ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS: - default: - return fmt.Errorf("invalid objectStore %q: must be %s, %s or %s", c.ObjectStore, ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS) + if err := c.ObjectStore.validate(); err != nil { + return err } - if c.MicroVM.Enabled && c.ObjectStore != ObjectStoreRustFS && c.MicroVM.AssetBucket == "" { + if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" { return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket") } if c.Timeout.Duration <= 0 { diff --git a/cmd/kubectl-ate/internal/install/config_test.go b/pkg/kagent/install/config_test.go similarity index 67% rename from cmd/kubectl-ate/internal/install/config_test.go rename to pkg/kagent/install/config_test.go index be58c28190..d3094e3fab 100644 --- a/cmd/kubectl-ate/internal/install/config_test.go +++ b/pkg/kagent/install/config_test.go @@ -41,7 +41,10 @@ func TestReadConfig(t *testing.T) { { name: "full", in: `postgres: external -objectStore: gcs +objectStore: + type: gcs + bucket: snapshots + gcs: {transferServiceAccount: sa@p.iam.gserviceaccount.com} microvm: {enabled: true, assetBucket: assets} timeout: 5m releases: @@ -49,7 +52,12 @@ releases: substrate: {valuesFiles: [db.yaml]} `, want: withDefaults(func(c *Config) { - c.Postgres, c.ObjectStore = PostgresExternal, ObjectStoreGCS + c.Postgres = Postgres{Type: PostgresExternal} + c.ObjectStore = ObjectStore{ + Type: ObjectStoreGCS, + Bucket: "snapshots", + GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}, + } c.MicroVM = MicroVM{Enabled: true, AssetBucket: "assets"} c.Timeout = metav1.Duration{Duration: 5 * time.Minute} c.Releases.All = ReleaseValues{ @@ -63,7 +71,34 @@ releases: {name: "unknown field", in: "postgress: external\n", wantErr: "postgress"}, {name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"}, {name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"}, + { + name: "object store as a string", + in: "objectStore: s3\n", + want: withDefaults(func(c *Config) { c.ObjectStore = ObjectStore{Type: ObjectStoreS3} }), + }, + { + name: "s3 object store", + in: "objectStore: {type: s3, bucket: b, s3: {region: us-east-1, endpoint: 'http://minio:9000', pathStyle: true, credentialsSecret: creds}}\n", + want: withDefaults(func(c *Config) { + c.ObjectStore = ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{ + Region: "us-east-1", Endpoint: "http://minio:9000", PathStyle: true, CredentialsSecret: "creds", + }} + }), + }, + { + name: "postgres as a struct", + in: "postgres: {type: external}\n", + want: withDefaults(func(c *Config) { c.Postgres = Postgres{Type: PostgresExternal} }), + }, + {name: "rustfs alias", in: "objectStore: rustfs\n", want: DefaultConfig()}, + {name: "rustfs alias in a struct", in: "objectStore: {type: rustfs}\n", want: DefaultConfig()}, {name: "bad object store", in: "objectStore: nfs\n", wantErr: "invalid objectStore"}, + {name: "unknown postgres field", in: "postgres: {type: external, host: db}\n", wantErr: "host"}, + {name: "bundled with bucket", in: "objectStore: {bucket: b}\n", wantErr: "objectStore"}, + {name: "bucket with path", in: "objectStore: {type: s3, bucket: b/dir}\n", wantErr: "objectStore.bucket"}, + {name: "gcs block on s3", in: "objectStore: {type: s3, gcs: {transferServiceAccount: sa}}\n", wantErr: "objectStore.gcs"}, + {name: "s3 endpoint without scheme", in: "objectStore: {type: s3, s3: {endpoint: minio:9000}}\n", wantErr: "objectStore.s3.endpoint"}, + {name: "unknown object store field", in: "objectStore: {type: s3, buckets: b}\n", wantErr: "buckets"}, {name: "microvm needs bucket", in: "objectStore: s3\nmicrovm: {enabled: true}\n", wantErr: "microvm.assetBucket"}, {name: "bad timeout", in: "timeout: 0s\n", wantErr: "timeout"}, } diff --git a/cmd/kubectl-ate/internal/install/helm.go b/pkg/kagent/install/helm.go similarity index 100% rename from cmd/kubectl-ate/internal/install/helm.go rename to pkg/kagent/install/helm.go diff --git a/cmd/kubectl-ate/internal/install/install_test.go b/pkg/kagent/install/install_test.go similarity index 62% rename from cmd/kubectl-ate/internal/install/install_test.go rename to pkg/kagent/install/install_test.go index 5823b8f3fe..245cca59f8 100644 --- a/cmd/kubectl-ate/internal/install/install_test.go +++ b/pkg/kagent/install/install_test.go @@ -17,7 +17,9 @@ package install import ( "bytes" "context" + "io" "reflect" + "slices" "strings" "testing" @@ -120,7 +122,7 @@ func TestLoadEmbeddedCharts(t *testing.T) { func TestBootstrapKeepsExistingSecrets(t *testing.T) { ctx := context.Background() kc := fake.NewClientset() - in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStoreRustFS}}} + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreBundled}}}} for _, s := range []func(context.Context, func(string)) (string, error){ in.createPodCertCAs, in.createIdentitySecrets, in.configureObjectStore, @@ -203,32 +205,49 @@ func TestApplyPostgresManifest(t *testing.T) { } func TestCheckExternalObjectStore(t *testing.T) { - gcs := map[string]any{ - "ateApi": map[string]any{"snapshotLocation": "gs://b/"}, - "objectStorePlugin": map[string]any{"transferServiceAccount": "sa@p.iam.gserviceaccount.com"}, - } + s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"} + gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}} tests := []struct { name string store ObjectStore - user map[string]any + user map[release]map[string]any + stored map[release]map[string]any secrets []string wantErr string }{ - {name: "s3", store: ObjectStoreS3, user: map[string]any{"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, - {name: "per-template locations", store: ObjectStoreS3, user: map[string]any{"preview": []any{"PerTemplateSnapshotLocation"}}}, - {name: "no location", store: ObjectStoreS3, wantErr: "ateApi.snapshotLocation"}, - {name: "gcs", store: ObjectStoreGCS, user: gcs}, - {name: "gcs without service account", store: ObjectStoreGCS, user: map[string]any{"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}, wantErr: "transferServiceAccount"}, + {name: "s3", store: s3}, + {name: "gcs", store: gcs}, + { + name: "location from chart values", + store: ObjectStore{Type: ObjectStoreS3}, + user: map[release]map[string]any{substrateRelease: {"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, + }, + { + name: "location from the release", + store: ObjectStore{Type: ObjectStoreS3}, + stored: map[release]map[string]any{substrateRelease: {"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, + }, + { + name: "per-template locations", + store: ObjectStore{Type: ObjectStoreS3}, + user: map[release]map[string]any{substrateRelease: {"preview": []any{"PerTemplateSnapshotLocation"}}}, + }, + {name: "no location", store: ObjectStore{Type: ObjectStoreS3}, wantErr: "objectStore.bucket"}, + {name: "gcs without service account", store: ObjectStore{Type: ObjectStoreGCS, Bucket: "b"}, wantErr: "objectStore.gcs.transferServiceAccount"}, { name: "credentials secret present", - store: ObjectStoreS3, - user: map[string]any{"preview": []any{"ALL"}, "objectStore": map[string]any{"credentialsSecret": "s3-creds"}}, + store: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{CredentialsSecret: "s3-creds"}}, secrets: []string{"s3-creds"}, }, { name: "credentials secret missing", - store: ObjectStoreS3, - user: map[string]any{"preview": []any{"ALL"}, "objectStore": map[string]any{"credentialsSecret": "s3-creds"}}, + store: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{CredentialsSecret: "s3-creds"}}, + wantErr: "doesn't exist", + }, + { + name: "credentials secret from chart values", + store: s3, + user: map[release]map[string]any{ateletRelease: {"objectStore": map[string]any{"credentialsSecret": "s3-creds"}}}, wantErr: "doesn't exist", }, } @@ -240,8 +259,14 @@ func TestCheckExternalObjectStore(t *testing.T) { t.Fatal(err) } } - in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: tc.store}}} - _, err := in.checkExternalObjectStore(context.Background(), tc.user) + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: tc.store}}, user: map[string]map[string]any{}} + for r, v := range tc.user { + in.user[r.name] = v + } + atelet, substrate := in.externalStoreValues() + _, err := in.checkExternalObjectStore(context.Background(), + mergeValues(tc.stored[substrateRelease], tc.user[substrateRelease], substrate), + mergeValues(tc.stored[ateletRelease], tc.user[ateletRelease], atelet)) if tc.wantErr == "" { if err != nil { t.Fatalf("checkExternalObjectStore() = %v", err) @@ -256,7 +281,7 @@ func TestCheckExternalObjectStore(t *testing.T) { } func TestExternalObjectStoreValues(t *testing.T) { - in := &Installer{opts: Options{Config: Config{ObjectStore: ObjectStoreGCS, Postgres: PostgresExternal}}} + in := &Installer{opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreGCS}, Postgres: Postgres{Type: PostgresExternal}}}} defaults, forced, err := in.ateletValues(context.Background()) if err != nil { t.Fatal(err) @@ -275,3 +300,64 @@ func TestExternalObjectStoreValues(t *testing.T) { t.Errorf("substrate rustfs.enabled = %v, want false", got) } } + +func TestExternalStoreValuesS3(t *testing.T) { + userEnv := map[string]any{"name": "HTTPS_PROXY", "value": "http://proxy:3128"} + in := &Installer{ + opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{ + Region: "us-east-1", Endpoint: "http://minio:9000", PathStyle: true, CredentialsSecret: "creds", + }}}}, + user: map[string]map[string]any{substrateRelease.name: {"objectStorePlugin": map[string]any{"extraEnv": []any{userEnv}}}}, + } + atelet, substrate := in.externalStoreValues() + wantAtelet := map[string]any{"objectStore": map[string]any{ + "backend": "s3", "region": "us-east-1", "endpoint": "http://minio:9000", "pathStyle": true, "credentialsSecret": "creds", + }} + if !reflect.DeepEqual(atelet, wantAtelet) { + t.Errorf("atelet values = %v, want %v", atelet, wantAtelet) + } + secret := func(key string) map[string]any { + return map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": "creds", "key": key}}} + } + wantSubstrate := map[string]any{ + "rustfs": map[string]any{"enabled": false}, + "atelet": map[string]any{"storageBackend": "s3"}, + "ateApi": map[string]any{"snapshotLocation": "gs://b/"}, + "objectStorePlugin": map[string]any{"extraEnv": []any{ + userEnv, + map[string]any{"name": "AWS_REGION", "value": "us-east-1"}, + map[string]any{"name": "AWS_ENDPOINT_URL", "value": "http://minio:9000"}, + map[string]any{"name": "AWS_S3_USE_PATH_STYLE", "value": "true"}, + secret("AWS_ACCESS_KEY_ID"), + secret("AWS_SECRET_ACCESS_KEY"), + }}, + } + if !reflect.DeepEqual(substrate, wantSubstrate) { + t.Errorf("substrate values = %v, want %v", substrate, wantSubstrate) + } +} + +func TestInstallStepsDatabase(t *testing.T) { + tests := []struct { + postgres PostgresType + wantCheck bool + }{ + {PostgresBundled, false}, + {PostgresExternal, true}, + } + for _, tc := range tests { + t.Run(string(tc.postgres), func(t *testing.T) { + cfg := DefaultConfig() + cfg.Postgres.Type = tc.postgres + in := &Installer{opts: Options{Config: cfg}, report: PlainReporter{W: io.Discard}} + steps, err := in.InstallSteps() + if err != nil { + t.Fatal(err) + } + got := slices.ContainsFunc(steps, func(s Step) bool { return s.Name == StepCheckDatabase }) + if got != tc.wantCheck { + t.Errorf("has %q = %t, want %t", StepCheckDatabase, got, tc.wantCheck) + } + }) + } +} diff --git a/cmd/kubectl-ate/internal/install/installer.go b/pkg/kagent/install/installer.go similarity index 61% rename from cmd/kubectl-ate/internal/install/installer.go rename to pkg/kagent/install/installer.go index a2c8353fd9..45ec7df55c 100644 --- a/cmd/kubectl-ate/internal/install/installer.go +++ b/pkg/kagent/install/installer.go @@ -20,7 +20,6 @@ import ( "fmt" "io" "io/fs" - "os" "time" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -40,27 +39,27 @@ const ( PodCertNamespace = "podcertificate-controller-system" ) -// PostgresMode selects where the control plane's database comes from. -type PostgresMode string +// PostgresType selects where the control plane's database comes from. +type PostgresType string const ( // PostgresBundled applies manifests/ate-install/postgres for evaluation. // It has no backups or high availability. - PostgresBundled PostgresMode = "bundled" + PostgresBundled PostgresType = "bundled" // PostgresExternal uses connection Secrets the user created. - PostgresExternal PostgresMode = "external" + PostgresExternal PostgresType = "external" ) -// ObjectStore selects where snapshots are stored. -type ObjectStore string +// ObjectStoreType selects where snapshots are stored. +type ObjectStoreType string const ( - // ObjectStoreRustFS runs RustFS in the cluster with the substrate release. - ObjectStoreRustFS ObjectStore = "rustfs" - // ObjectStoreS3 and ObjectStoreGCS use a bucket the user configures with - // the charts' objectStore, objectStorePlugin and ateApi values. - ObjectStoreS3 ObjectStore = "s3" - ObjectStoreGCS ObjectStore = "gcs" + // ObjectStoreBundled runs RustFS in the cluster with the substrate + // release, for evaluation. + ObjectStoreBundled ObjectStoreType = "bundled" + // ObjectStoreS3 and ObjectStoreGCS use a bucket the user created. + ObjectStoreS3 ObjectStoreType = "s3" + ObjectStoreGCS ObjectStoreType = "gcs" ) // Options configures an Installer. @@ -68,21 +67,10 @@ type Options struct { Config Kubeconfig string Context string - // ChartDir loads charts from a directory instead of the embedded copies. - ChartDir string + // Charts holds a directory per chart. Nil uses the embedded charts. + Charts fs.FS // HelmLog receives Helm's debug log. Nil discards it. HelmLog io.Writer - // DryRun makes Upgrade write manifest diffs to Out instead of applying. - DryRun bool - Out io.Writer - // IncludeCRDs makes Uninstall delete the CRDs and every custom resource. - IncludeCRDs bool - // Purge makes Uninstall delete the namespaces, with their CA keys, - // secrets and database volume, and the published trust bundles. - Purge bool - // Confirm is shown what Uninstall will delete and returns whether to go - // on. Nil means go on. - Confirm func(ctx context.Context, summary string) (bool, error) } // Installer runs install steps against one cluster. @@ -119,9 +107,9 @@ func New(opts Options, r Reporter) (*Installer, error) { if err != nil { return nil, fmt.Errorf("while creating Kubernetes client: %w", err) } - charts := fs.FS(chartsfs.FS) - if opts.ChartDir != "" { - charts = os.DirFS(opts.ChartDir) + charts := opts.Charts + if charts == nil { + charts = chartsfs.FS } user, err := opts.userValues() if err != nil { @@ -140,46 +128,61 @@ func New(opts Options, r Reporter) (*Installer, error) { }, nil } -// step is one unit of work. run returns a short summary for the report. -type step struct { - name string - run func(ctx context.Context, progress func(string)) (string, error) +// Step is one unit of work. Run returns a short summary for the report. +type Step struct { + Name string + Run func(ctx context.Context, progress func(string)) (string, error) } -func (in *Installer) run(ctx context.Context, steps []step) error { +// StepCheckDatabase is the name of the install step that checks the +// connection Secrets for an external database. A caller that sets up the +// database itself inserts its steps before this one. +const StepCheckDatabase = "Check database secrets" + +// Run runs steps in order and reports each one. It stops at the first error. +func (in *Installer) Run(ctx context.Context, steps []Step) error { names := make([]string, len(steps)) for i, s := range steps { - names[i] = s.name + names[i] = s.Name } in.report.Report(Event{Kind: Plan, Steps: names}) for _, s := range steps { - in.report.Report(Event{Kind: StepStarted, Step: s.name}) + in.report.Report(Event{Kind: StepStarted, Step: s.Name}) start := time.Now() progress := func(msg string) { - in.report.Report(Event{Kind: StepProgress, Step: s.name, Msg: msg}) + in.report.Report(Event{Kind: StepProgress, Step: s.Name, Msg: msg}) } - msg, err := s.run(ctx, progress) + msg, err := s.Run(ctx, progress) elapsed := time.Since(start) if err != nil { - in.report.Report(Event{Kind: StepFailed, Step: s.name, Err: err, Elapsed: elapsed}) - return fmt.Errorf("%s: %w", s.name, err) + in.report.Report(Event{Kind: StepFailed, Step: s.Name, Err: err, Elapsed: elapsed}) + return fmt.Errorf("%s: %w", s.Name, err) } - in.report.Report(Event{Kind: StepDone, Step: s.name, Msg: msg, Elapsed: elapsed}) + in.report.Report(Event{Kind: StepDone, Step: s.Name, Msg: msg, Elapsed: elapsed}) } return nil } -// Install installs every release in order. Rerunning it is safe and resumes a -// failed install: existing secrets are kept and existing releases are -// upgraded in place when their chart version matches. +// Install runs InstallSteps. Rerunning it is safe and resumes a failed +// install: existing secrets are kept and existing releases are upgraded in +// place when their chart version matches. func (in *Installer) Install(ctx context.Context) error { - if err := in.opts.validate(); err != nil { + steps, err := in.InstallSteps() + if err != nil { return err } - if in.opts.Postgres == PostgresBundled { - in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres: external with a managed database for anything else."}) + return in.Run(ctx, steps) +} + +// InstallSteps returns the steps that install every release in order. +func (in *Installer) InstallSteps() ([]Step, error) { + if err := in.opts.validate(); err != nil { + return nil, err + } + if in.opts.Postgres.Type == PostgresBundled { + in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres.type to external with a managed database for anything else."}) } - steps := []step{ + steps := []Step{ {"Check cluster", in.preflight}, {"Install CRDs", in.helmStep(crdsRelease, noValues)}, {"Create pod certificate CAs", in.createPodCertCAs}, @@ -189,21 +192,21 @@ func (in *Installer) Install(ctx context.Context) error { {"Configure object storage", in.configureObjectStore}, {"Install atelet", in.helmStep(ateletRelease, in.ateletValues)}, } - if in.opts.Postgres == PostgresBundled { + if in.opts.Postgres.Type == PostgresBundled { steps = append(steps, - step{"Install PostgreSQL", in.applyPostgres}, - step{"Set up database", in.setupDatabase}, + Step{"Install PostgreSQL", in.applyPostgres}, + Step{"Set up database", in.setupDatabase}, ) } else { - steps = append(steps, step{"Check database secrets", in.checkDatabaseSecrets}) + steps = append(steps, Step{StepCheckDatabase, in.checkDatabaseSecrets}) } - steps = append(steps, step{"Install Substrate", in.helmStep(substrateRelease, in.substrateValues)}) + steps = append(steps, Step{"Install Substrate", in.helmStep(substrateRelease, in.substrateValues)}) if in.opts.MicroVM.Enabled { steps = append(steps, - step{"Stage microVM assets", in.stageMicroVMAssets}, - step{"Apply microVM SandboxConfig", in.applyMicroVMSandboxConfig}, + Step{"Stage microVM assets", in.stageMicroVMAssets}, + Step{"Apply microVM SandboxConfig", in.applyMicroVMSandboxConfig}, ) } - steps = append(steps, step{"Check API", in.checkAPI}) - return in.run(ctx, steps) + steps = append(steps, Step{"Check API", in.checkAPI}) + return steps, nil } diff --git a/cmd/kubectl-ate/internal/install/microvm.go b/pkg/kagent/install/microvm.go similarity index 98% rename from cmd/kubectl-ate/internal/install/microvm.go rename to pkg/kagent/install/microvm.go index 5784d73f85..b4d13dc24c 100644 --- a/cmd/kubectl-ate/internal/install/microvm.go +++ b/pkg/kagent/install/microvm.go @@ -37,9 +37,9 @@ import ( "sigs.k8s.io/yaml" atev1 "go.ate.dev/substrate/api/infra/k8s/ate/v1" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/microvmassets" "go.ate.dev/substrate/internal/portforward" ateinstall "go.ate.dev/substrate/manifests/ate-install" + "go.ate.dev/substrate/pkg/kagent/microvmassets" "go.ate.dev/substrate/pkg/objectstorage" ) @@ -54,7 +54,7 @@ const ( // assetTarget is where the microVM assets are staged. endpoint, region and // pathStyle come from atelet's objectStore values for an S3 store. type assetTarget struct { - store ObjectStore + store ObjectStoreType scheme string bucket string endpoint string @@ -75,7 +75,7 @@ func (in *Installer) microVMTarget(ctx context.Context) (assetTarget, error) { if bucket == "" { return assetTarget{}, errors.New("substrate release has no rustfs.bucket") } - return assetTarget{store: ObjectStoreRustFS, scheme: "s3", bucket: bucket}, nil + return assetTarget{store: ObjectStoreBundled, scheme: "s3", bucket: bucket}, nil } atelet, err := in.installedValues(ateletRelease) if err != nil { @@ -295,7 +295,7 @@ func (b gcsBucket) put(ctx context.Context, object string, r io.Reader) error { // uses the caller's own cloud credentials. func (in *Installer) assetStore(ctx context.Context, t assetTarget) (assetBucket, func(), error) { switch t.store { - case ObjectStoreRustFS: + case ObjectStoreBundled: access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err diff --git a/cmd/kubectl-ate/internal/install/postgres.go b/pkg/kagent/install/postgres.go similarity index 100% rename from cmd/kubectl-ate/internal/install/postgres.go rename to pkg/kagent/install/postgres.go diff --git a/cmd/kubectl-ate/internal/install/report.go b/pkg/kagent/install/report.go similarity index 100% rename from cmd/kubectl-ate/internal/install/report.go rename to pkg/kagent/install/report.go diff --git a/cmd/kubectl-ate/internal/install/status.go b/pkg/kagent/install/status.go similarity index 100% rename from cmd/kubectl-ate/internal/install/status.go rename to pkg/kagent/install/status.go diff --git a/cmd/kubectl-ate/internal/install/steps.go b/pkg/kagent/install/steps.go similarity index 80% rename from cmd/kubectl-ate/internal/install/steps.go rename to pkg/kagent/install/steps.go index b4de63fd78..481255ef75 100644 --- a/cmd/kubectl-ate/internal/install/steps.go +++ b/pkg/kagent/install/steps.go @@ -166,9 +166,9 @@ func (in *Installer) ateletValues(context.Context) (map[string]any, map[string]a // The release keeps its storage values. return nil, forced, nil } - if in.opts.ObjectStore != ObjectStoreRustFS { - forced["objectStore"] = map[string]any{"backend": string(in.opts.ObjectStore)} - return nil, forced, nil + if in.opts.ObjectStore.Type != ObjectStoreBundled { + atelet, _ := in.externalStoreValues() + return nil, mergeValues(forced, atelet), nil } defaults := map[string]any{ "objectStore": map[string]any{ @@ -191,21 +191,79 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s // The release keeps its storage and database values. return nil, forced, nil } - if in.opts.ObjectStore == ObjectStoreRustFS { + if in.opts.ObjectStore.Type == ObjectStoreBundled { access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err } forced["rustfs"] = map[string]any{"accessKey": access, "secretKey": secret} } else { - forced["rustfs"] = map[string]any{"enabled": false} + _, substrate := in.externalStoreValues() + forced = mergeValues(forced, substrate) } - if in.opts.Postgres == PostgresBundled { + if in.opts.Postgres.Type == PostgresBundled { forced["postgres"] = map[string]any{"clientCertificates": map[string]any{"enabled": true}} } return nil, forced, nil } +// externalStoreValues returns the values Config.ObjectStore sets on the +// substrate-atelet and substrate releases for an s3 or gcs store. The +// substrate chart has no objectStore values, so its control-plane plugin gets +// the S3 settings as extra environment variables. +func (in *Installer) externalStoreValues() (atelet, substrate map[string]any) { + o := in.opts.ObjectStore + store := map[string]any{"backend": string(o.Type)} + plugin := map[string]any{} + substrate = map[string]any{ + "rustfs": map[string]any{"enabled": false}, + "atelet": map[string]any{"storageBackend": string(o.Type)}, + "objectStorePlugin": plugin, + } + if o.Bucket != "" { + substrate["ateApi"] = map[string]any{"snapshotLocation": "gs://" + o.Bucket + "/"} + } + if sa := o.GCS.TransferServiceAccount; sa != "" { + plugin["transferServiceAccount"] = sa + } + + var env []any + addEnv := func(name string, value any) { + e := map[string]any{"name": name} + if s, ok := value.(string); ok { + e["value"] = s + } else { + e["valueFrom"] = value + } + env = append(env, e) + } + s3 := o.S3 + if s3.Region != "" { + store["region"] = s3.Region + addEnv("AWS_REGION", s3.Region) + } + if s3.Endpoint != "" { + store["endpoint"] = s3.Endpoint + addEnv("AWS_ENDPOINT_URL", s3.Endpoint) + } + if s3.PathStyle { + store["pathStyle"] = true + addEnv("AWS_S3_USE_PATH_STYLE", "true") + } + if name := s3.CredentialsSecret; name != "" { + store["credentialsSecret"] = name + for _, key := range []string{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} { + addEnv(key, map[string]any{"secretKeyRef": map[string]any{"name": name, "key": key}}) + } + } + if env != nil { + // Lists replace in a merge, so keep the user's entries. + user, _ := lookup(in.user[substrateRelease.name], "objectStorePlugin", "extraEnv").([]any) + plugin["extraEnv"] = append(append([]any{}, user...), env...) + } + return map[string]any{"objectStore": store}, substrate +} + func (in *Installer) waitTrustBundles(ctx context.Context, progress func(string)) (string, error) { ctbs, err := clustertrustbundle.NewClient(in.kube, nil) if err != nil { diff --git a/cmd/kubectl-ate/internal/install/tui/tui.go b/pkg/kagent/install/tui/tui.go similarity index 99% rename from cmd/kubectl-ate/internal/install/tui/tui.go rename to pkg/kagent/install/tui/tui.go index adebb80721..6ea4c79810 100644 --- a/cmd/kubectl-ate/internal/install/tui/tui.go +++ b/pkg/kagent/install/tui/tui.go @@ -25,7 +25,7 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/install" + "go.ate.dev/substrate/pkg/kagent/install" ) // progressLines is how many of a running step's latest progress lines show. diff --git a/cmd/kubectl-ate/internal/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go similarity index 90% rename from cmd/kubectl-ate/internal/install/tui/tui_test.go rename to pkg/kagent/install/tui/tui_test.go index 18e00ea92c..84ca770fdd 100644 --- a/cmd/kubectl-ate/internal/install/tui/tui_test.go +++ b/pkg/kagent/install/tui/tui_test.go @@ -22,7 +22,7 @@ import ( "github.com/google/go-cmp/cmp" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/install" + "go.ate.dev/substrate/pkg/kagent/install" ) func TestView(t *testing.T) { @@ -68,7 +68,12 @@ func TestHint(t *testing.T) { install.DefaultConfig(), func() install.Config { c := install.DefaultConfig() - c.Postgres, c.ObjectStore = install.PostgresExternal, install.ObjectStoreGCS + c.Postgres = install.Postgres{Type: install.PostgresExternal} + c.ObjectStore = install.ObjectStore{ + Type: install.ObjectStoreGCS, + Bucket: "snapshots", + GCS: install.GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}, + } c.MicroVM = install.MicroVM{Enabled: true, AssetBucket: "assets"} c.Releases.All.ValuesFiles = []string{"/etc/gke.yaml"} return c diff --git a/cmd/kubectl-ate/internal/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go similarity index 65% rename from cmd/kubectl-ate/internal/install/tui/wizard.go rename to pkg/kagent/install/tui/wizard.go index 6289f24d65..d169513f73 100644 --- a/cmd/kubectl-ate/internal/install/tui/wizard.go +++ b/pkg/kagent/install/tui/wizard.go @@ -18,42 +18,38 @@ import ( "errors" "fmt" "io" - "os" "strings" "github.com/charmbracelet/huh" "sigs.k8s.io/yaml" - "go.ate.dev/substrate/cmd/kubectl-ate/internal/install" + "go.ate.dev/substrate/pkg/kagent/install" ) // Wizard asks for the install choices, stores them in cfg, and prints the // matching config to w. It returns install.ErrAborted if the user declines or // quits. func Wizard(w io.Writer, cfg *install.Config) error { - var valuesFile string - if len(cfg.Releases.All.ValuesFiles) > 0 { - valuesFile = cfg.Releases.All.ValuesFiles[0] - } - external := func() bool { return cfg.ObjectStore != install.ObjectStoreRustFS } + store := &cfg.ObjectStore + external := func() bool { return store.Type != install.ObjectStoreBundled } confirmed := true form := huh.NewForm( huh.NewGroup( - huh.NewSelect[install.PostgresMode](). + huh.NewSelect[install.PostgresType](). Title("PostgreSQL"). Options( huh.NewOption("Bundled: single replica, for evaluation only", install.PostgresBundled), huh.NewOption("External: I created the connection Secrets", install.PostgresExternal), ). - Value(&cfg.Postgres), - huh.NewSelect[install.ObjectStore](). + Value(&cfg.Postgres.Type), + huh.NewSelect[install.ObjectStoreType](). Title("Snapshot storage"). Options( - huh.NewOption("RustFS in the cluster", install.ObjectStoreRustFS), + huh.NewOption("RustFS in the cluster", install.ObjectStoreBundled), huh.NewOption("Amazon S3 or compatible", install.ObjectStoreS3), huh.NewOption("Google Cloud Storage", install.ObjectStoreGCS), ). - Value(&cfg.ObjectStore), + Value(&store.Type), huh.NewConfirm(). Title("Install the microVM sandbox class?"). Description("Downloads the Kata and Cloud Hypervisor guest assets and stages them in the object store."). @@ -61,11 +57,31 @@ func Wizard(w io.Writer, cfg *install.Config) error { ), huh.NewGroup( huh.NewInput(). - Title("Values file"). - Description("Sets objectStore.*, ateApi.snapshotLocation and, for GCS, objectStorePlugin.transferServiceAccount."). - Value(&valuesFile). - Validate(fileExists), + Title("Snapshot bucket"). + Value(&store.Bucket). + Validate(notEmpty), ).WithHideFunc(func() bool { return !external() }), + huh.NewGroup( + huh.NewInput(). + Title("S3 region"). + Description("Empty uses the AWS SDK default."). + Value(&store.S3.Region), + huh.NewInput(). + Title("S3 endpoint"). + Description("URL of an S3-compatible service. Empty uses AWS."). + Value(&store.S3.Endpoint), + huh.NewInput(). + Title("Credentials Secret"). + Description("Secret in "+install.SystemNamespace+" with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. Empty uses ambient credentials."). + Value(&store.S3.CredentialsSecret), + ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreS3 }), + huh.NewGroup( + huh.NewInput(). + Title("Transfer service account"). + Description("Google service account the snapshot exchange impersonates."). + Value(&store.GCS.TransferServiceAccount). + Validate(notEmpty), + ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreGCS }), huh.NewGroup( huh.NewInput(). Title("MicroVM asset bucket"). @@ -90,10 +106,15 @@ func Wizard(w io.Writer, cfg *install.Config) error { if !confirmed { return install.ErrAborted } - if external() { - cfg.Releases.All.ValuesFiles = []string{valuesFile} - } else { - cfg.Releases.All.ValuesFiles = nil + // Drop answers to questions the final choices hid. + if !external() { + store.Bucket = "" + } + if store.Type != install.ObjectStoreS3 { + store.S3 = install.S3ObjectStore{} + } + if store.Type != install.ObjectStoreGCS { + store.GCS = install.GCSObjectStore{} } if !external() || !cfg.MicroVM.Enabled { cfg.MicroVM.AssetBucket = "" @@ -115,16 +136,6 @@ func Hint(cfg *install.Config) (string, error) { return "To skip these questions next time, save this as install.yaml and run kubectl ate install -c install.yaml:\n\n" + string(data) + "\n", nil } -func fileExists(p string) error { - if p == "" { - return errors.New("required") - } - if _, err := os.Stat(p); err != nil { - return fmt.Errorf("can't read %s", p) - } - return nil -} - func notEmpty(s string) error { if strings.TrimSpace(s) == "" { return errors.New("required") diff --git a/cmd/kubectl-ate/internal/install/uninstall.go b/pkg/kagent/install/uninstall.go similarity index 88% rename from cmd/kubectl-ate/internal/install/uninstall.go rename to pkg/kagent/install/uninstall.go index cea244ec66..375ba5a228 100644 --- a/cmd/kubectl-ate/internal/install/uninstall.go +++ b/pkg/kagent/install/uninstall.go @@ -36,21 +36,33 @@ import ( "go.ate.dev/substrate/internal/clustertrustbundle" ) -// ErrAborted is returned when Options.Confirm declines an uninstall. +// ErrAborted is returned when UninstallOptions.Confirm declines an uninstall. var ErrAborted = errors.New("aborted") var workerPoolsGVR = atev1.GroupVersion.WithResource("workerpools") +// UninstallOptions configures Uninstall. +type UninstallOptions struct { + // IncludeCRDs deletes the CRDs and every custom resource. + IncludeCRDs bool + // Purge deletes the namespaces, with their CA keys, secrets and database + // volume, and the published trust bundles. + Purge bool + // Confirm is shown what Uninstall will delete and returns whether to go + // on. Nil means go on. + Confirm func(ctx context.Context, summary string) (bool, error) +} + // Uninstall removes the releases in the reverse of install order. It keeps // CRDs, secrets, the database volume and the namespaces unless -// Options.IncludeCRDs or Options.Purge say otherwise. -func (in *Installer) Uninstall(ctx context.Context) error { - summary, err := in.inventory(ctx) +// opts.IncludeCRDs or opts.Purge say otherwise. +func (in *Installer) Uninstall(ctx context.Context, opts UninstallOptions) error { + summary, err := in.inventory(ctx, opts) if err != nil { return err } - if in.opts.Confirm != nil { - ok, err := in.opts.Confirm(ctx, summary) + if opts.Confirm != nil { + ok, err := opts.Confirm(ctx, summary) if err != nil { return err } @@ -58,34 +70,34 @@ func (in *Installer) Uninstall(ctx context.Context) error { return ErrAborted } } - steps := []step{ + steps := []Step{ {"Delete worker pools", in.deleteWorkerPools}, {"Uninstall Substrate", in.uninstallStep(substrateRelease)}, {"Remove PostgreSQL", in.removePostgres}, {"Uninstall atelet", in.uninstallStep(ateletRelease)}, {"Uninstall certificate controller", in.uninstallStep(podCertRelease)}, } - if in.opts.IncludeCRDs { - steps = append(steps, step{"Uninstall CRDs", in.uninstallStep(crdsRelease)}) + if opts.IncludeCRDs { + steps = append(steps, Step{"Uninstall CRDs", in.uninstallStep(crdsRelease)}) } - if in.opts.Purge { + if opts.Purge { steps = append(steps, - step{"Delete trust bundles", in.deleteTrustBundles}, - step{"Delete namespaces", in.deleteNamespaces}, + Step{"Delete trust bundles", in.deleteTrustBundles}, + Step{"Delete namespaces", in.deleteNamespaces}, ) } - return in.run(ctx, steps) + return in.Run(ctx, steps) } -// inventory describes what Uninstall will remove, for Options.Confirm. -func (in *Installer) inventory(ctx context.Context) (string, error) { +// inventory describes what Uninstall will remove, for opts.Confirm. +func (in *Installer) inventory(ctx context.Context, opts UninstallOptions) (string, error) { var installed []string for _, r := range allReleases { rel, err := in.helm.get(r) if err != nil { return "", err } - if rel != nil && (r != crdsRelease || in.opts.IncludeCRDs) { + if rel != nil && (r != crdsRelease || opts.IncludeCRDs) { installed = append(installed, r.name) } } @@ -98,10 +110,10 @@ func (in *Installer) inventory(ctx context.Context) (string, error) { if pools != nil { fmt.Fprintf(&b, "WorkerPools to delete: %d, and every actor running on them\n", len(pools.Items)) } - if in.opts.IncludeCRDs { + if opts.IncludeCRDs { b.WriteString("CRDs to delete, with every WorkerPool, SandboxConfig and CSIDriverConfig\n") } - if in.opts.Purge { + if opts.Purge { fmt.Fprintf(&b, "Namespaces to delete: %s and %s, with their CA keys, secrets and database volume\n", SystemNamespace, PodCertNamespace) } return b.String(), nil diff --git a/cmd/kubectl-ate/internal/install/uninstall_test.go b/pkg/kagent/install/uninstall_test.go similarity index 100% rename from cmd/kubectl-ate/internal/install/uninstall_test.go rename to pkg/kagent/install/uninstall_test.go diff --git a/cmd/kubectl-ate/internal/install/upgrade.go b/pkg/kagent/install/upgrade.go similarity index 87% rename from cmd/kubectl-ate/internal/install/upgrade.go rename to pkg/kagent/install/upgrade.go index 48cd7a4716..e57eaf01d8 100644 --- a/cmd/kubectl-ate/internal/install/upgrade.go +++ b/pkg/kagent/install/upgrade.go @@ -45,28 +45,34 @@ var upgradeOrder = []struct { {substrateRelease, "Upgrade Substrate"}, } -// Upgrade moves every release to the charts in this binary. With -// Options.DryRun it writes each release's manifest diff to Options.Out. -func (in *Installer) Upgrade(ctx context.Context) error { +// UpgradeOptions configures Upgrade. +type UpgradeOptions struct { + // DryRun writes each release's manifest diff to Out instead of applying. + DryRun bool + Out io.Writer +} + +// Upgrade moves every release to the charts in this binary. +func (in *Installer) Upgrade(ctx context.Context, opts UpgradeOptions) error { in.upgrading = true - steps := []step{ + steps := []Step{ {"Check versions", in.checkVersions}, {"Check CRD storage versions", in.checkCRDStorage}, } - if in.opts.DryRun { + if opts.DryRun { for _, u := range upgradeOrder { - steps = append(steps, step{"Diff " + u.r.name, in.diffStep(u.r, in.valuesFor(u.r))}) + steps = append(steps, Step{"Diff " + u.r.name, in.diffStep(u.r, in.valuesFor(u.r), opts.Out)}) } - return in.run(ctx, steps) + return in.Run(ctx, steps) } steps = append(steps, - step{"Create pod certificate CAs", in.createPodCertCAs}, - step{upgradeOrder[0].name, in.helmStep(podCertRelease, noValues)}, - step{"Wait for trust bundles", in.waitTrustBundles}, - step{"Create identity secrets", in.createIdentitySecrets}, + Step{"Create pod certificate CAs", in.createPodCertCAs}, + Step{upgradeOrder[0].name, in.helmStep(podCertRelease, noValues)}, + Step{"Wait for trust bundles", in.waitTrustBundles}, + Step{"Create identity secrets", in.createIdentitySecrets}, ) for _, u := range upgradeOrder[1:] { - steps = append(steps, step{u.name, in.helmStep(u.r, in.valuesFor(u.r))}) + steps = append(steps, Step{u.name, in.helmStep(u.r, in.valuesFor(u.r))}) } // A new release may pin new microVM assets. microVM, err := in.hasMicroVMSandboxConfig(ctx) @@ -75,12 +81,12 @@ func (in *Installer) Upgrade(ctx context.Context) error { } if microVM { steps = append(steps, - step{"Stage microVM assets", in.stageMicroVMAssets}, - step{"Apply microVM SandboxConfig", in.applyMicroVMSandboxConfig}, + Step{"Stage microVM assets", in.stageMicroVMAssets}, + Step{"Apply microVM SandboxConfig", in.applyMicroVMSandboxConfig}, ) } - steps = append(steps, step{"Check API", in.checkAPI}) - return in.run(ctx, steps) + steps = append(steps, Step{"Check API", in.checkAPI}) + return in.Run(ctx, steps) } // hasMicroVMSandboxConfig reports whether the microvm SandboxConfig exists. @@ -224,8 +230,9 @@ func checkStoredVersions(ctx context.Context, c apiextclient.Interface, crds []a return nil } -// diffStep writes the change an upgrade of r would make to its manifest. -func (in *Installer) diffStep(r release, vf valuesFunc) func(context.Context, func(string)) (string, error) { +// diffStep writes the change an upgrade of r would make to its manifest to +// out. +func (in *Installer) diffStep(r release, vf valuesFunc, out io.Writer) func(context.Context, func(string)) (string, error) { return func(ctx context.Context, _ func(string)) (string, error) { ch, err := loadChart(in.charts, r.chart) if err != nil { @@ -243,7 +250,7 @@ func (in *Installer) diffStep(r release, vf valuesFunc) func(context.Context, fu if err != nil { return "", err } - n, err := writeDiff(in.opts.Out, r.name, cur.Manifest, next) + n, err := writeDiff(out, r.name, cur.Manifest, next) if err != nil { return "", err } diff --git a/cmd/kubectl-ate/internal/install/upgrade_test.go b/pkg/kagent/install/upgrade_test.go similarity index 100% rename from cmd/kubectl-ate/internal/install/upgrade_test.go rename to pkg/kagent/install/upgrade_test.go diff --git a/cmd/kubectl-ate/internal/microvmassets/microvmassets.go b/pkg/kagent/microvmassets/microvmassets.go similarity index 100% rename from cmd/kubectl-ate/internal/microvmassets/microvmassets.go rename to pkg/kagent/microvmassets/microvmassets.go diff --git a/cmd/kubectl-ate/internal/microvmassets/microvmassets_test.go b/pkg/kagent/microvmassets/microvmassets_test.go similarity index 100% rename from cmd/kubectl-ate/internal/microvmassets/microvmassets_test.go rename to pkg/kagent/microvmassets/microvmassets_test.go