kubectl-ate: typed postgres and objectStore config for install

cmd/kubectl-ate/internal/cmd/install.go

@@ -43,8 +43,18 @@ var (
 
 const configHelp = `--config takes a YAML file, or - for stdin:
 
-  postgres: bundled        # or external
-  objectStore: rustfs      # or s3, gcs
+  postgres:
+    type: bundled          # or external
+  objectStore:
+    type: bundled          # or s3, gcs
+    bucket: my-snapshots   # for s3 or gcs
+    s3:                    # all optional
+      region: us-east-1
+      endpoint: https://s3.example.com
+      pathStyle: true
+      credentialsSecret: s3-creds # in the system namespace
+    gcs:
+      transferServiceAccount: transfer@my-project.iam.gserviceaccount.com
   microvm:
     enabled: true
     assetBucket: my-assets # for s3 or gcs
@@ -79,10 +89,9 @@ a RustFS for snapshots. On a terminal it asks for these choices first and
 prints the matching config.
 
 For anything but evaluation, create the database connection Secrets
-described in charts/substrate/README.md and set postgres: external. To store
-snapshots in a bucket, set objectStore to s3 or gcs and set the charts'
-objectStore.*, ateApi.snapshotLocation and, for GCS,
-objectStorePlugin.transferServiceAccount values.
+described in charts/substrate/README.md and set postgres.type to external.
+To store snapshots in a bucket, set objectStore.type to s3 or gcs and set
+objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount.
 
 microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each
 node architecture, caches them under the user cache directory, uploads them to

cmd/kubectl-ate/internal/install/bootstrap.go

@@ -345,17 +345,22 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) (
 	if err := ensureNamespace(ctx, in.kube, SystemNamespace); err != nil {
 		return "", err
 	}
-	if in.opts.ObjectStore != ObjectStoreRustFS {
-		// A resumed install keeps the values the release already has.
-		vals := in.user[substrateRelease.name]
-		rel, err := in.helm.get(substrateRelease)
-		if err != nil {
-			return "", err
-		}
-		if rel != nil {
-			vals = mergeValues(rel.Config, in.user[substrateRelease.name])
+	if in.opts.ObjectStore.Type != ObjectStoreBundled {
+		// A resumed install keeps the values the releases already have.
+		atelet, substrate := in.externalStoreValues()
+		vals := map[release]map[string]any{}
+		for r, typed := range map[release]map[string]any{ateletRelease: atelet, substrateRelease: substrate} {
+			rel, err := in.helm.get(r)
+			if err != nil {
+				return "", err
+			}
+			var stored map[string]any
+			if rel != nil {
+				stored = rel.Config
+			}
+			vals[r] = mergeValues(stored, in.user[r.name], typed)
 		}
-		return in.checkExternalObjectStore(ctx, vals)
+		return in.checkExternalObjectStore(ctx, vals[substrateRelease], vals[ateletRelease])
 	}
 	if ok, err := secretExists(ctx, in.kube, SystemNamespace, rustfsCredentialsSecret); ok || err != nil {
 		return "already present", err
@@ -377,18 +382,19 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) (
 }
 
 // checkExternalObjectStore fails before any release changes when a value the
-// charts need for an external store is missing from vals.
-func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[string]any) (string, error) {
-	location, _ := lookupString(vals, "ateApi", "snapshotLocation")
-	if location == "" && !previewEnabled(vals, "PerTemplateSnapshotLocation") {
-		return "", errors.New("set ateApi.snapshotLocation to the bucket URI for snapshots, such as gs://my-bucket/")
-	}
-	if in.opts.ObjectStore == ObjectStoreGCS {
-		if sa, _ := lookupString(vals, "objectStorePlugin", "transferServiceAccount"); sa == "" {
-			return "", errors.New("set objectStorePlugin.transferServiceAccount to the Google service account for snapshot transfers")
+// charts need for an external store is missing from the substrate or atelet
+// release values.
+func (in *Installer) checkExternalObjectStore(ctx context.Context, substrate, atelet map[string]any) (string, error) {
+	location, _ := lookupString(substrate, "ateApi", "snapshotLocation")
+	if location == "" && !previewEnabled(substrate, "PerTemplateSnapshotLocation") {
+		return "", errors.New("set objectStore.bucket to the snapshot bucket")
+	}
+	if in.opts.ObjectStore.Type == ObjectStoreGCS {
+		if sa, _ := lookupString(substrate, "objectStorePlugin", "transferServiceAccount"); sa == "" {
+			return "", errors.New("set objectStore.gcs.transferServiceAccount to the Google service account for snapshot transfers")
 		}
 	}
-	if name, _ := lookupString(vals, "objectStore", "credentialsSecret"); name != "" {
+	if name, _ := lookupString(atelet, "objectStore", "credentialsSecret"); name != "" {
 		ok, err := secretExists(ctx, in.kube, SystemNamespace, name)
 		if err != nil {
 			return "", err
@@ -398,9 +404,9 @@ func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[stri
 		}
 	}
 	if location == "" {
-		return string(in.opts.ObjectStore) + ", per-template locations", nil
+		return string(in.opts.ObjectStore.Type) + ", per-template locations", nil
 	}
-	return string(in.opts.ObjectStore) + " " + location, nil
+	return string(in.opts.ObjectStore.Type) + " " + location, nil
 }
 
 // lookup returns the value at path in vals, or nil.

cmd/kubectl-ate/internal/install/config.go

@@ -16,10 +16,12 @@ package install
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"helm.sh/helm/v4/pkg/cli/values"
@@ -30,9 +32,9 @@ import (
 // Config describes an installation. Install, upgrade and uninstall read it
 // from the file passed to --config.
 type Config struct {
-	Postgres    PostgresMode `json:"postgres,omitempty"`
-	ObjectStore ObjectStore  `json:"objectStore,omitempty"`
-	MicroVM     MicroVM      `json:"microvm,omitzero"`
+	Postgres    Postgres    `json:"postgres,omitzero"`
+	ObjectStore ObjectStore `json:"objectStore,omitzero"`
+	MicroVM     MicroVM     `json:"microvm,omitzero"`
 	// Timeout bounds each wait for workloads to become ready.
 	Timeout metav1.Duration `json:"timeout,omitzero"`
 	// Releases holds Helm values for the releases.
@@ -55,6 +57,111 @@ type ReleaseValues struct {
 	Values      map[string]any `json:"values,omitempty"`
 }
 
+// Postgres configures the control plane's database. External uses the
+// connection Secrets described in charts/substrate/README.md.
+type Postgres struct {
+	Type PostgresType `json:"type,omitempty"`
+}
+
+// UnmarshalJSON also accepts the type alone, as in postgres: external.
+func (p *Postgres) UnmarshalJSON(b []byte) error {
+	var t PostgresType
+	if json.Unmarshal(b, &t) == nil {
+		*p = Postgres{Type: t}
+		return nil
+	}
+	type plain Postgres
+	return decodeStrict(b, (*plain)(p))
+}
+
+// ObjectStore configures snapshot storage. The s3 and gcs settings fill the
+// charts' storage values on install, over the values in Releases.
+type ObjectStore struct {
+	Type ObjectStoreType `json:"type,omitempty"`
+	// Bucket holds snapshots with an s3 or gcs store. It sets
+	// ateApi.snapshotLocation to gs://<bucket>/, the URI both backends use.
+	// Leave it empty to set snapshot locations per template with the
+	// PerTemplateSnapshotLocation preview gate.
+	Bucket string         `json:"bucket,omitempty"`
+	S3     S3ObjectStore  `json:"s3,omitzero"`
+	GCS    GCSObjectStore `json:"gcs,omitzero"`
+}
+
+// S3ObjectStore configures an s3 store. Empty fields use the AWS SDK defaults.
+type S3ObjectStore struct {
+	Region string `json:"region,omitempty"`
+	// Endpoint is the URL of an S3-compatible service.
+	Endpoint  string `json:"endpoint,omitempty"`
+	PathStyle bool   `json:"pathStyle,omitempty"`
+	// CredentialsSecret names a Secret in SystemNamespace with
+	// AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. Empty uses the
+	// pods' ambient credentials.
+	CredentialsSecret string `json:"credentialsSecret,omitempty"`
+}
+
+// GCSObjectStore configures a gcs store.
+type GCSObjectStore struct {
+	// TransferServiceAccount is the Google service account the snapshot
+	// exchange impersonates for actor-scoped transfers.
+	TransferServiceAccount string `json:"transferServiceAccount,omitempty"`
+}
+
+// UnmarshalJSON also accepts the type alone, as in objectStore: s3.
+func (o *ObjectStore) UnmarshalJSON(b []byte) error {
+	var t ObjectStoreType
+	if json.Unmarshal(b, &t) == nil {
+		*o = ObjectStore{Type: t}
+		return nil
+	}
+	type plain ObjectStore
+	return decodeStrict(b, (*plain)(o))
+}
+
+// UnmarshalJSON reads rustfs, the v1.0.0 name, as bundled.
+func (t *ObjectStoreType) UnmarshalJSON(b []byte) error {
+	var s string
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+	if s == "rustfs" {
+		s = string(ObjectStoreBundled)
+	}
+	*t = ObjectStoreType(s)
+	return nil
+}
+
+// decodeStrict decodes b into v, keeping the fields b doesn't set.
+func decodeStrict(b []byte, v any) error {
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.DisallowUnknownFields()
+	return dec.Decode(v)
+}
+
+func (o ObjectStore) validate() error {
+	switch o.Type {
+	case ObjectStoreBundled:
+		if o != (ObjectStore{Type: ObjectStoreBundled}) {
+			return errors.New("objectStore bundled takes no other settings")
+		}
+	case ObjectStoreS3, ObjectStoreGCS:
+	default:
+		return fmt.Errorf("invalid objectStore.type %q: must be %s, %s or %s", o.Type, ObjectStoreBundled, ObjectStoreS3, ObjectStoreGCS)
+	}
+	if strings.ContainsAny(o.Bucket, ":/") {
+		return fmt.Errorf("objectStore.bucket %q must be a bucket name, not a URI", o.Bucket)
+	}
+	if o.S3 != (S3ObjectStore{}) && o.Type != ObjectStoreS3 {
+		return errors.New("objectStore.s3 needs type s3")
+	}
+	if o.GCS != (GCSObjectStore{}) && o.Type != ObjectStoreGCS {
+		return errors.New("objectStore.gcs needs type gcs")
+	}
+	if e := o.S3.Endpoint; e != "" && !strings.HasPrefix(e, "http://") && !strings.HasPrefix(e, "https://") {
+		return fmt.Errorf("objectStore.s3.endpoint %q must be an http or https URL", e)
+	}
+	return nil
+}
+
 // MicroVM configures the microvm sandbox class.
 type MicroVM struct {
 	// Enabled stages the microVM guest assets and applies the microvm
@@ -68,8 +175,8 @@ type MicroVM struct {
 // DefaultConfig is an evaluation install: bundled PostgreSQL and RustFS.
 func DefaultConfig() Config {
 	return Config{
-		Postgres:    PostgresBundled,
-		ObjectStore: ObjectStoreRustFS,
+		Postgres:    Postgres{Type: PostgresBundled},
+		ObjectStore: ObjectStore{Type: ObjectStoreBundled},
 		Timeout:     metav1.Duration{Duration: 10 * time.Minute},
 	}
 }
@@ -99,17 +206,15 @@ func ReadConfig(r io.Reader, dir string) (Config, error) {
 }
 
 func (c Config) validate() error {
-	switch c.Postgres {
+	switch c.Postgres.Type {
 	case PostgresBundled, PostgresExternal:
 	default:
-		return fmt.Errorf("invalid postgres %q: must be %s or %s", c.Postgres, PostgresBundled, PostgresExternal)
+		return fmt.Errorf("invalid postgres.type %q: must be %s or %s", c.Postgres.Type, PostgresBundled, PostgresExternal)
 	}
-	switch c.ObjectStore {
-	case ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS:
-	default:
-		return fmt.Errorf("invalid objectStore %q: must be %s, %s or %s", c.ObjectStore, ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS)
+	if err := c.ObjectStore.validate(); err != nil {
+		return err
 	}
-	if c.MicroVM.Enabled && c.ObjectStore != ObjectStoreRustFS && c.MicroVM.AssetBucket == "" {
+	if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" {
 		return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket")
 	}
 	if c.Timeout.Duration <= 0 {

cmd/kubectl-ate/internal/install/config_test.go

@@ -41,15 +41,23 @@ func TestReadConfig(t *testing.T) {
 		{
 			name: "full",
 			in: `postgres: external
-objectStore: gcs
+objectStore:
+  type: gcs
+  bucket: snapshots
+  gcs: {transferServiceAccount: sa@p.iam.gserviceaccount.com}
 microvm: {enabled: true, assetBucket: assets}
 timeout: 5m
 releases:
   all: {valuesFiles: [gke.yaml, /abs.yaml], values: {image: {tag: v1}}}
   substrate: {valuesFiles: [db.yaml]}
 `,
 			want: withDefaults(func(c *Config) {
-				c.Postgres, c.ObjectStore = PostgresExternal, ObjectStoreGCS
+				c.Postgres = Postgres{Type: PostgresExternal}
+				c.ObjectStore = ObjectStore{
+					Type:   ObjectStoreGCS,
+					Bucket: "snapshots",
+					GCS:    GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"},
+				}
 				c.MicroVM = MicroVM{Enabled: true, AssetBucket: "assets"}
 				c.Timeout = metav1.Duration{Duration: 5 * time.Minute}
 				c.Releases.All = ReleaseValues{
@@ -63,7 +71,34 @@ releases:
 		{name: "unknown field", in: "postgress: external\n", wantErr: "postgress"},
 		{name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"},
 		{name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"},
+		{
+			name: "object store as a string",
+			in:   "objectStore: s3\n",
+			want: withDefaults(func(c *Config) { c.ObjectStore = ObjectStore{Type: ObjectStoreS3} }),
+		},
+		{
+			name: "s3 object store",
+			in:   "objectStore: {type: s3, bucket: b, s3: {region: us-east-1, endpoint: 'http://minio:9000', pathStyle: true, credentialsSecret: creds}}\n",
+			want: withDefaults(func(c *Config) {
+				c.ObjectStore = ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{
+					Region: "us-east-1", Endpoint: "http://minio:9000", PathStyle: true, CredentialsSecret: "creds",
+				}}
+			}),
+		},
+		{
+			name: "postgres as a struct",
+			in:   "postgres: {type: external}\n",
+			want: withDefaults(func(c *Config) { c.Postgres = Postgres{Type: PostgresExternal} }),
+		},
+		{name: "rustfs alias", in: "objectStore: rustfs\n", want: DefaultConfig()},
+		{name: "rustfs alias in a struct", in: "objectStore: {type: rustfs}\n", want: DefaultConfig()},
 		{name: "bad object store", in: "objectStore: nfs\n", wantErr: "invalid objectStore"},
+		{name: "unknown postgres field", in: "postgres: {type: external, host: db}\n", wantErr: "host"},
+		{name: "bundled with bucket", in: "objectStore: {bucket: b}\n", wantErr: "objectStore"},
+		{name: "bucket with path", in: "objectStore: {type: s3, bucket: b/dir}\n", wantErr: "objectStore.bucket"},
+		{name: "gcs block on s3", in: "objectStore: {type: s3, gcs: {transferServiceAccount: sa}}\n", wantErr: "objectStore.gcs"},
+		{name: "s3 endpoint without scheme", in: "objectStore: {type: s3, s3: {endpoint: minio:9000}}\n", wantErr: "objectStore.s3.endpoint"},
+		{name: "unknown object store field", in: "objectStore: {type: s3, buckets: b}\n", wantErr: "buckets"},
 		{name: "microvm needs bucket", in: "objectStore: s3\nmicrovm: {enabled: true}\n", wantErr: "microvm.assetBucket"},
 		{name: "bad timeout", in: "timeout: 0s\n", wantErr: "timeout"},
 	}