diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go index 1d1a7f5b2a..f249b5cfda 100644 --- a/cmd/kubectl-ate/internal/cmd/install.go +++ b/cmd/kubectl-ate/internal/cmd/install.go @@ -43,8 +43,18 @@ var ( const configHelp = `--config takes a YAML file, or - for stdin: - postgres: bundled # or external - objectStore: rustfs # or s3, gcs + postgres: + type: bundled # or external + objectStore: + type: bundled # or s3, gcs + bucket: my-snapshots # for s3 or gcs + s3: # all optional + region: us-east-1 + endpoint: https://s3.example.com + pathStyle: true + credentialsSecret: s3-creds # in the system namespace + gcs: + transferServiceAccount: transfer@my-project.iam.gserviceaccount.com microvm: enabled: true assetBucket: my-assets # for s3 or gcs @@ -79,10 +89,9 @@ a RustFS for snapshots. On a terminal it asks for these choices first and prints the matching config. For anything but evaluation, create the database connection Secrets -described in charts/substrate/README.md and set postgres: external. To store -snapshots in a bucket, set objectStore to s3 or gcs and set the charts' -objectStore.*, ateApi.snapshotLocation and, for GCS, -objectStorePlugin.transferServiceAccount values. +described in charts/substrate/README.md and set postgres.type to external. +To store snapshots in a bucket, set objectStore.type to s3 or gcs and set +objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount. microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each node architecture, caches them under the user cache directory, uploads them to diff --git a/cmd/kubectl-ate/internal/install/bootstrap.go b/cmd/kubectl-ate/internal/install/bootstrap.go index ae86dd50ce..f4cb5e810a 100644 --- a/cmd/kubectl-ate/internal/install/bootstrap.go +++ b/cmd/kubectl-ate/internal/install/bootstrap.go @@ -345,17 +345,22 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) ( if err := ensureNamespace(ctx, in.kube, SystemNamespace); err != nil { return "", err } - if in.opts.ObjectStore != ObjectStoreRustFS { - // A resumed install keeps the values the release already has. - vals := in.user[substrateRelease.name] - rel, err := in.helm.get(substrateRelease) - if err != nil { - return "", err - } - if rel != nil { - vals = mergeValues(rel.Config, in.user[substrateRelease.name]) + if in.opts.ObjectStore.Type != ObjectStoreBundled { + // A resumed install keeps the values the releases already have. + atelet, substrate := in.externalStoreValues() + vals := map[release]map[string]any{} + for r, typed := range map[release]map[string]any{ateletRelease: atelet, substrateRelease: substrate} { + rel, err := in.helm.get(r) + if err != nil { + return "", err + } + var stored map[string]any + if rel != nil { + stored = rel.Config + } + vals[r] = mergeValues(stored, in.user[r.name], typed) } - return in.checkExternalObjectStore(ctx, vals) + return in.checkExternalObjectStore(ctx, vals[substrateRelease], vals[ateletRelease]) } if ok, err := secretExists(ctx, in.kube, SystemNamespace, rustfsCredentialsSecret); ok || err != nil { return "already present", err @@ -377,18 +382,19 @@ func (in *Installer) configureObjectStore(ctx context.Context, _ func(string)) ( } // checkExternalObjectStore fails before any release changes when a value the -// charts need for an external store is missing from vals. -func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[string]any) (string, error) { - location, _ := lookupString(vals, "ateApi", "snapshotLocation") - if location == "" && !previewEnabled(vals, "PerTemplateSnapshotLocation") { - return "", errors.New("set ateApi.snapshotLocation to the bucket URI for snapshots, such as gs://my-bucket/") - } - if in.opts.ObjectStore == ObjectStoreGCS { - if sa, _ := lookupString(vals, "objectStorePlugin", "transferServiceAccount"); sa == "" { - return "", errors.New("set objectStorePlugin.transferServiceAccount to the Google service account for snapshot transfers") +// charts need for an external store is missing from the substrate or atelet +// release values. +func (in *Installer) checkExternalObjectStore(ctx context.Context, substrate, atelet map[string]any) (string, error) { + location, _ := lookupString(substrate, "ateApi", "snapshotLocation") + if location == "" && !previewEnabled(substrate, "PerTemplateSnapshotLocation") { + return "", errors.New("set objectStore.bucket to the snapshot bucket") + } + if in.opts.ObjectStore.Type == ObjectStoreGCS { + if sa, _ := lookupString(substrate, "objectStorePlugin", "transferServiceAccount"); sa == "" { + return "", errors.New("set objectStore.gcs.transferServiceAccount to the Google service account for snapshot transfers") } } - if name, _ := lookupString(vals, "objectStore", "credentialsSecret"); name != "" { + if name, _ := lookupString(atelet, "objectStore", "credentialsSecret"); name != "" { ok, err := secretExists(ctx, in.kube, SystemNamespace, name) if err != nil { return "", err @@ -398,9 +404,9 @@ func (in *Installer) checkExternalObjectStore(ctx context.Context, vals map[stri } } if location == "" { - return string(in.opts.ObjectStore) + ", per-template locations", nil + return string(in.opts.ObjectStore.Type) + ", per-template locations", nil } - return string(in.opts.ObjectStore) + " " + location, nil + return string(in.opts.ObjectStore.Type) + " " + location, nil } // lookup returns the value at path in vals, or nil. diff --git a/cmd/kubectl-ate/internal/install/config.go b/cmd/kubectl-ate/internal/install/config.go index 2dcfba4013..d45387c29f 100644 --- a/cmd/kubectl-ate/internal/install/config.go +++ b/cmd/kubectl-ate/internal/install/config.go @@ -16,10 +16,12 @@ package install import ( "bytes" + "encoding/json" "errors" "fmt" "io" "path/filepath" + "strings" "time" "helm.sh/helm/v4/pkg/cli/values" @@ -30,9 +32,9 @@ import ( // Config describes an installation. Install, upgrade and uninstall read it // from the file passed to --config. type Config struct { - Postgres PostgresMode `json:"postgres,omitempty"` - ObjectStore ObjectStore `json:"objectStore,omitempty"` - MicroVM MicroVM `json:"microvm,omitzero"` + Postgres Postgres `json:"postgres,omitzero"` + ObjectStore ObjectStore `json:"objectStore,omitzero"` + MicroVM MicroVM `json:"microvm,omitzero"` // Timeout bounds each wait for workloads to become ready. Timeout metav1.Duration `json:"timeout,omitzero"` // Releases holds Helm values for the releases. @@ -55,6 +57,111 @@ type ReleaseValues struct { Values map[string]any `json:"values,omitempty"` } +// Postgres configures the control plane's database. External uses the +// connection Secrets described in charts/substrate/README.md. +type Postgres struct { + Type PostgresType `json:"type,omitempty"` +} + +// UnmarshalJSON also accepts the type alone, as in postgres: external. +func (p *Postgres) UnmarshalJSON(b []byte) error { + var t PostgresType + if json.Unmarshal(b, &t) == nil { + *p = Postgres{Type: t} + return nil + } + type plain Postgres + return decodeStrict(b, (*plain)(p)) +} + +// ObjectStore configures snapshot storage. The s3 and gcs settings fill the +// charts' storage values on install, over the values in Releases. +type ObjectStore struct { + Type ObjectStoreType `json:"type,omitempty"` + // Bucket holds snapshots with an s3 or gcs store. It sets + // ateApi.snapshotLocation to gs:///, the URI both backends use. + // Leave it empty to set snapshot locations per template with the + // PerTemplateSnapshotLocation preview gate. + Bucket string `json:"bucket,omitempty"` + S3 S3ObjectStore `json:"s3,omitzero"` + GCS GCSObjectStore `json:"gcs,omitzero"` +} + +// S3ObjectStore configures an s3 store. Empty fields use the AWS SDK defaults. +type S3ObjectStore struct { + Region string `json:"region,omitempty"` + // Endpoint is the URL of an S3-compatible service. + Endpoint string `json:"endpoint,omitempty"` + PathStyle bool `json:"pathStyle,omitempty"` + // CredentialsSecret names a Secret in SystemNamespace with + // AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. Empty uses the + // pods' ambient credentials. + CredentialsSecret string `json:"credentialsSecret,omitempty"` +} + +// GCSObjectStore configures a gcs store. +type GCSObjectStore struct { + // TransferServiceAccount is the Google service account the snapshot + // exchange impersonates for actor-scoped transfers. + TransferServiceAccount string `json:"transferServiceAccount,omitempty"` +} + +// UnmarshalJSON also accepts the type alone, as in objectStore: s3. +func (o *ObjectStore) UnmarshalJSON(b []byte) error { + var t ObjectStoreType + if json.Unmarshal(b, &t) == nil { + *o = ObjectStore{Type: t} + return nil + } + type plain ObjectStore + return decodeStrict(b, (*plain)(o)) +} + +// UnmarshalJSON reads rustfs, the v1.0.0 name, as bundled. +func (t *ObjectStoreType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + if s == "rustfs" { + s = string(ObjectStoreBundled) + } + *t = ObjectStoreType(s) + return nil +} + +// decodeStrict decodes b into v, keeping the fields b doesn't set. +func decodeStrict(b []byte, v any) error { + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + return dec.Decode(v) +} + +func (o ObjectStore) validate() error { + switch o.Type { + case ObjectStoreBundled: + if o != (ObjectStore{Type: ObjectStoreBundled}) { + return errors.New("objectStore bundled takes no other settings") + } + case ObjectStoreS3, ObjectStoreGCS: + default: + return fmt.Errorf("invalid objectStore.type %q: must be %s, %s or %s", o.Type, ObjectStoreBundled, ObjectStoreS3, ObjectStoreGCS) + } + if strings.ContainsAny(o.Bucket, ":/") { + return fmt.Errorf("objectStore.bucket %q must be a bucket name, not a URI", o.Bucket) + } + if o.S3 != (S3ObjectStore{}) && o.Type != ObjectStoreS3 { + return errors.New("objectStore.s3 needs type s3") + } + if o.GCS != (GCSObjectStore{}) && o.Type != ObjectStoreGCS { + return errors.New("objectStore.gcs needs type gcs") + } + if e := o.S3.Endpoint; e != "" && !strings.HasPrefix(e, "http://") && !strings.HasPrefix(e, "https://") { + return fmt.Errorf("objectStore.s3.endpoint %q must be an http or https URL", e) + } + return nil +} + // MicroVM configures the microvm sandbox class. type MicroVM struct { // Enabled stages the microVM guest assets and applies the microvm @@ -68,8 +175,8 @@ type MicroVM struct { // DefaultConfig is an evaluation install: bundled PostgreSQL and RustFS. func DefaultConfig() Config { return Config{ - Postgres: PostgresBundled, - ObjectStore: ObjectStoreRustFS, + Postgres: Postgres{Type: PostgresBundled}, + ObjectStore: ObjectStore{Type: ObjectStoreBundled}, Timeout: metav1.Duration{Duration: 10 * time.Minute}, } } @@ -99,17 +206,15 @@ func ReadConfig(r io.Reader, dir string) (Config, error) { } func (c Config) validate() error { - switch c.Postgres { + switch c.Postgres.Type { case PostgresBundled, PostgresExternal: default: - return fmt.Errorf("invalid postgres %q: must be %s or %s", c.Postgres, PostgresBundled, PostgresExternal) + return fmt.Errorf("invalid postgres.type %q: must be %s or %s", c.Postgres.Type, PostgresBundled, PostgresExternal) } - switch c.ObjectStore { - case ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS: - default: - return fmt.Errorf("invalid objectStore %q: must be %s, %s or %s", c.ObjectStore, ObjectStoreRustFS, ObjectStoreS3, ObjectStoreGCS) + if err := c.ObjectStore.validate(); err != nil { + return err } - if c.MicroVM.Enabled && c.ObjectStore != ObjectStoreRustFS && c.MicroVM.AssetBucket == "" { + if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" { return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket") } if c.Timeout.Duration <= 0 { diff --git a/cmd/kubectl-ate/internal/install/config_test.go b/cmd/kubectl-ate/internal/install/config_test.go index be58c28190..d3094e3fab 100644 --- a/cmd/kubectl-ate/internal/install/config_test.go +++ b/cmd/kubectl-ate/internal/install/config_test.go @@ -41,7 +41,10 @@ func TestReadConfig(t *testing.T) { { name: "full", in: `postgres: external -objectStore: gcs +objectStore: + type: gcs + bucket: snapshots + gcs: {transferServiceAccount: sa@p.iam.gserviceaccount.com} microvm: {enabled: true, assetBucket: assets} timeout: 5m releases: @@ -49,7 +52,12 @@ releases: substrate: {valuesFiles: [db.yaml]} `, want: withDefaults(func(c *Config) { - c.Postgres, c.ObjectStore = PostgresExternal, ObjectStoreGCS + c.Postgres = Postgres{Type: PostgresExternal} + c.ObjectStore = ObjectStore{ + Type: ObjectStoreGCS, + Bucket: "snapshots", + GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}, + } c.MicroVM = MicroVM{Enabled: true, AssetBucket: "assets"} c.Timeout = metav1.Duration{Duration: 5 * time.Minute} c.Releases.All = ReleaseValues{ @@ -63,7 +71,34 @@ releases: {name: "unknown field", in: "postgress: external\n", wantErr: "postgress"}, {name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"}, {name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"}, + { + name: "object store as a string", + in: "objectStore: s3\n", + want: withDefaults(func(c *Config) { c.ObjectStore = ObjectStore{Type: ObjectStoreS3} }), + }, + { + name: "s3 object store", + in: "objectStore: {type: s3, bucket: b, s3: {region: us-east-1, endpoint: 'http://minio:9000', pathStyle: true, credentialsSecret: creds}}\n", + want: withDefaults(func(c *Config) { + c.ObjectStore = ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{ + Region: "us-east-1", Endpoint: "http://minio:9000", PathStyle: true, CredentialsSecret: "creds", + }} + }), + }, + { + name: "postgres as a struct", + in: "postgres: {type: external}\n", + want: withDefaults(func(c *Config) { c.Postgres = Postgres{Type: PostgresExternal} }), + }, + {name: "rustfs alias", in: "objectStore: rustfs\n", want: DefaultConfig()}, + {name: "rustfs alias in a struct", in: "objectStore: {type: rustfs}\n", want: DefaultConfig()}, {name: "bad object store", in: "objectStore: nfs\n", wantErr: "invalid objectStore"}, + {name: "unknown postgres field", in: "postgres: {type: external, host: db}\n", wantErr: "host"}, + {name: "bundled with bucket", in: "objectStore: {bucket: b}\n", wantErr: "objectStore"}, + {name: "bucket with path", in: "objectStore: {type: s3, bucket: b/dir}\n", wantErr: "objectStore.bucket"}, + {name: "gcs block on s3", in: "objectStore: {type: s3, gcs: {transferServiceAccount: sa}}\n", wantErr: "objectStore.gcs"}, + {name: "s3 endpoint without scheme", in: "objectStore: {type: s3, s3: {endpoint: minio:9000}}\n", wantErr: "objectStore.s3.endpoint"}, + {name: "unknown object store field", in: "objectStore: {type: s3, buckets: b}\n", wantErr: "buckets"}, {name: "microvm needs bucket", in: "objectStore: s3\nmicrovm: {enabled: true}\n", wantErr: "microvm.assetBucket"}, {name: "bad timeout", in: "timeout: 0s\n", wantErr: "timeout"}, } diff --git a/cmd/kubectl-ate/internal/install/install_test.go b/cmd/kubectl-ate/internal/install/install_test.go index 5823b8f3fe..41f202ef77 100644 --- a/cmd/kubectl-ate/internal/install/install_test.go +++ b/cmd/kubectl-ate/internal/install/install_test.go @@ -120,7 +120,7 @@ func TestLoadEmbeddedCharts(t *testing.T) { func TestBootstrapKeepsExistingSecrets(t *testing.T) { ctx := context.Background() kc := fake.NewClientset() - in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStoreRustFS}}} + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreBundled}}}} for _, s := range []func(context.Context, func(string)) (string, error){ in.createPodCertCAs, in.createIdentitySecrets, in.configureObjectStore, @@ -203,32 +203,49 @@ func TestApplyPostgresManifest(t *testing.T) { } func TestCheckExternalObjectStore(t *testing.T) { - gcs := map[string]any{ - "ateApi": map[string]any{"snapshotLocation": "gs://b/"}, - "objectStorePlugin": map[string]any{"transferServiceAccount": "sa@p.iam.gserviceaccount.com"}, - } + s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"} + gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}} tests := []struct { name string store ObjectStore - user map[string]any + user map[release]map[string]any + stored map[release]map[string]any secrets []string wantErr string }{ - {name: "s3", store: ObjectStoreS3, user: map[string]any{"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, - {name: "per-template locations", store: ObjectStoreS3, user: map[string]any{"preview": []any{"PerTemplateSnapshotLocation"}}}, - {name: "no location", store: ObjectStoreS3, wantErr: "ateApi.snapshotLocation"}, - {name: "gcs", store: ObjectStoreGCS, user: gcs}, - {name: "gcs without service account", store: ObjectStoreGCS, user: map[string]any{"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}, wantErr: "transferServiceAccount"}, + {name: "s3", store: s3}, + {name: "gcs", store: gcs}, + { + name: "location from chart values", + store: ObjectStore{Type: ObjectStoreS3}, + user: map[release]map[string]any{substrateRelease: {"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, + }, + { + name: "location from the release", + store: ObjectStore{Type: ObjectStoreS3}, + stored: map[release]map[string]any{substrateRelease: {"ateApi": map[string]any{"snapshotLocation": "gs://b/"}}}, + }, + { + name: "per-template locations", + store: ObjectStore{Type: ObjectStoreS3}, + user: map[release]map[string]any{substrateRelease: {"preview": []any{"PerTemplateSnapshotLocation"}}}, + }, + {name: "no location", store: ObjectStore{Type: ObjectStoreS3}, wantErr: "objectStore.bucket"}, + {name: "gcs without service account", store: ObjectStore{Type: ObjectStoreGCS, Bucket: "b"}, wantErr: "objectStore.gcs.transferServiceAccount"}, { name: "credentials secret present", - store: ObjectStoreS3, - user: map[string]any{"preview": []any{"ALL"}, "objectStore": map[string]any{"credentialsSecret": "s3-creds"}}, + store: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{CredentialsSecret: "s3-creds"}}, secrets: []string{"s3-creds"}, }, { name: "credentials secret missing", - store: ObjectStoreS3, - user: map[string]any{"preview": []any{"ALL"}, "objectStore": map[string]any{"credentialsSecret": "s3-creds"}}, + store: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{CredentialsSecret: "s3-creds"}}, + wantErr: "doesn't exist", + }, + { + name: "credentials secret from chart values", + store: s3, + user: map[release]map[string]any{ateletRelease: {"objectStore": map[string]any{"credentialsSecret": "s3-creds"}}}, wantErr: "doesn't exist", }, } @@ -240,8 +257,14 @@ func TestCheckExternalObjectStore(t *testing.T) { t.Fatal(err) } } - in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: tc.store}}} - _, err := in.checkExternalObjectStore(context.Background(), tc.user) + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: tc.store}}, user: map[string]map[string]any{}} + for r, v := range tc.user { + in.user[r.name] = v + } + atelet, substrate := in.externalStoreValues() + _, err := in.checkExternalObjectStore(context.Background(), + mergeValues(tc.stored[substrateRelease], tc.user[substrateRelease], substrate), + mergeValues(tc.stored[ateletRelease], tc.user[ateletRelease], atelet)) if tc.wantErr == "" { if err != nil { t.Fatalf("checkExternalObjectStore() = %v", err) @@ -256,7 +279,7 @@ func TestCheckExternalObjectStore(t *testing.T) { } func TestExternalObjectStoreValues(t *testing.T) { - in := &Installer{opts: Options{Config: Config{ObjectStore: ObjectStoreGCS, Postgres: PostgresExternal}}} + in := &Installer{opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreGCS}, Postgres: Postgres{Type: PostgresExternal}}}} defaults, forced, err := in.ateletValues(context.Background()) if err != nil { t.Fatal(err) @@ -275,3 +298,39 @@ func TestExternalObjectStoreValues(t *testing.T) { t.Errorf("substrate rustfs.enabled = %v, want false", got) } } + +func TestExternalStoreValuesS3(t *testing.T) { + userEnv := map[string]any{"name": "HTTPS_PROXY", "value": "http://proxy:3128"} + in := &Installer{ + opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreS3, Bucket: "b", S3: S3ObjectStore{ + Region: "us-east-1", Endpoint: "http://minio:9000", PathStyle: true, CredentialsSecret: "creds", + }}}}, + user: map[string]map[string]any{substrateRelease.name: {"objectStorePlugin": map[string]any{"extraEnv": []any{userEnv}}}}, + } + atelet, substrate := in.externalStoreValues() + wantAtelet := map[string]any{"objectStore": map[string]any{ + "backend": "s3", "region": "us-east-1", "endpoint": "http://minio:9000", "pathStyle": true, "credentialsSecret": "creds", + }} + if !reflect.DeepEqual(atelet, wantAtelet) { + t.Errorf("atelet values = %v, want %v", atelet, wantAtelet) + } + secret := func(key string) map[string]any { + return map[string]any{"name": key, "valueFrom": map[string]any{"secretKeyRef": map[string]any{"name": "creds", "key": key}}} + } + wantSubstrate := map[string]any{ + "rustfs": map[string]any{"enabled": false}, + "atelet": map[string]any{"storageBackend": "s3"}, + "ateApi": map[string]any{"snapshotLocation": "gs://b/"}, + "objectStorePlugin": map[string]any{"extraEnv": []any{ + userEnv, + map[string]any{"name": "AWS_REGION", "value": "us-east-1"}, + map[string]any{"name": "AWS_ENDPOINT_URL", "value": "http://minio:9000"}, + map[string]any{"name": "AWS_S3_USE_PATH_STYLE", "value": "true"}, + secret("AWS_ACCESS_KEY_ID"), + secret("AWS_SECRET_ACCESS_KEY"), + }}, + } + if !reflect.DeepEqual(substrate, wantSubstrate) { + t.Errorf("substrate values = %v, want %v", substrate, wantSubstrate) + } +} diff --git a/cmd/kubectl-ate/internal/install/installer.go b/cmd/kubectl-ate/internal/install/installer.go index a2c8353fd9..eaa18915ef 100644 --- a/cmd/kubectl-ate/internal/install/installer.go +++ b/cmd/kubectl-ate/internal/install/installer.go @@ -40,27 +40,27 @@ const ( PodCertNamespace = "podcertificate-controller-system" ) -// PostgresMode selects where the control plane's database comes from. -type PostgresMode string +// PostgresType selects where the control plane's database comes from. +type PostgresType string const ( // PostgresBundled applies manifests/ate-install/postgres for evaluation. // It has no backups or high availability. - PostgresBundled PostgresMode = "bundled" + PostgresBundled PostgresType = "bundled" // PostgresExternal uses connection Secrets the user created. - PostgresExternal PostgresMode = "external" + PostgresExternal PostgresType = "external" ) -// ObjectStore selects where snapshots are stored. -type ObjectStore string +// ObjectStoreType selects where snapshots are stored. +type ObjectStoreType string const ( - // ObjectStoreRustFS runs RustFS in the cluster with the substrate release. - ObjectStoreRustFS ObjectStore = "rustfs" - // ObjectStoreS3 and ObjectStoreGCS use a bucket the user configures with - // the charts' objectStore, objectStorePlugin and ateApi values. - ObjectStoreS3 ObjectStore = "s3" - ObjectStoreGCS ObjectStore = "gcs" + // ObjectStoreBundled runs RustFS in the cluster with the substrate + // release, for evaluation. + ObjectStoreBundled ObjectStoreType = "bundled" + // ObjectStoreS3 and ObjectStoreGCS use a bucket the user created. + ObjectStoreS3 ObjectStoreType = "s3" + ObjectStoreGCS ObjectStoreType = "gcs" ) // Options configures an Installer. @@ -176,8 +176,8 @@ func (in *Installer) Install(ctx context.Context) error { if err := in.opts.validate(); err != nil { return err } - if in.opts.Postgres == PostgresBundled { - in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres: external with a managed database for anything else."}) + if in.opts.Postgres.Type == PostgresBundled { + in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres.type to external with a managed database for anything else."}) } steps := []step{ {"Check cluster", in.preflight}, @@ -189,7 +189,7 @@ func (in *Installer) Install(ctx context.Context) error { {"Configure object storage", in.configureObjectStore}, {"Install atelet", in.helmStep(ateletRelease, in.ateletValues)}, } - if in.opts.Postgres == PostgresBundled { + if in.opts.Postgres.Type == PostgresBundled { steps = append(steps, step{"Install PostgreSQL", in.applyPostgres}, step{"Set up database", in.setupDatabase}, diff --git a/cmd/kubectl-ate/internal/install/microvm.go b/cmd/kubectl-ate/internal/install/microvm.go index 5784d73f85..d86e68aa3d 100644 --- a/cmd/kubectl-ate/internal/install/microvm.go +++ b/cmd/kubectl-ate/internal/install/microvm.go @@ -54,7 +54,7 @@ const ( // assetTarget is where the microVM assets are staged. endpoint, region and // pathStyle come from atelet's objectStore values for an S3 store. type assetTarget struct { - store ObjectStore + store ObjectStoreType scheme string bucket string endpoint string @@ -75,7 +75,7 @@ func (in *Installer) microVMTarget(ctx context.Context) (assetTarget, error) { if bucket == "" { return assetTarget{}, errors.New("substrate release has no rustfs.bucket") } - return assetTarget{store: ObjectStoreRustFS, scheme: "s3", bucket: bucket}, nil + return assetTarget{store: ObjectStoreBundled, scheme: "s3", bucket: bucket}, nil } atelet, err := in.installedValues(ateletRelease) if err != nil { @@ -295,7 +295,7 @@ func (b gcsBucket) put(ctx context.Context, object string, r io.Reader) error { // uses the caller's own cloud credentials. func (in *Installer) assetStore(ctx context.Context, t assetTarget) (assetBucket, func(), error) { switch t.store { - case ObjectStoreRustFS: + case ObjectStoreBundled: access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err diff --git a/cmd/kubectl-ate/internal/install/steps.go b/cmd/kubectl-ate/internal/install/steps.go index b4de63fd78..481255ef75 100644 --- a/cmd/kubectl-ate/internal/install/steps.go +++ b/cmd/kubectl-ate/internal/install/steps.go @@ -166,9 +166,9 @@ func (in *Installer) ateletValues(context.Context) (map[string]any, map[string]a // The release keeps its storage values. return nil, forced, nil } - if in.opts.ObjectStore != ObjectStoreRustFS { - forced["objectStore"] = map[string]any{"backend": string(in.opts.ObjectStore)} - return nil, forced, nil + if in.opts.ObjectStore.Type != ObjectStoreBundled { + atelet, _ := in.externalStoreValues() + return nil, mergeValues(forced, atelet), nil } defaults := map[string]any{ "objectStore": map[string]any{ @@ -191,21 +191,79 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s // The release keeps its storage and database values. return nil, forced, nil } - if in.opts.ObjectStore == ObjectStoreRustFS { + if in.opts.ObjectStore.Type == ObjectStoreBundled { access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err } forced["rustfs"] = map[string]any{"accessKey": access, "secretKey": secret} } else { - forced["rustfs"] = map[string]any{"enabled": false} + _, substrate := in.externalStoreValues() + forced = mergeValues(forced, substrate) } - if in.opts.Postgres == PostgresBundled { + if in.opts.Postgres.Type == PostgresBundled { forced["postgres"] = map[string]any{"clientCertificates": map[string]any{"enabled": true}} } return nil, forced, nil } +// externalStoreValues returns the values Config.ObjectStore sets on the +// substrate-atelet and substrate releases for an s3 or gcs store. The +// substrate chart has no objectStore values, so its control-plane plugin gets +// the S3 settings as extra environment variables. +func (in *Installer) externalStoreValues() (atelet, substrate map[string]any) { + o := in.opts.ObjectStore + store := map[string]any{"backend": string(o.Type)} + plugin := map[string]any{} + substrate = map[string]any{ + "rustfs": map[string]any{"enabled": false}, + "atelet": map[string]any{"storageBackend": string(o.Type)}, + "objectStorePlugin": plugin, + } + if o.Bucket != "" { + substrate["ateApi"] = map[string]any{"snapshotLocation": "gs://" + o.Bucket + "/"} + } + if sa := o.GCS.TransferServiceAccount; sa != "" { + plugin["transferServiceAccount"] = sa + } + + var env []any + addEnv := func(name string, value any) { + e := map[string]any{"name": name} + if s, ok := value.(string); ok { + e["value"] = s + } else { + e["valueFrom"] = value + } + env = append(env, e) + } + s3 := o.S3 + if s3.Region != "" { + store["region"] = s3.Region + addEnv("AWS_REGION", s3.Region) + } + if s3.Endpoint != "" { + store["endpoint"] = s3.Endpoint + addEnv("AWS_ENDPOINT_URL", s3.Endpoint) + } + if s3.PathStyle { + store["pathStyle"] = true + addEnv("AWS_S3_USE_PATH_STYLE", "true") + } + if name := s3.CredentialsSecret; name != "" { + store["credentialsSecret"] = name + for _, key := range []string{"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"} { + addEnv(key, map[string]any{"secretKeyRef": map[string]any{"name": name, "key": key}}) + } + } + if env != nil { + // Lists replace in a merge, so keep the user's entries. + user, _ := lookup(in.user[substrateRelease.name], "objectStorePlugin", "extraEnv").([]any) + plugin["extraEnv"] = append(append([]any{}, user...), env...) + } + return map[string]any{"objectStore": store}, substrate +} + func (in *Installer) waitTrustBundles(ctx context.Context, progress func(string)) (string, error) { ctbs, err := clustertrustbundle.NewClient(in.kube, nil) if err != nil { diff --git a/cmd/kubectl-ate/internal/install/tui/tui_test.go b/cmd/kubectl-ate/internal/install/tui/tui_test.go index 18e00ea92c..d0274ccf5f 100644 --- a/cmd/kubectl-ate/internal/install/tui/tui_test.go +++ b/cmd/kubectl-ate/internal/install/tui/tui_test.go @@ -68,7 +68,12 @@ func TestHint(t *testing.T) { install.DefaultConfig(), func() install.Config { c := install.DefaultConfig() - c.Postgres, c.ObjectStore = install.PostgresExternal, install.ObjectStoreGCS + c.Postgres = install.Postgres{Type: install.PostgresExternal} + c.ObjectStore = install.ObjectStore{ + Type: install.ObjectStoreGCS, + Bucket: "snapshots", + GCS: install.GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}, + } c.MicroVM = install.MicroVM{Enabled: true, AssetBucket: "assets"} c.Releases.All.ValuesFiles = []string{"/etc/gke.yaml"} return c diff --git a/cmd/kubectl-ate/internal/install/tui/wizard.go b/cmd/kubectl-ate/internal/install/tui/wizard.go index 6289f24d65..f49da0995e 100644 --- a/cmd/kubectl-ate/internal/install/tui/wizard.go +++ b/cmd/kubectl-ate/internal/install/tui/wizard.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "io" - "os" "strings" "github.com/charmbracelet/huh" @@ -31,29 +30,26 @@ import ( // matching config to w. It returns install.ErrAborted if the user declines or // quits. func Wizard(w io.Writer, cfg *install.Config) error { - var valuesFile string - if len(cfg.Releases.All.ValuesFiles) > 0 { - valuesFile = cfg.Releases.All.ValuesFiles[0] - } - external := func() bool { return cfg.ObjectStore != install.ObjectStoreRustFS } + store := &cfg.ObjectStore + external := func() bool { return store.Type != install.ObjectStoreBundled } confirmed := true form := huh.NewForm( huh.NewGroup( - huh.NewSelect[install.PostgresMode](). + huh.NewSelect[install.PostgresType](). Title("PostgreSQL"). Options( huh.NewOption("Bundled: single replica, for evaluation only", install.PostgresBundled), huh.NewOption("External: I created the connection Secrets", install.PostgresExternal), ). - Value(&cfg.Postgres), - huh.NewSelect[install.ObjectStore](). + Value(&cfg.Postgres.Type), + huh.NewSelect[install.ObjectStoreType](). Title("Snapshot storage"). Options( - huh.NewOption("RustFS in the cluster", install.ObjectStoreRustFS), + huh.NewOption("RustFS in the cluster", install.ObjectStoreBundled), huh.NewOption("Amazon S3 or compatible", install.ObjectStoreS3), huh.NewOption("Google Cloud Storage", install.ObjectStoreGCS), ). - Value(&cfg.ObjectStore), + Value(&store.Type), huh.NewConfirm(). Title("Install the microVM sandbox class?"). Description("Downloads the Kata and Cloud Hypervisor guest assets and stages them in the object store."). @@ -61,11 +57,31 @@ func Wizard(w io.Writer, cfg *install.Config) error { ), huh.NewGroup( huh.NewInput(). - Title("Values file"). - Description("Sets objectStore.*, ateApi.snapshotLocation and, for GCS, objectStorePlugin.transferServiceAccount."). - Value(&valuesFile). - Validate(fileExists), + Title("Snapshot bucket"). + Value(&store.Bucket). + Validate(notEmpty), ).WithHideFunc(func() bool { return !external() }), + huh.NewGroup( + huh.NewInput(). + Title("S3 region"). + Description("Empty uses the AWS SDK default."). + Value(&store.S3.Region), + huh.NewInput(). + Title("S3 endpoint"). + Description("URL of an S3-compatible service. Empty uses AWS."). + Value(&store.S3.Endpoint), + huh.NewInput(). + Title("Credentials Secret"). + Description("Secret in "+install.SystemNamespace+" with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. Empty uses ambient credentials."). + Value(&store.S3.CredentialsSecret), + ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreS3 }), + huh.NewGroup( + huh.NewInput(). + Title("Transfer service account"). + Description("Google service account the snapshot exchange impersonates."). + Value(&store.GCS.TransferServiceAccount). + Validate(notEmpty), + ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreGCS }), huh.NewGroup( huh.NewInput(). Title("MicroVM asset bucket"). @@ -90,10 +106,15 @@ func Wizard(w io.Writer, cfg *install.Config) error { if !confirmed { return install.ErrAborted } - if external() { - cfg.Releases.All.ValuesFiles = []string{valuesFile} - } else { - cfg.Releases.All.ValuesFiles = nil + // Drop answers to questions the final choices hid. + if !external() { + store.Bucket = "" + } + if store.Type != install.ObjectStoreS3 { + store.S3 = install.S3ObjectStore{} + } + if store.Type != install.ObjectStoreGCS { + store.GCS = install.GCSObjectStore{} } if !external() || !cfg.MicroVM.Enabled { cfg.MicroVM.AssetBucket = "" @@ -115,16 +136,6 @@ func Hint(cfg *install.Config) (string, error) { return "To skip these questions next time, save this as install.yaml and run kubectl ate install -c install.yaml:\n\n" + string(data) + "\n", nil } -func fileExists(p string) error { - if p == "" { - return errors.New("required") - } - if _, err := os.Stat(p); err != nil { - return fmt.Errorf("can't read %s", p) - } - return nil -} - func notEmpty(s string) error { if strings.TrimSpace(s) == "" { return errors.New("required")