diff --git a/charts/substrate-atelet/README.md b/charts/substrate-atelet/README.md index 4f756e6b0b..60f774f654 100644 --- a/charts/substrate-atelet/README.md +++ b/charts/substrate-atelet/README.md @@ -44,6 +44,15 @@ This chart reads the same value paths as the substrate chart for `atelet`, `image`, `imagePullSecrets` and `global`, so the substrate release's values file can configure both releases. +These values are only in this chart: + +| Key | Default | Description | +|---|---|---| +| `atelet.extraVolumes` | `[]` | Volumes added to the atelet pod, such as a host file a credential provider reads | +| `atelet.extraVolumeMounts` | `[]` | Volume mounts added to the atelet container | +| `podMonitors.enabled` | `false` | Render a PodMonitor that scrapes atelet's `/metrics` | +| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus | + Object storage is set under `objectStore` instead of `rustfs` and `atelet.storageBackend`: diff --git a/charts/substrate-atelet/templates/atelet.yaml b/charts/substrate-atelet/templates/atelet.yaml index c8374bbf94..0d8f89aa22 100644 --- a/charts/substrate-atelet/templates/atelet.yaml +++ b/charts/substrate-atelet/templates/atelet.yaml @@ -308,6 +308,9 @@ spec: subPath: config.yaml {{- end }} readOnly: true +{{- end }} +{{- with .Values.atelet.extraVolumeMounts }} +{{- toYaml . | nindent 8 }} {{- end }} volumes: - name: objectstore-plugin @@ -368,3 +371,21 @@ spec: matchLabels: podcert.ate.dev/canarying: live path: trust-bundle.pem +{{- with .Values.atelet.extraVolumes }} +{{- toYaml . | nindent 6 }} +{{- end }} +{{- if .Values.podMonitors.enabled }} +--- +apiVersion: {{ .Values.podMonitors.apiVersion }} +kind: PodMonitor +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: atelet + podMetricsEndpoints: + - port: prometheus + path: /metrics +{{- end }} diff --git a/charts/substrate-atelet/values.yaml b/charts/substrate-atelet/values.yaml index b68515745e..684cee37c6 100644 --- a/charts/substrate-atelet/values.yaml +++ b/charts/substrate-atelet/values.yaml @@ -61,6 +61,10 @@ atelet: nodeSelector: {} extraArgs: [] extraEnv: [] + # Added to the atelet pod's volumes and its container's volumeMounts, such + # as a host file a credential provider reads. + extraVolumes: [] + extraVolumeMounts: [] resources: requests: cpu: 50m @@ -94,6 +98,13 @@ otel: enabled: true endpoint: "" +# A PodMonitor that scrapes atelet's /metrics endpoint. It needs the +# Prometheus Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure +# Managed Prometheus. +podMonitors: + enabled: false + apiVersion: monitoring.coreos.com/v1 + # Values under `global` are visible to this chart and to every subchart. global: # -- Mirror registry that overrides where every image is pulled from. It wins diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 35d7bc96c8..79abefe720 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -89,6 +89,7 @@ See `values.yaml` for the full set; the important keys: | `atenetEgress.extraCredentialProviders` | `[]` | Credential providers you deploy, added after the bundled `k8s.io` provider, each `{name, address}`. `name` is the host of the `ate-secret://` URIs it serves; `address` is the `host:port` the egress gateway dials over mTLS. Its serving certificate must chain to the servicedns CA. See [egress credential injection](../../docs/guides/egress-credential-injection.md#the-plugin-api) | | `atenetEgress.agentgateway.logLevel` | `info` | Egress proxy `RUST_LOG` filter, e.g. `debug` or `info,agentgateway=debug` when troubleshooting | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `rustfs.storageClassName` | `""` | StorageClass of the RustFS data volume; empty uses the cluster's default | | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs:///` with bundled RustFS | | `atelet.enabled` | `true` | Deprecated. Set to `false` and install atelet with the [substrate-atelet](../substrate-atelet/README.md) chart | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | @@ -112,6 +113,8 @@ See `values.yaml` for the full set; the important keys: | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateworkers, plus the router access log. Actor events go to stdout when OTLP logs are disabled | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | +| `podMonitors.enabled` | `false` | Render PodMonitors that scrape `/metrics` on ate-api-server, ate-controller, atenet-router and atenet-egress; ate-controller's only while `otel.metrics.enabled`, since it serves no `/metrics` otherwise | +| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus | ## Router timeouts diff --git a/charts/substrate/templates/podmonitors.yaml b/charts/substrate/templates/podmonitors.yaml new file mode 100644 index 0000000000..ebfe5d4c52 --- /dev/null +++ b/charts/substrate/templates/podmonitors.yaml @@ -0,0 +1,38 @@ +{{/* +Copyright 2026 The Agent Substrate Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.podMonitors.enabled }} +{{- $targets := list (list "ate-api-server" "prometheus") (list "atenet-router" "gw-metrics") (list "atenet-egress" "stats") }} +{{- /* ate-controller closes its metrics port when OTEL_METRICS_EXPORTER is none. */}} +{{- if .Values.otel.metrics.enabled }} +{{- $targets = append $targets (list "ate-controller" "metrics") }} +{{- end }} +{{- range $targets }} +--- +apiVersion: {{ $.Values.podMonitors.apiVersion }} +kind: PodMonitor +metadata: + name: {{ include "substrate.fullname" (list (index . 0) $) }} + namespace: {{ $.Release.Namespace }} +spec: + selector: + matchLabels: + app: {{ index . 0 }} + podMetricsEndpoints: + - port: {{ index . 1 }} + path: /metrics +{{- end }} +{{- end }} diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index 95a5b10053..d8d39a6514 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -32,6 +32,9 @@ metadata: spec: accessModes: - ReadWriteOnce + {{- with .Values.rustfs.storageClassName }} + storageClassName: {{ . | quote }} + {{- end }} resources: requests: storage: {{ .Values.rustfs.storageSize }} diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 87646b96fc..fb8e9abf0e 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -265,6 +265,25 @@ tests: value: - protocol: TCP port: api +- it: leaves the RustFS volume on the default StorageClass + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - notExists: + path: spec.storageClassName +- it: puts the RustFS volume on rustfs.storageClassName + template: rustfs.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + set: + rustfs.storageClassName: managed-csi-premium + asserts: + - equal: + path: spec.storageClassName + value: managed-csi-premium - it: supplies the required API config and bootstraps the release client template: ate-api-server.yaml diff --git a/charts/substrate/tests/podmonitors_test.yaml b/charts/substrate/tests/podmonitors_test.yaml new file mode 100644 index 0000000000..b008145705 --- /dev/null +++ b/charts/substrate/tests/podmonitors_test.yaml @@ -0,0 +1,83 @@ +# Copyright 2026 The Agent Substrate Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +suite: PodMonitors +templates: +- podmonitors.yaml +release: + name: custom + namespace: custom-system +tests: +- it: renders no PodMonitors by default + template: podmonitors.yaml + asserts: + - hasDocuments: + count: 0 +- it: scrapes each control plane component's metrics port + template: podmonitors.yaml + set: + podMonitors: + enabled: true + apiVersion: azmonitoring.coreos.com/v1 + asserts: + - hasDocuments: + count: 4 + - isAPIVersion: + of: azmonitoring.coreos.com/v1 + - isKind: + of: PodMonitor + - equal: + path: metadata.namespace + value: custom-system + - equal: + path: metadata.name + value: custom-ate-api-server + documentIndex: 0 + - equal: + path: spec + value: + selector: + matchLabels: + app: ate-api-server + podMetricsEndpoints: + - port: prometheus + path: /metrics + documentIndex: 0 + - equal: + path: spec.podMetricsEndpoints[0].port + value: gw-metrics + documentIndex: 1 + - equal: + path: spec.podMetricsEndpoints[0].port + value: stats + documentIndex: 2 + - equal: + path: spec.selector.matchLabels.app + value: ate-controller + documentIndex: 3 + - equal: + path: spec.podMetricsEndpoints[0].port + value: metrics + documentIndex: 3 +- it: skips ate-controller when it serves no metrics + template: podmonitors.yaml + set: + podMonitors.enabled: true + otel.metrics.enabled: false + asserts: + - hasDocuments: + count: 3 + - notEqual: + path: spec.selector.matchLabels.app + value: ate-controller diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 5d0f7a4dce..e4bb553973 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -47,6 +47,8 @@ postgres: rustfs: enabled: true storageSize: 1Gi + # StorageClass of the data volume. Empty uses the cluster's default. + storageClassName: "" bucket: ate-snapshots accessKey: rustfsadmin secretKey: rustfsadmin @@ -229,6 +231,14 @@ otel: enabled: true endpoint: "" +# PodMonitors that scrape the control plane's /metrics endpoints. The +# substrate-atelet chart has its own for atelet. They need the Prometheus +# Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure Managed +# Prometheus. +podMonitors: + enabled: false + apiVersion: monitoring.coreos.com/v1 + # Values under `global` are visible to this chart and to every subchart. A parent # chart or an operator sets one value here instead of one value per chart. global: diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go index 63fce1ddf7..47b3dd69de 100644 --- a/cmd/atelet/helm_test.go +++ b/cmd/atelet/helm_test.go @@ -30,7 +30,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/utils/ptr" "sigs.k8s.io/yaml" ) @@ -145,6 +147,60 @@ providers: []` } } +func TestHelmAteletExtraVolumes(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet", + "--set-json", `atelet.extraVolumes=[{"name": "extra", "hostPath": {"path": "/etc/extra.json", "type": "File"}}]`, + "--set-json", `atelet.extraVolumeMounts=[{"name": "extra", "mountPath": "/etc/extra.json", "readOnly": true}]`) + for _, obj := range objs { + if obj.GetKind() != "DaemonSet" { + continue + } + var ds appsv1.DaemonSet + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &ds); err != nil { + t.Fatal(err) + } + pod := ds.Spec.Template.Spec + wantVolume := corev1.Volume{Name: "extra", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/extra.json", Type: ptr.To(corev1.HostPathFile)}}} + if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return reflect.DeepEqual(v, wantVolume) }) { + t.Errorf("atelet volumes %v don't include %v", pod.Volumes, wantVolume) + } + wantMount := corev1.VolumeMount{Name: "extra", MountPath: "/etc/extra.json", ReadOnly: true} + if !slices.ContainsFunc(pod.Containers[0].VolumeMounts, func(m corev1.VolumeMount) bool { return reflect.DeepEqual(m, wantMount) }) { + t.Errorf("atelet volume mounts %v don't include %v", pod.Containers[0].VolumeMounts, wantMount) + } + return + } + t.Fatal("no atelet DaemonSet rendered") +} + +func TestHelmAteletPodMonitor(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet", + "--set", "podMonitors.enabled=true", "--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1") + for _, obj := range objs { + if obj.GetKind() != "PodMonitor" { + continue + } + if got := obj.GetAPIVersion(); got != "azmonitoring.coreos.com/v1" { + t.Errorf("PodMonitor apiVersion = %q, want azmonitoring.coreos.com/v1", got) + } + want := map[string]any{ + "selector": map[string]any{"matchLabels": map[string]any{"app": "atelet"}}, + "podMetricsEndpoints": []any{map[string]any{"port": "prometheus", "path": "/metrics"}}, + } + if diff := cmp.Diff(want, obj.Object["spec"]); diff != "" { + t.Errorf("PodMonitor spec mismatch (-want +got):\n%s", diff) + } + return + } + t.Fatal("no PodMonitor rendered") +} + func TestHelmSandboxAssetStorage(t *testing.T) { for _, tc := range []struct { name, backend string @@ -285,6 +341,8 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) { "--set", "preview[0]=ALL", "--set", "otel.endpoint=http://collector:4317", "--set", "atelet.nodeSelector.pool=agents", + "--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml", + "--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider", "--set", "image.tag=test", }, substrate: append([]string{"--set-json", `objectStorePlugin.extraEnv=[ diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go index 2b21932eec..888c4aa459 100644 --- a/cmd/kubectl-ate/internal/cmd/install.go +++ b/cmd/kubectl-ate/internal/cmd/install.go @@ -34,6 +34,7 @@ import ( var ( configPath string + profile string chartDir string noTUI bool upgradeDryRun bool @@ -43,6 +44,12 @@ var ( const configHelp = `--config takes a YAML file, or - for stdin: + profile: aks # or generic + aks: # for profile: aks + acr: my.azurecr.io # pull the images from it + nodePool: agents # run atelet only in this pool + acrCredentialProvider: true # pull actor images from ACR + managedPrometheus: true # scrape metrics with Azure Managed Prometheus postgres: type: bundled # or external objectStore: @@ -58,6 +65,8 @@ const configHelp = `--config takes a YAML file, or - for stdin: microvm: enabled: true assetBucket: my-assets # for s3 or gcs + storageClass: fast # bundled volumes; empty for the default + otelEndpoint: http://otel-collector:4317 # your OTLP collector timeout: 10m # each wait for workloads releases: # Helm values all: # every release except substrate-crds @@ -71,7 +80,8 @@ const configHelp = `--config takes a YAML file, or - for stdin: substrate: # substrate values: otel: - endpoint: http://otel-collector:4317 + traces: + samplingRatio: 0.1 A release's own valuesFiles and values win over the ones in all.` @@ -93,10 +103,45 @@ described in charts/substrate/README.md and set postgres.type to external. To store snapshots in a bucket, set objectStore.type to s3 or gcs and set objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount. +storageClass sets the StorageClass of the bundled PostgreSQL and RustFS +volumes when they are created. On AKS, managed-csi-premium uses Premium SSDs. + +otelEndpoint points atelet and the control plane at an OTLP collector you +already run, for traces, metrics and logs. Install doesn't deploy one. + +profile selects the kind of cluster, and --profile sets it from the command +line. generic makes no assumptions. aks is for Azure Kubernetes Service. It +checks that a NetworkPolicy engine is installed, since AKS ignores +NetworkPolicy without one. With aks.acr set, install pulls every image it installs, including the +bundled PostgreSQL's, from that Azure Container Registry. Import the images +under their usual paths with az acr import and attach the registry with +az aks update --attach-acr first. + +aks.nodePool runs atelet only on the nodes of that pool. Give each WorkerPool +the nodeSelector kubernetes.azure.com/agentpool with the pool's name. With +microvm.enabled, install checks that atelet's nodes have /dev/kvm, which needs +a generation 2 VM size with nested virtualization. + +aks.acrCredentialProvider lets atelet pull actor images from Azure Container +Registry. atelet runs the node's acr-credential-provider, which reads the +node's /etc/kubernetes/azure.json, so atelet can pull from every registry the +kubelet identity can. Grant that identity AcrPull, or attach the registry. On +a cluster that uses a service principal instead of a managed identity, that +file holds the service principal's secret, so atelet holds it too. With +profile aks and the option off, install and upgrade reset atelet's +imageCredentialProvider values, unless the config's releases set them. + +aks.managedPrometheus creates PodMonitors so Azure Managed Prometheus scrapes +the metrics of atelet and the control plane. Enable it on the cluster first +with az aks update --enable-azure-monitor-metrics. + microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each node architecture, caches them under the user cache directory, uploads them to the object store, and applies the microvm SandboxConfig. Upgrade restages them. +Each release keeps the values an earlier install set, so turning an option off +doesn't undo it. To undo it, set the chart values under releases. + ` + configHelp, Example: ` kubectl ate install -c install.yaml kubectl ate install -c - < 0 { + return fmt.Errorf("invalid node pool %q: %s", s, strings.Join(errs, "; ")) + } + return nil +} + +// ValidateStorageClass checks that s is empty or a valid StorageClass name. +func ValidateStorageClass(s string) error { + if s == "" { + return nil + } + if errs := validation.IsDNS1123Subdomain(s); len(errs) > 0 { + return fmt.Errorf("invalid StorageClass name %q: %s", s, strings.Join(errs, "; ")) + } + return nil +} + +// ValidateOTelEndpoint checks that s is empty or an http or https URL. +func ValidateOTelEndpoint(s string) error { + if s == "" { + return nil + } + u, err := url.Parse(s) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return fmt.Errorf("invalid OTLP endpoint %q: must be a URL like http://otel-collector:4317", s) + } + return nil +} + func (c Config) validate() error { + if err := validateProfile(c.Profile); err != nil { + return err + } + if c.AKS != (AKS{}) && c.Profile != ProfileAKS { + return fmt.Errorf("aks needs profile: %s", ProfileAKS) + } + if err := ValidateACR(c.AKS.ACR); err != nil { + return fmt.Errorf("aks.acr: %w", err) + } + if err := ValidateNodePool(c.AKS.NodePool); err != nil { + return fmt.Errorf("aks.nodePool: %w", err) + } switch c.Postgres.Type { case PostgresBundled, PostgresExternal: default: @@ -214,6 +339,17 @@ func (c Config) validate() error { if err := c.ObjectStore.validate(); err != nil { return err } + if c.StorageClass != "" { + if err := ValidateStorageClass(c.StorageClass); err != nil { + return fmt.Errorf("storageClass: %w", err) + } + if c.Postgres.Type != PostgresBundled && c.ObjectStore.Type != ObjectStoreBundled { + return fmt.Errorf("storageClass needs postgres.type: %s or objectStore.type: %s", PostgresBundled, ObjectStoreBundled) + } + } + if err := ValidateOTelEndpoint(c.OTelEndpoint); err != nil { + return fmt.Errorf("otelEndpoint: %w", err) + } if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" { return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket") } @@ -223,12 +359,97 @@ func (c Config) validate() error { return nil } +func validateProfile(p Profile) error { + if !slices.Contains(Profiles, p) { + return fmt.Errorf("invalid profile %q: must be one of %v", p, Profiles) + } + return nil +} + +// imageRegistry returns the registry to pull every image kubectl ate installs +// from, or "" to keep each image's own. +func (c Config) imageRegistry() string { + if c.Profile == ProfileAKS { + return c.AKS.ACR + } + return "" +} + +// aksNodePoolLabel is the node label AKS sets to the node's pool name. +const aksNodePoolLabel = "kubernetes.azure.com/agentpool" + +// The kubelet's credential provider files on AKS nodes. The provider reads +// the kubelet identity from aksCloudConfig. +const ( + aksCredentialProviderConfig = "/var/lib/kubelet/credential-provider-config.yaml" + aksCredentialProviderBinDir = "/var/lib/kubelet/credential-provider" + aksCloudConfig = "/etc/kubernetes/azure.json" + aksCloudConfigVolume = "aks-cloud-config" +) + +// nodeSelector returns the node selector atelet runs with, or nil for every +// node. +func (c Config) nodeSelector() map[string]string { + if c.Profile == ProfileAKS && c.AKS.NodePool != "" { + return map[string]string{aksNodePoolLabel: c.AKS.NodePool} + } + return nil +} + +// profileValues returns the profile's Helm values for r. +func (c Config) profileValues(r release) map[string]any { + vals := map[string]any{} + if reg := c.imageRegistry(); reg != "" { + vals["global"] = map[string]any{"imageRegistry": reg} + } + if c.Profile == ProfileAKS && c.AKS.ManagedPrometheus && (r == ateletRelease || r == substrateRelease) { + vals["podMonitors"] = map[string]any{"enabled": true, "apiVersion": managedPrometheusAPI} + } + if r == ateletRelease { + atelet := map[string]any{} + if sel := c.nodeSelector(); sel != nil { + nodeSelector := map[string]any{} + for k, v := range sel { + nodeSelector[k] = v + } + atelet["nodeSelector"] = nodeSelector + } + if c.Profile == ProfileAKS { + // Off resets the chart defaults, so atelet loses the kubelet + // identity's registry access that an earlier install gave it. + // userValues adds the cloud config mount when it's on. + config, binDir := "", "" + if c.AKS.ACRCredentialProvider { + config, binDir = aksCredentialProviderConfig, aksCredentialProviderBinDir + } + atelet["imageCredentialProviderConfig"] = config + atelet["imageCredentialProviderConfigYAML"] = "" + atelet["imageCredentialProviderBinDir"] = binDir + atelet["extraVolumes"] = []any{} + atelet["extraVolumeMounts"] = []any{} + } + if len(atelet) > 0 { + vals["atelet"] = atelet + } + } + return vals +} + func (r *Releases) all() []*ReleaseValues { return []*ReleaseValues{&r.All, &r.PodCert, &r.Atelet, &r.Substrate} } -// userValues returns each release's values, keyed by release name: All's -// files and values, then the release's own. +// otelValues returns the Helm values that point r at OTelEndpoint. +func (c Config) otelValues(r release) map[string]any { + if c.OTelEndpoint == "" || (r != ateletRelease && r != substrateRelease) { + return nil + } + return map[string]any{"otel": map[string]any{"endpoint": c.OTelEndpoint}} +} + +// userValues returns each release's values, keyed by release name: the +// profile's and OTelEndpoint's, All's files and values, then the release's +// own. func (c Config) userValues() (map[string]map[string]any, error) { all, err := c.Releases.All.read() if err != nil { @@ -244,11 +465,37 @@ func (c Config) userValues() (map[string]map[string]any, error) { if err != nil { return nil, err } - user[r.name] = mergeValues(all, own) + user[r.name] = mergeValues(c.profileValues(r), c.otelValues(r), all, own) } + c.addCloudConfigMount(user[ateletRelease.name]) return user, nil } +// addCloudConfigMount mounts the AKS cloud config, which the ACR credential +// provider reads, into atelet. It appends to the release's own extraVolumes +// and extraVolumeMounts, since lists replace in a merge. +func (c Config) addCloudConfigMount(vals map[string]any) { + if c.Profile != ProfileAKS || !c.AKS.ACRCredentialProvider { + return + } + atelet, ok := vals["atelet"].(map[string]any) + if !ok { + atelet = map[string]any{} + vals["atelet"] = atelet + } + volumes, _ := atelet["extraVolumes"].([]any) + mounts, _ := atelet["extraVolumeMounts"].([]any) + atelet["extraVolumes"] = append(slices.Clone(volumes), map[string]any{ + "name": aksCloudConfigVolume, + "hostPath": map[string]any{"path": aksCloudConfig, "type": "File"}, + }) + atelet["extraVolumeMounts"] = append(slices.Clone(mounts), map[string]any{ + "name": aksCloudConfigVolume, + "mountPath": aksCloudConfig, + "readOnly": true, + }) +} + func (rv ReleaseValues) read() (map[string]any, error) { vo := values.Options{ValueFiles: rv.ValuesFiles} files, err := vo.MergeValues(nil) diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go index d3094e3fab..db297d5c64 100644 --- a/pkg/kagent/install/config_test.go +++ b/pkg/kagent/install/config_test.go @@ -34,6 +34,7 @@ func TestReadConfig(t *testing.T) { tests := []struct { name string in string + profile Profile want Config wantErr string }{ @@ -67,7 +68,39 @@ releases: c.Releases.Substrate.ValuesFiles = []string{"/cfg/db.yaml"} }), }, + { + name: "aks", + in: "profile: aks\naks: {acr: r.azurecr.io}\n", + want: withDefaults(func(c *Config) { + c.Profile, c.AKS.ACR = ProfileAKS, "r.azurecr.io" + }), + }, + { + name: "profile argument sets profile", + in: "aks: {acr: r.azurecr.io}\n", + profile: ProfileAKS, + want: withDefaults(func(c *Config) { + c.Profile, c.AKS.ACR = ProfileAKS, "r.azurecr.io" + }), + }, + {name: "storage class", in: "storageClass: managed-csi-premium\n", want: withDefaults(func(c *Config) { c.StorageClass = "managed-csi-premium" })}, + {name: "bad storage class", in: "storageClass: Premium_SSD\n", wantErr: "invalid StorageClass name"}, + {name: "storage class without bundled volumes", in: "postgres: external\nobjectStore: s3\nstorageClass: fast\n", wantErr: "storageClass needs"}, + {name: "profile argument matches", in: "profile: aks\n", profile: ProfileAKS, want: withDefaults(func(c *Config) { c.Profile = ProfileAKS })}, {name: "empty", in: "", wantErr: "config is empty"}, + {name: "bad profile", in: "profile: eks\n", wantErr: "invalid profile"}, + {name: "bad profile argument", in: "{}", profile: "eks", wantErr: "invalid profile"}, + {name: "profile argument conflicts", in: "profile: aks\n", profile: ProfileGeneric, wantErr: "conflicts"}, + {name: "aks needs profile", in: "aks: {acr: r.azurecr.io}\n", wantErr: "aks needs profile: aks"}, + {name: "acr with path", in: "profile: aks\naks: {acr: r.azurecr.io/substrate}\n", wantErr: "invalid ACR"}, + {name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"}, + {name: "node pool", in: "profile: aks\naks: {nodePool: agents}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.NodePool = ProfileAKS, "agents" })}, + {name: "acr credential provider", in: "profile: aks\naks: {acrCredentialProvider: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ACRCredentialProvider = ProfileAKS, true })}, + {name: "managed prometheus", in: "profile: aks\naks: {managedPrometheus: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ManagedPrometheus = ProfileAKS, true })}, + {name: "otel endpoint", in: "otelEndpoint: http://collector:4317\n", want: withDefaults(func(c *Config) { c.OTelEndpoint = "http://collector:4317" })}, + {name: "otel endpoint without scheme", in: "otelEndpoint: collector:4317\n", wantErr: "invalid OTLP endpoint"}, + {name: "bad node pool", in: "profile: aks\naks: {nodePool: 'agents pool'}\n", wantErr: "invalid node pool"}, + {name: "acr with space", in: "profile: aks\naks: {acr: 'r.azurecr.io '}\n", wantErr: "invalid ACR"}, {name: "unknown field", in: "postgress: external\n", wantErr: "postgress"}, {name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"}, {name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"}, @@ -104,7 +137,7 @@ releases: } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got, err := ReadConfig(strings.NewReader(tc.in), "/cfg") + got, err := ReadConfig(strings.NewReader(tc.in), "/cfg", tc.profile) if tc.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { t.Fatalf("ReadConfig() error = %v, want it to contain %q", err, tc.wantErr) @@ -157,3 +190,118 @@ func TestConfigUserValues(t *testing.T) { t.Errorf("userValues() mismatch (-want +got):\n%s", diff) } } + +var ( + cloudConfigVolume = map[string]any{"name": "aks-cloud-config", "hostPath": map[string]any{"path": "/etc/kubernetes/azure.json", "type": "File"}} + cloudConfigMount = map[string]any{"name": "aks-cloud-config", "mountPath": "/etc/kubernetes/azure.json", "readOnly": true} +) + +func TestConfigUserValuesProfile(t *testing.T) { + registry := func(r string) map[string]any { + return map[string]any{"global": map[string]any{"imageRegistry": r}} + } + otel := func(endpoint string) map[string]any { + return map[string]any{"endpoint": endpoint} + } + substrateValues := registry("mirror.example.com") + substrateValues["otel"] = otel("http://other:4317") + c := Config{ + Profile: ProfileAKS, + AKS: AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true, ManagedPrometheus: true}, + OTelEndpoint: "http://collector:4317", + Releases: Releases{ + Atelet: ReleaseValues{Values: map[string]any{"atelet": map[string]any{"nodeSelector": map[string]any{"disk": "ssd"}}}}, + Substrate: ReleaseValues{Values: substrateValues}, + }, + } + got, err := c.userValues() + if err != nil { + t.Fatal(err) + } + // The release's own values win over the profile's and otelEndpoint's. + // Only atelet gets the node pool, and podcert gets no otel or + // PodMonitors. + podMonitors := map[string]any{"enabled": true, "apiVersion": "azmonitoring.coreos.com/v1"} + atelet := registry("r.azurecr.io") + atelet["otel"] = otel("http://collector:4317") + atelet["podMonitors"] = podMonitors + atelet["atelet"] = map[string]any{ + "nodeSelector": map[string]any{ + "kubernetes.azure.com/agentpool": "agents", + "disk": "ssd", + }, + "imageCredentialProviderConfig": "/var/lib/kubelet/credential-provider-config.yaml", + "imageCredentialProviderConfigYAML": "", + "imageCredentialProviderBinDir": "/var/lib/kubelet/credential-provider", + "extraVolumes": []any{cloudConfigVolume}, + "extraVolumeMounts": []any{cloudConfigMount}, + } + substrate := registry("mirror.example.com") + substrate["otel"] = otel("http://other:4317") + substrate["podMonitors"] = podMonitors + want := map[string]map[string]any{ + "substrate-podcert": registry("r.azurecr.io"), + "substrate-atelet": atelet, + "substrate": substrate, + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("userValues() mismatch (-want +got):\n%s", diff) + } +} + +func TestConfigUserValuesACRCredentialProvider(t *testing.T) { + provider := func(config, binDir string, volumes, mounts []any) map[string]any { + return map[string]any{ + "imageCredentialProviderConfig": config, + "imageCredentialProviderConfigYAML": "", + "imageCredentialProviderBinDir": binDir, + "extraVolumes": volumes, + "extraVolumeMounts": mounts, + } + } + const config, binDir = "/var/lib/kubelet/credential-provider-config.yaml", "/var/lib/kubelet/credential-provider" + userVolume := map[string]any{"name": "extra", "emptyDir": map[string]any{}} + userMount := map[string]any{"name": "extra", "mountPath": "/extra"} + for _, tc := range []struct { + name string + c Config + want map[string]any + }{ + {name: "generic leaves it alone", c: Config{Profile: ProfileGeneric}}, + { + name: "aks on", + c: Config{Profile: ProfileAKS, AKS: AKS{ACRCredentialProvider: true}}, + want: provider(config, binDir, []any{cloudConfigVolume}, []any{cloudConfigMount}), + }, + { + name: "aks on keeps the release's own volumes", + c: Config{Profile: ProfileAKS, AKS: AKS{ACRCredentialProvider: true}, Releases: Releases{Atelet: ReleaseValues{Values: map[string]any{ + "atelet": map[string]any{"extraVolumes": []any{userVolume}, "extraVolumeMounts": []any{userMount}}, + }}}}, + want: provider(config, binDir, []any{userVolume, cloudConfigVolume}, []any{userMount, cloudConfigMount}), + }, + { + name: "aks off resets it", + c: Config{Profile: ProfileAKS}, + want: provider("", "", []any{}, []any{}), + }, + { + name: "aks off keeps the release's own", + c: Config{Profile: ProfileAKS, Releases: Releases{Atelet: ReleaseValues{Values: map[string]any{ + "atelet": provider("/etc/provider.yaml", "/opt/provider", []any{userVolume}, []any{userMount}), + }}}}, + want: provider("/etc/provider.yaml", "/opt/provider", []any{userVolume}, []any{userMount}), + }, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := tc.c.userValues() + if err != nil { + t.Fatal(err) + } + atelet, _ := got[ateletRelease.name]["atelet"].(map[string]any) + if diff := cmp.Diff(tc.want, atelet); diff != "" { + t.Errorf("atelet values mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/kagent/install/install_test.go b/pkg/kagent/install/install_test.go index 245cca59f8..fef8a0aab9 100644 --- a/pkg/kagent/install/install_test.go +++ b/pkg/kagent/install/install_test.go @@ -24,9 +24,11 @@ import ( "testing" chart "helm.sh/helm/v4/pkg/chart/v2" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/ptr" chartsfs "go.ate.dev/substrate/charts" ) @@ -180,7 +182,7 @@ func secretData(t *testing.T, kc *fake.Clientset) map[string]any { func TestApplyPostgresManifest(t *testing.T) { ctx := context.Background() - manifest, err := postgresManifest() + manifest, err := postgresManifest(Config{}) if err != nil { t.Fatal(err) } @@ -204,6 +206,127 @@ func TestApplyPostgresManifest(t *testing.T) { } } +func postgresStatefulSetFor(t *testing.T, c Config) *appsv1.StatefulSet { + t.Helper() + ctx := context.Background() + manifest, err := postgresManifest(c) + if err != nil { + t.Fatal(err) + } + kc := fake.NewClientset() + if err := applyManifest(ctx, kc, manifest); err != nil { + t.Fatal(err) + } + ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return ss +} + +func TestPostgresManifestRegistry(t *testing.T) { + images := func(c Config) []string { + t.Helper() + ss := postgresStatefulSetFor(t, c) + var out []string + for _, c := range append(ss.Spec.Template.Spec.InitContainers, ss.Spec.Template.Spec.Containers...) { + out = append(out, c.Image) + } + return out + } + base, mirrored := images(Config{}), images(Config{Profile: ProfileAKS, AKS: AKS{ACR: "r.azurecr.io"}}) + if len(base) == 0 || len(base) != len(mirrored) { + t.Fatalf("got images %v and %v, want the same nonzero count", base, mirrored) + } + // The digest stays, so the mirror must hold the same image. + for i := range base { + if want := "r.azurecr.io/" + base[i]; mirrored[i] != want { + t.Errorf("image %d = %s, want %s", i, mirrored[i], want) + } + } +} + +func TestPostgresManifestStorageClass(t *testing.T) { + for _, sc := range []string{"", "managed-csi-premium"} { + ss := postgresStatefulSetFor(t, Config{StorageClass: sc}) + vct := ss.Spec.VolumeClaimTemplates[0].Spec + if got := ptr.Deref(vct.StorageClassName, ""); got != sc { + t.Errorf("StorageClass %q: storageClassName = %q", sc, got) + } + // The overlay keeps the kind overlay's patches. + if got := vct.Resources.Requests.Storage().String(); got != "1Gi" { + t.Errorf("StorageClass %q: volume size = %s, want 1Gi", sc, got) + } + } +} + +func TestPostgresStorageClass(t *testing.T) { + for _, tc := range []struct { + name string + existing *string + want string + }{ + {name: "new install uses the config's", want: "managed-csi-premium"}, + {name: "existing keeps the default", existing: ptr.To(""), want: ""}, + {name: "existing keeps its own", existing: ptr.To("managed-csi"), want: "managed-csi"}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := fake.NewClientset() + if tc.existing != nil { + ss := postgresStatefulSetFor(t, Config{StorageClass: *tc.existing}) + if _, err := kc.AppsV1().StatefulSets(SystemNamespace).Create(t.Context(), ss, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + got, err := postgresStorageClass(t.Context(), kc, "managed-csi-premium") + if err != nil { + t.Fatal(err) + } + if got != tc.want { + t.Errorf("postgresStorageClass() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestSubstrateValuesStorageClass(t *testing.T) { + kc := fake.NewClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace, Name: rustfsCredentialsSecret}, + Data: map[string][]byte{"AWS_ACCESS_KEY_ID": []byte("a"), "AWS_SECRET_ACCESS_KEY": []byte("s")}, + }) + in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreBundled}, Postgres: Postgres{Type: PostgresBundled}, StorageClass: "fast"}}} + defaults, _, err := in.substrateValues(context.Background()) + if err != nil { + t.Fatal(err) + } + want := map[string]any{"rustfs": map[string]any{"storageClassName": "fast"}} + if !reflect.DeepEqual(defaults, want) { + t.Errorf("substrate defaults = %v, want %v", defaults, want) + } +} + +func TestCheckPodCertificateAPI(t *testing.T) { + pcr := []metav1.APIResource{{Name: "podcertificaterequests"}} + for _, tc := range []struct { + name string + resources []*metav1.APIResourceList + wantErr bool + }{ + {name: "v1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: pcr}}}, + {name: "v1beta1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1beta1", APIResources: pcr}}}, + {name: "v1 without podcertificaterequests", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}}}, wantErr: true}, + {name: "neither", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + kc := fake.NewClientset() + kc.Resources = tc.resources + if err := checkPodCertificateAPI(&Installer{kube: kc}); (err != nil) != tc.wantErr { + t.Errorf("checkPodCertificateAPI() = %v, wantErr %v", err, tc.wantErr) + } + }) + } +} + func TestCheckExternalObjectStore(t *testing.T) { s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"} gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}} diff --git a/pkg/kagent/install/installer.go b/pkg/kagent/install/installer.go index 45ec7df55c..8a8c1a9033 100644 --- a/pkg/kagent/install/installer.go +++ b/pkg/kagent/install/installer.go @@ -23,6 +23,7 @@ import ( "time" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -182,6 +183,9 @@ func (in *Installer) InstallSteps() ([]Step, error) { if in.opts.Postgres.Type == PostgresBundled { in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres.type to external with a managed database for anything else."}) } + if sel := in.opts.nodeSelector(); sel != nil { + in.report.Report(Event{Kind: Note, Msg: fmt.Sprintf("atelet runs only on nodes labeled %s. Give each WorkerPool the same spec.template.nodeSelector, or its workers won't become ready.", labels.SelectorFromSet(sel))}) + } steps := []Step{ {"Check cluster", in.preflight}, {"Install CRDs", in.helmStep(crdsRelease, noValues)}, @@ -192,6 +196,9 @@ func (in *Installer) InstallSteps() ([]Step, error) { {"Configure object storage", in.configureObjectStore}, {"Install atelet", in.helmStep(ateletRelease, in.ateletValues)}, } + if in.opts.Profile == ProfileAKS && in.opts.MicroVM.Enabled { + steps = append(steps, Step{"Check KVM", in.checkKVM}) + } if in.opts.Postgres.Type == PostgresBundled { steps = append(steps, Step{"Install PostgreSQL", in.applyPostgres}, diff --git a/pkg/kagent/install/postgres.go b/pkg/kagent/install/postgres.go index 8cc002de24..755a0fd25a 100644 --- a/pkg/kagent/install/postgres.go +++ b/pkg/kagent/install/postgres.go @@ -26,6 +26,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -34,9 +35,12 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/remotecommand" + "k8s.io/utils/ptr" "sigs.k8s.io/kustomize/api/krusty" kusttypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/resid" + "sigs.k8s.io/yaml" ateinstall "go.ate.dev/substrate/manifests/ate-install" "go.ate.dev/substrate/pkg/postgressetup" @@ -57,7 +61,13 @@ const ( // applyPostgres applies manifests/ate-install/kind/postgres and waits for it. // That overlay sizes the database for evaluation rather than production. func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (string, error) { - manifest, err := postgresManifest() + c := in.opts.Config + sc, err := postgresStorageClass(ctx, in.kube, c.StorageClass) + if err != nil { + return "", err + } + c.StorageClass = sc + manifest, err := postgresManifest(c) if err != nil { return "", err } @@ -71,7 +81,24 @@ func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) ( return "statefulset/" + postgresStatefulSet + " ready", nil } -func postgresManifest() ([]byte, error) { +// postgresStorageClass returns the StorageClass to apply the bundled +// PostgreSQL with. An existing StatefulSet keeps its own, since its +// volumeClaimTemplates can't change. +func postgresStorageClass(ctx context.Context, kc kubernetes.Interface, sc string) (string, error) { + ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return sc, nil + } + if err != nil { + return "", err + } + return ptr.Deref(ss.Spec.VolumeClaimTemplates[0].Spec.StorageClassName, ""), nil +} + +// postgresManifest builds the bundled PostgreSQL manifest. c's image registry +// replaces the image's, the way the charts' global.imageRegistry does, and c's +// StorageClass sets the volume's. +func postgresManifest(c Config) ([]byte, error) { mem := filesys.MakeFsInMemory() err := fs.WalkDir(ateinstall.Postgres, ".", func(p string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { @@ -86,10 +113,42 @@ func postgresManifest() ([]byte, error) { if err != nil { return nil, err } + dir := "/kind/postgres" + overlay := kusttypes.Kustomization{ + TypeMeta: kusttypes.TypeMeta{APIVersion: kusttypes.KustomizationVersion, Kind: kusttypes.KustomizationKind}, + Resources: []string{"../kind/postgres"}, + } + if r := c.imageRegistry(); r != "" { + overlay.Images = []kusttypes.Image{{Name: "postgres", NewName: r + "/postgres"}} + } + if sc := c.StorageClass; sc != "" { + patch, err := yaml.Marshal([]map[string]any{{ + "op": "add", + "path": "/spec/volumeClaimTemplates/0/spec/storageClassName", + "value": sc, + }}) + if err != nil { + return nil, err + } + overlay.Patches = []kusttypes.Patch{{ + Target: &kusttypes.Selector{ResId: resid.NewResIdWithNamespace(resid.NewGvk("apps", "v1", "StatefulSet"), postgresStatefulSet, SystemNamespace)}, + Patch: string(patch), + }} + } + if overlay.Images != nil || overlay.Patches != nil { + k, err := yaml.Marshal(overlay) + if err != nil { + return nil, err + } + dir = "/overlay" + if err := mem.WriteFile(dir+"/kustomization.yaml", k); err != nil { + return nil, err + } + } opts := krusty.MakeDefaultOptions() // The overlay references its base outside its own directory. opts.LoadRestrictions = kusttypes.LoadRestrictionsNone - res, err := krusty.MakeKustomizer(opts).Run(mem, "/kind/postgres") + res, err := krusty.MakeKustomizer(opts).Run(mem, dir) if err != nil { return nil, fmt.Errorf("while building the PostgreSQL manifest: %w", err) } diff --git a/pkg/kagent/install/steps.go b/pkg/kagent/install/steps.go index 481255ef75..82a86ed18f 100644 --- a/pkg/kagent/install/steps.go +++ b/pkg/kagent/install/steps.go @@ -16,6 +16,7 @@ package install import ( "context" + "errors" "fmt" "time" @@ -48,6 +49,14 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri if err := checkPodCertificateAPI(in); err != nil { return "", err } + summary := "Kubernetes " + sv.GitVersion + if in.opts.Profile == ProfileAKS { + aks, err := in.checkAKS(ctx, progress) + if err != nil { + return "", err + } + summary += ", " + aks + } progress("checking for an existing install") rel, err := in.helm.get(substrateRelease) @@ -64,25 +73,27 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri return "", err } } - return "Kubernetes " + sv.GitVersion, nil + return summary, nil } // checkPodCertificateAPI checks that the cluster serves PodCertificateRequest, -// which the certificate controller signs. +// which the certificate controller signs, at v1 or v1beta1. func checkPodCertificateAPI(in *Installer) error { - const gv = "certificates.k8s.io/v1beta1" - res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv) - if err == nil { + for _, gv := range []string{"certificates.k8s.io/v1", "certificates.k8s.io/v1beta1"} { + res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return fmt.Errorf("while discovering %s: %w", gv, err) + } for _, r := range res.APIResources { if r.Name == "podcertificaterequests" { return nil } } } - if err != nil && !apierrors.IsNotFound(err) { - return fmt.Errorf("while discovering %s: %w", gv, err) - } - return fmt.Errorf("cluster doesn't serve %s podcertificaterequests; enable the PodCertificateRequest feature gate and API", gv) + return errors.New("cluster serves neither certificates.k8s.io/v1 nor v1beta1 podcertificaterequests; enable the PodCertificateRequest feature gate and API") } // valuesFunc returns installer defaults, which user values override, and @@ -191,12 +202,16 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s // The release keeps its storage and database values. return nil, forced, nil } + var defaults map[string]any if in.opts.ObjectStore.Type == ObjectStoreBundled { access, secret, err := in.rustfsCredentials(ctx) if err != nil { return nil, nil, err } forced["rustfs"] = map[string]any{"accessKey": access, "secretKey": secret} + if sc := in.opts.StorageClass; sc != "" { + defaults = map[string]any{"rustfs": map[string]any{"storageClassName": sc}} + } } else { _, substrate := in.externalStoreValues() forced = mergeValues(forced, substrate) @@ -204,7 +219,7 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s if in.opts.Postgres.Type == PostgresBundled { forced["postgres"] = map[string]any{"clientCertificates": map[string]any{"enabled": true}} } - return nil, forced, nil + return defaults, forced, nil } // externalStoreValues returns the values Config.ObjectStore sets on the diff --git a/pkg/kagent/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go index 84ca770fdd..cca4261a99 100644 --- a/pkg/kagent/install/tui/tui_test.go +++ b/pkg/kagent/install/tui/tui_test.go @@ -78,6 +78,13 @@ func TestHint(t *testing.T) { c.Releases.All.ValuesFiles = []string{"/etc/gke.yaml"} return c }(), + func() install.Config { + c := install.DefaultConfig() + c.Profile, c.AKS.ACR = install.ProfileAKS, "r.azurecr.io" + c.AKS.ManagedPrometheus = true + c.OTelEndpoint = "http://collector:4317" + return c + }(), } { hint, err := Hint(&cfg) if err != nil { @@ -87,7 +94,7 @@ func TestHint(t *testing.T) { if !ok { t.Fatalf("Hint() has no config:\n%s", hint) } - got, err := install.ReadConfig(strings.NewReader(doc), "") + got, err := install.ReadConfig(strings.NewReader(doc), "", "") if err != nil { t.Fatalf("ReadConfig(Hint()): %v\n%s", err, doc) } diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go index d169513f73..303dd530f0 100644 --- a/pkg/kagent/install/tui/wizard.go +++ b/pkg/kagent/install/tui/wizard.go @@ -27,13 +27,44 @@ import ( ) // Wizard asks for the install choices, stores them in cfg, and prints the -// matching config to w. It returns install.ErrAborted if the user declines or -// quits. -func Wizard(w io.Writer, cfg *install.Config) error { +// matching config to w. askProfile asks for the profile, otherwise cfg's is +// kept. It returns install.ErrAborted if the user declines or quits. +func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error { store := &cfg.ObjectStore external := func() bool { return store.Type != install.ObjectStoreBundled } + aks := func() bool { return cfg.Profile == install.ProfileAKS } + bundledVolumes := func() bool { return cfg.Postgres.Type == install.PostgresBundled || !external() } confirmed := true form := huh.NewForm( + huh.NewGroup( + huh.NewSelect[install.Profile](). + Title("Cluster"). + Options( + huh.NewOption("Any Kubernetes cluster", install.ProfileGeneric), + huh.NewOption("Azure Kubernetes Service (AKS)", install.ProfileAKS), + ). + Value(&cfg.Profile), + ).WithHideFunc(func() bool { return !askProfile }), + huh.NewGroup( + huh.NewInput(). + Title("Azure Container Registry"). + Description("Login server to pull Agent Substrate's images from, like myregistry.azurecr.io. It must hold\nthe images under their usual paths, and the cluster must be attached to it. Leave empty for the public registries."). + Value(&cfg.AKS.ACR). + Validate(install.ValidateACR), + huh.NewInput(). + Title("Node pool"). + Description("Runs atelet only in this pool. WorkerPools then need the nodeSelector\nkubernetes.azure.com/agentpool set to it. Leave empty for every node."). + Value(&cfg.AKS.NodePool). + Validate(install.ValidateNodePool), + huh.NewConfirm(). + Title("Pull actor images from ACR?"). + Description("Runs the node's acr-credential-provider in atelet. atelet reads the node's\n/etc/kubernetes/azure.json and gets every registry the kubelet identity can pull from.\nOn a service principal cluster, that file holds the service principal's secret."). + Value(&cfg.AKS.ACRCredentialProvider), + huh.NewConfirm(). + Title("Scrape metrics with Azure Managed Prometheus?"). + Description("Creates PodMonitors for atelet and the control plane.\nNeeds az aks update --enable-azure-monitor-metrics."). + Value(&cfg.AKS.ManagedPrometheus), + ).WithHideFunc(func() bool { return !aks() }), huh.NewGroup( huh.NewSelect[install.PostgresType](). Title("PostgreSQL"). @@ -82,6 +113,15 @@ func Wizard(w io.Writer, cfg *install.Config) error { Value(&store.GCS.TransferServiceAccount). Validate(notEmpty), ).WithHideFunc(func() bool { return store.Type != install.ObjectStoreGCS }), + huh.NewGroup( + huh.NewInput(). + Title("StorageClass"). + Description("For the bundled volumes. Leave empty for the cluster's default."). + PlaceholderFunc(func() string { return strings.Join(storageClassSuggestions(cfg.Profile), ", ") }, &cfg.Profile). + SuggestionsFunc(func() []string { return storageClassSuggestions(cfg.Profile) }, &cfg.Profile). + Value(&cfg.StorageClass). + Validate(install.ValidateStorageClass), + ).WithHideFunc(func() bool { return !bundledVolumes() }), huh.NewGroup( huh.NewInput(). Title("MicroVM asset bucket"). @@ -89,6 +129,13 @@ func Wizard(w io.Writer, cfg *install.Config) error { Value(&cfg.MicroVM.AssetBucket). Validate(notEmpty), ).WithHideFunc(func() bool { return !external() || !cfg.MicroVM.Enabled }), + huh.NewGroup( + huh.NewInput(). + Title("OTLP endpoint"). + Description("An OTLP collector you already run, for traces, metrics and logs, like\nhttp://otel-collector.observability:4317. Install doesn't deploy one. Leave empty for none."). + Value(&cfg.OTelEndpoint). + Validate(install.ValidateOTelEndpoint), + ), huh.NewGroup( huh.NewConfirm(). Title("Install Agent Substrate?"). @@ -119,6 +166,12 @@ func Wizard(w io.Writer, cfg *install.Config) error { if !external() || !cfg.MicroVM.Enabled { cfg.MicroVM.AssetBucket = "" } + if !bundledVolumes() { + cfg.StorageClass = "" + } + if !aks() { + cfg.AKS = install.AKS{} + } hint, err := Hint(cfg) if err != nil { return err @@ -136,6 +189,14 @@ func Hint(cfg *install.Config) (string, error) { return "To skip these questions next time, save this as install.yaml and run kubectl ate install -c install.yaml:\n\n" + string(data) + "\n", nil } +// storageClassSuggestions lists the StorageClasses to suggest on profile p. +func storageClassSuggestions(p install.Profile) []string { + if p == install.ProfileAKS { + return []string{"managed-csi-premium"} + } + return nil +} + func notEmpty(s string) error { if strings.TrimSpace(s) == "" { return errors.New("required") diff --git a/pkg/kagent/install/uninstall.go b/pkg/kagent/install/uninstall.go index 375ba5a228..6c279731af 100644 --- a/pkg/kagent/install/uninstall.go +++ b/pkg/kagent/install/uninstall.go @@ -217,7 +217,7 @@ func (in *Installer) uninstallStep(r release) func(context.Context, func(string) // removePostgres deletes the bundled PostgreSQL objects that install applied, // identified by kubectl ate's field manager. Its volume stays until --purge. func (in *Installer) removePostgres(ctx context.Context, _ func(string)) (string, error) { - manifest, err := postgresManifest() + manifest, err := postgresManifest(Config{}) if err != nil { return "", err } diff --git a/pkg/kagent/install/upgrade.go b/pkg/kagent/install/upgrade.go index e57eaf01d8..3822ce8f3e 100644 --- a/pkg/kagent/install/upgrade.go +++ b/pkg/kagent/install/upgrade.go @@ -55,10 +55,7 @@ type UpgradeOptions struct { // Upgrade moves every release to the charts in this binary. func (in *Installer) Upgrade(ctx context.Context, opts UpgradeOptions) error { in.upgrading = true - steps := []Step{ - {"Check versions", in.checkVersions}, - {"Check CRD storage versions", in.checkCRDStorage}, - } + steps := in.upgradeChecks() if opts.DryRun { for _, u := range upgradeOrder { steps = append(steps, Step{"Diff " + u.r.name, in.diffStep(u.r, in.valuesFor(u.r), opts.Out)}) @@ -89,6 +86,19 @@ func (in *Installer) Upgrade(ctx context.Context, opts UpgradeOptions) error { return in.Run(ctx, steps) } +// upgradeChecks returns the steps that check the cluster before Upgrade +// changes it. +func (in *Installer) upgradeChecks() []Step { + steps := []Step{ + {"Check versions", in.checkVersions}, + {"Check CRD storage versions", in.checkCRDStorage}, + } + if in.opts.Profile == ProfileAKS { + steps = append(steps, Step{"Check AKS", in.checkAKS}) + } + return steps +} + // hasMicroVMSandboxConfig reports whether the microvm SandboxConfig exists. func (in *Installer) hasMicroVMSandboxConfig(ctx context.Context) (bool, error) { _, err := in.dyn.Resource(sandboxConfigsGVR).Get(ctx, microVMSandboxConfig, metav1.GetOptions{}) diff --git a/pkg/kagent/install/upgrade_test.go b/pkg/kagent/install/upgrade_test.go index 852f840f19..f5da754489 100644 --- a/pkg/kagent/install/upgrade_test.go +++ b/pkg/kagent/install/upgrade_test.go @@ -20,6 +20,7 @@ import ( "strings" "testing" + "github.com/google/go-cmp/cmp" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -127,3 +128,22 @@ func TestUpgradeKeepsStorageValues(t *testing.T) { } } } + +func TestUpgradeChecks(t *testing.T) { + for _, tc := range []struct { + profile Profile + want []string + }{ + {profile: ProfileGeneric, want: []string{"Check versions", "Check CRD storage versions"}}, + {profile: ProfileAKS, want: []string{"Check versions", "Check CRD storage versions", "Check AKS"}}, + } { + in := &Installer{opts: Options{Config: Config{Profile: tc.profile}}} + var got []string + for _, s := range in.upgradeChecks() { + got = append(got, s.Name) + } + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("profile %s: upgradeChecks() mismatch (-want +got):\n%s", tc.profile, diff) + } + } +}