From 863f13a00a7e2cbf770772e22fa70e1e16f66805 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 07:38:01 -0500
Subject: [PATCH 01/10] kubectl-ate: add install profiles and an AKS registry
 option

Add a profile field to the install config and a --profile flag on
install, upgrade and uninstall. generic keeps the current behavior. aks
adds an aks section. Its acr field pulls every image install deploys from
an Azure Container Registry: it sets global.imageRegistry on the charts
and renames the bundled PostgreSQL image.

The wizard asks for the cluster type unless --profile is given.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 cmd/kubectl-ate/internal/cmd/install.go |  36 ++++++--
 pkg/kagent/install/config.go            | 104 +++++++++++++++++++++++-
 pkg/kagent/install/config_test.go       |  52 +++++++++++-
 pkg/kagent/install/install_test.go      |  36 +++++++-
 pkg/kagent/install/postgres.go          |  24 +++++-
 pkg/kagent/install/tui/tui_test.go      |   7 +-
 pkg/kagent/install/tui/wizard.go        |  26 +++++-
 pkg/kagent/install/uninstall.go         |   2 +-
 8 files changed, 265 insertions(+), 22 deletions(-)

diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index 2b21932eec..155fb15a49 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -34,6 +34,7 @@ import (
 
 var (
 	configPath    string
+	profile       string
 	chartDir      string
 	noTUI         bool
 	upgradeDryRun bool
@@ -43,6 +44,9 @@ var (
 
 const configHelp = `--config takes a YAML file, or - for stdin:
 
+  profile: aks             # or generic
+  aks:                     # for profile: aks
+    acr: my.azurecr.io     # pull the images from it
   postgres:
     type: bundled          # or external
   objectStore:
@@ -93,10 +97,20 @@ described in charts/substrate/README.md and set postgres.type to external.
 To store snapshots in a bucket, set objectStore.type to s3 or gcs and set
 objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount.
 
+profile selects the kind of cluster, and --profile sets it from the command
+line. generic makes no assumptions. aks is for Azure Kubernetes Service:
+with aks.acr set, install pulls every image it installs, including the
+bundled PostgreSQL's, from that Azure Container Registry. Import the images
+under their usual paths with az acr import and attach the registry with
+az aks update --attach-acr first.
+
 microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each
 node architecture, caches them under the user cache directory, uploads them to
 the object store, and applies the microvm SandboxConfig. Upgrade restages them.
 
+Each release keeps the values an earlier install set, so turning an option off
+doesn't undo it. To undo it, set the chart values under releases.
+
 ` + configHelp,
 	Example: `  kubectl ate install -c install.yaml
   kubectl ate install -c - <<EOF
@@ -111,7 +125,7 @@ the object store, and applies the microvm SandboxConfig. Upgrade restages them.
 		}
 		defer closeTerminal(tty)
 		if useTUI(cmd, tty) && configPath == "" {
-			if err := tui.Wizard(cmd.ErrOrStderr(), &opts.Config); err != nil {
+			if err := tui.Wizard(cmd.ErrOrStderr(), &opts.Config, profile == ""); err != nil {
 				return err
 			}
 		}
@@ -128,8 +142,8 @@ kubectl ate: the certificate controller, atelet, the CRDs, then the substrate
 release. It doesn't roll WorkerPool worker images, which
 docs/reference/version-skew-policy.md upgrades after atelet.
 
-Each release keeps the values it was installed with. The config's
-releases values are layered on top. Upgrade also reads timeout and
+Each release keeps the values it was installed with. The values from the
+config's profile and releases are layered on top. Upgrade also reads timeout and
 microvm.assetBucket from the config. It refuses downgrades, skipped minor
 releases, and CRD changes that would strand stored objects.
 
@@ -181,24 +195,29 @@ data.`,
 	},
 }
 
-// options reads the config and finds the terminal to read keys and answers
-// from. Without --config the config is install.DefaultConfig. The terminal is
-// nil when there is none. Close it with closeTerminal.
+// options reads the config, applies --profile, and finds the terminal to read
+// keys and answers from. Without --config the config is
+// install.DefaultConfig. The terminal is nil when there is none. Close it
+// with closeTerminal.
 func options(cmd *cobra.Command) (install.Options, *os.File, error) {
 	var opts install.Options
 	if chartDir != "" {
 		opts.Charts = os.DirFS(chartDir)
 	}
+	p := install.Profile(profile)
 	var err error
 	switch configPath {
 	case "":
 		opts.Config = install.DefaultConfig()
+		if err := opts.Config.SetProfile(p); err != nil {
+			return opts, nil, err
+		}
 	case "-":
-		opts.Config, err = install.ReadConfig(cmd.InOrStdin(), ".")
+		opts.Config, err = install.ReadConfig(cmd.InOrStdin(), ".", p)
 	default:
 		var f *os.File
 		if f, err = os.Open(configPath); err == nil {
-			opts.Config, err = install.ReadConfig(f, filepath.Dir(configPath))
+			opts.Config, err = install.ReadConfig(f, filepath.Dir(configPath), p)
 			f.Close()
 		}
 	}
@@ -337,6 +356,7 @@ func init() {
 	for _, c := range []*cobra.Command{installCmd, upgradeCmd, uninstallCmd} {
 		c.Flags().StringVarP(&configPath, "config", "c", "", "Install config file, or - for stdin")
 		c.Flags().BoolVar(&noTUI, "no-tui", false, "Print plain progress lines instead of the interactive view")
+		c.Flags().StringVar(&profile, "profile", "", fmt.Sprintf("Kind of cluster, one of %v; the same as profile in the config", install.Profiles))
 	}
 	for _, c := range []*cobra.Command{installCmd, upgradeCmd} {
 		c.Flags().StringVar(&chartDir, "chart-dir", "", "Load charts from this directory instead of the ones built into kubectl ate")
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index d45387c29f..4870333eff 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -21,8 +21,10 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
+	"unicode"
 
 	"helm.sh/helm/v4/pkg/cli/values"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,6 +34,10 @@ import (
 // Config describes an installation. Install, upgrade and uninstall read it
 // from the file passed to --config.
 type Config struct {
+	// Profile selects the kind of cluster. Each profile but generic has its
+	// own section of options.
+	Profile     Profile     `json:"profile,omitempty"`
+	AKS         AKS         `json:"aks,omitzero"`
 	Postgres    Postgres    `json:"postgres,omitzero"`
 	ObjectStore ObjectStore `json:"objectStore,omitzero"`
 	MicroVM     MicroVM     `json:"microvm,omitzero"`
@@ -162,6 +168,28 @@ func (o ObjectStore) validate() error {
 	return nil
 }
 
+// Profile selects the kind of cluster to install on.
+type Profile string
+
+const (
+	// ProfileGeneric makes no assumptions about the cluster.
+	ProfileGeneric Profile = "generic"
+	// ProfileAKS installs on Azure Kubernetes Service.
+	ProfileAKS Profile = "aks"
+)
+
+// Profiles lists the valid profiles.
+var Profiles = []Profile{ProfileGeneric, ProfileAKS}
+
+// AKS configures the aks profile.
+type AKS struct {
+	// ACR is the login server of an Azure Container Registry to pull every
+	// image kubectl ate installs from, like myregistry.azurecr.io. The
+	// registry must hold the images under their usual paths, and the cluster
+	// must be attached to it.
+	ACR string `json:"acr,omitempty"`
+}
+
 // MicroVM configures the microvm sandbox class.
 type MicroVM struct {
 	// Enabled stages the microVM guest assets and applies the microvm
@@ -175,6 +203,7 @@ type MicroVM struct {
 // DefaultConfig is an evaluation install: bundled PostgreSQL and RustFS.
 func DefaultConfig() Config {
 	return Config{
+		Profile:     ProfileGeneric,
 		Postgres:    Postgres{Type: PostgresBundled},
 		ObjectStore: ObjectStore{Type: ObjectStoreBundled},
 		Timeout:     metav1.Duration{Duration: 10 * time.Minute},
@@ -182,8 +211,9 @@ func DefaultConfig() Config {
 }
 
 // ReadConfig decodes a config from r and fills unset fields from
-// DefaultConfig. dir resolves relative ValuesFiles.
-func ReadConfig(r io.Reader, dir string) (Config, error) {
+// DefaultConfig. dir resolves relative ValuesFiles. A non-empty profile is
+// applied with SetProfile.
+func ReadConfig(r io.Reader, dir string, profile Profile) (Config, error) {
 	data, err := io.ReadAll(r)
 	if err != nil {
 		return Config{}, err
@@ -202,10 +232,51 @@ func ReadConfig(r io.Reader, dir string) (Config, error) {
 			}
 		}
 	}
+	if err := cfg.SetProfile(profile); err != nil {
+		return Config{}, err
+	}
 	return cfg, cfg.validate()
 }
 
+// SetProfile sets the profile unless p is empty. A profile already set to
+// anything but generic must equal p.
+func (c *Config) SetProfile(p Profile) error {
+	if p == "" {
+		return nil
+	}
+	if err := validateProfile(p); err != nil {
+		return err
+	}
+	if c.Profile != ProfileGeneric && c.Profile != p {
+		return fmt.Errorf("profile %s conflicts with profile: %s in the config", p, c.Profile)
+	}
+	c.Profile = p
+	return nil
+}
+
+// ValidateACR checks that s is empty or an Azure Container Registry login
+// server.
+func ValidateACR(s string) error {
+	if s == "" {
+		return nil
+	}
+	// Without a dot, image references read the name as a Docker Hub namespace.
+	if !strings.Contains(s, ".") || strings.ContainsFunc(s, func(r rune) bool { return r == '/' || unicode.IsSpace(r) }) {
+		return fmt.Errorf("invalid ACR %q: must be a login server like myregistry.azurecr.io", s)
+	}
+	return nil
+}
+
 func (c Config) validate() error {
+	if err := validateProfile(c.Profile); err != nil {
+		return err
+	}
+	if c.AKS != (AKS{}) && c.Profile != ProfileAKS {
+		return fmt.Errorf("aks needs profile: %s", ProfileAKS)
+	}
+	if err := ValidateACR(c.AKS.ACR); err != nil {
+		return fmt.Errorf("aks.acr: %w", err)
+	}
 	switch c.Postgres.Type {
 	case PostgresBundled, PostgresExternal:
 	default:
@@ -223,17 +294,42 @@ func (c Config) validate() error {
 	return nil
 }
 
+func validateProfile(p Profile) error {
+	if !slices.Contains(Profiles, p) {
+		return fmt.Errorf("invalid profile %q: must be one of %v", p, Profiles)
+	}
+	return nil
+}
+
+// imageRegistry returns the registry to pull every image kubectl ate installs
+// from, or "" to keep each image's own.
+func (c Config) imageRegistry() string {
+	if c.Profile == ProfileAKS {
+		return c.AKS.ACR
+	}
+	return ""
+}
+
+// profileValues returns Helm values for every release except substrate-crds.
+func (c Config) profileValues() map[string]any {
+	if r := c.imageRegistry(); r != "" {
+		return map[string]any{"global": map[string]any{"imageRegistry": r}}
+	}
+	return nil
+}
+
 func (r *Releases) all() []*ReleaseValues {
 	return []*ReleaseValues{&r.All, &r.PodCert, &r.Atelet, &r.Substrate}
 }
 
-// userValues returns each release's values, keyed by release name: All's
-// files and values, then the release's own.
+// userValues returns each release's values, keyed by release name: the
+// profile's, All's files and values, then the release's own.
 func (c Config) userValues() (map[string]map[string]any, error) {
 	all, err := c.Releases.All.read()
 	if err != nil {
 		return nil, err
 	}
+	all = mergeValues(c.profileValues(), all)
 	user := map[string]map[string]any{}
 	for r, rv := range map[release]ReleaseValues{
 		podCertRelease:   c.Releases.PodCert,
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index d3094e3fab..5a1feb6ab5 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -34,6 +34,7 @@ func TestReadConfig(t *testing.T) {
 	tests := []struct {
 		name    string
 		in      string
+		profile Profile
 		want    Config
 		wantErr string
 	}{
@@ -67,7 +68,30 @@ releases:
 				c.Releases.Substrate.ValuesFiles = []string{"/cfg/db.yaml"}
 			}),
 		},
+		{
+			name: "aks",
+			in:   "profile: aks\naks: {acr: r.azurecr.io}\n",
+			want: withDefaults(func(c *Config) {
+				c.Profile, c.AKS.ACR = ProfileAKS, "r.azurecr.io"
+			}),
+		},
+		{
+			name:    "profile argument sets profile",
+			in:      "aks: {acr: r.azurecr.io}\n",
+			profile: ProfileAKS,
+			want: withDefaults(func(c *Config) {
+				c.Profile, c.AKS.ACR = ProfileAKS, "r.azurecr.io"
+			}),
+		},
+		{name: "profile argument matches", in: "profile: aks\n", profile: ProfileAKS, want: withDefaults(func(c *Config) { c.Profile = ProfileAKS })},
 		{name: "empty", in: "", wantErr: "config is empty"},
+		{name: "bad profile", in: "profile: eks\n", wantErr: "invalid profile"},
+		{name: "bad profile argument", in: "{}", profile: "eks", wantErr: "invalid profile"},
+		{name: "profile argument conflicts", in: "profile: aks\n", profile: ProfileGeneric, wantErr: "conflicts"},
+		{name: "aks needs profile", in: "aks: {acr: r.azurecr.io}\n", wantErr: "aks needs profile: aks"},
+		{name: "acr with path", in: "profile: aks\naks: {acr: r.azurecr.io/substrate}\n", wantErr: "invalid ACR"},
+		{name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"},
+		{name: "acr with space", in: "profile: aks\naks: {acr: 'r.azurecr.io '}\n", wantErr: "invalid ACR"},
 		{name: "unknown field", in: "postgress: external\n", wantErr: "postgress"},
 		{name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"},
 		{name: "bad postgres", in: "postgres: cloud\n", wantErr: "invalid postgres"},
@@ -104,7 +128,7 @@ releases:
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			got, err := ReadConfig(strings.NewReader(tc.in), "/cfg")
+			got, err := ReadConfig(strings.NewReader(tc.in), "/cfg", tc.profile)
 			if tc.wantErr != "" {
 				if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
 					t.Fatalf("ReadConfig() error = %v, want it to contain %q", err, tc.wantErr)
@@ -157,3 +181,29 @@ func TestConfigUserValues(t *testing.T) {
 		t.Errorf("userValues() mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestConfigUserValuesProfile(t *testing.T) {
+	registry := func(r string) map[string]any {
+		return map[string]any{"global": map[string]any{"imageRegistry": r}}
+	}
+	c := Config{
+		Profile: ProfileAKS,
+		AKS:     AKS{ACR: "r.azurecr.io"},
+		Releases: Releases{
+			Substrate: ReleaseValues{Values: registry("mirror.example.com")},
+		},
+	}
+	got, err := c.userValues()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// The release's own values win over the profile's.
+	want := map[string]map[string]any{
+		"substrate-podcert": registry("r.azurecr.io"),
+		"substrate-atelet":  registry("r.azurecr.io"),
+		"substrate":         registry("mirror.example.com"),
+	}
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("userValues() mismatch (-want +got):\n%s", diff)
+	}
+}
diff --git a/pkg/kagent/install/install_test.go b/pkg/kagent/install/install_test.go
index 245cca59f8..64111c08f9 100644
--- a/pkg/kagent/install/install_test.go
+++ b/pkg/kagent/install/install_test.go
@@ -180,7 +180,7 @@ func secretData(t *testing.T, kc *fake.Clientset) map[string]any {
 
 func TestApplyPostgresManifest(t *testing.T) {
 	ctx := context.Background()
-	manifest, err := postgresManifest()
+	manifest, err := postgresManifest("")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -204,6 +204,40 @@ func TestApplyPostgresManifest(t *testing.T) {
 	}
 }
 
+func TestPostgresManifestRegistry(t *testing.T) {
+	ctx := context.Background()
+	images := func(registry string) []string {
+		t.Helper()
+		manifest, err := postgresManifest(registry)
+		if err != nil {
+			t.Fatal(err)
+		}
+		kc := fake.NewClientset()
+		if err := applyManifest(ctx, kc, manifest); err != nil {
+			t.Fatal(err)
+		}
+		ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		var out []string
+		for _, c := range append(ss.Spec.Template.Spec.InitContainers, ss.Spec.Template.Spec.Containers...) {
+			out = append(out, c.Image)
+		}
+		return out
+	}
+	base, mirrored := images(""), images("r.azurecr.io")
+	if len(base) == 0 || len(base) != len(mirrored) {
+		t.Fatalf("got images %v and %v, want the same nonzero count", base, mirrored)
+	}
+	// The digest stays, so the mirror must hold the same image.
+	for i := range base {
+		if want := "r.azurecr.io/" + base[i]; mirrored[i] != want {
+			t.Errorf("image %d = %s, want %s", i, mirrored[i], want)
+		}
+	}
+}
+
 func TestCheckExternalObjectStore(t *testing.T) {
 	s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"}
 	gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}}
diff --git a/pkg/kagent/install/postgres.go b/pkg/kagent/install/postgres.go
index 8cc002de24..3835d90406 100644
--- a/pkg/kagent/install/postgres.go
+++ b/pkg/kagent/install/postgres.go
@@ -37,6 +37,7 @@ import (
 	"sigs.k8s.io/kustomize/api/krusty"
 	kusttypes "sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
+	"sigs.k8s.io/yaml"
 
 	ateinstall "go.ate.dev/substrate/manifests/ate-install"
 	"go.ate.dev/substrate/pkg/postgressetup"
@@ -57,7 +58,7 @@ const (
 // applyPostgres applies manifests/ate-install/kind/postgres and waits for it.
 // That overlay sizes the database for evaluation rather than production.
 func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (string, error) {
-	manifest, err := postgresManifest()
+	manifest, err := postgresManifest(in.opts.imageRegistry())
 	if err != nil {
 		return "", err
 	}
@@ -71,7 +72,9 @@ func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (
 	return "statefulset/" + postgresStatefulSet + " ready", nil
 }
 
-func postgresManifest() ([]byte, error) {
+// postgresManifest builds the bundled PostgreSQL manifest. A non-empty
+// registry replaces the image's, the way the charts' global.imageRegistry does.
+func postgresManifest(registry string) ([]byte, error) {
 	mem := filesys.MakeFsInMemory()
 	err := fs.WalkDir(ateinstall.Postgres, ".", func(p string, d fs.DirEntry, err error) error {
 		if err != nil || d.IsDir() {
@@ -86,10 +89,25 @@ func postgresManifest() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	dir := "/kind/postgres"
+	if registry != "" {
+		k, err := yaml.Marshal(kusttypes.Kustomization{
+			TypeMeta:  kusttypes.TypeMeta{APIVersion: kusttypes.KustomizationVersion, Kind: kusttypes.KustomizationKind},
+			Resources: []string{"../kind/postgres"},
+			Images:    []kusttypes.Image{{Name: "postgres", NewName: registry + "/postgres"}},
+		})
+		if err != nil {
+			return nil, err
+		}
+		dir = "/registry"
+		if err := mem.WriteFile(dir+"/kustomization.yaml", k); err != nil {
+			return nil, err
+		}
+	}
 	opts := krusty.MakeDefaultOptions()
 	// The overlay references its base outside its own directory.
 	opts.LoadRestrictions = kusttypes.LoadRestrictionsNone
-	res, err := krusty.MakeKustomizer(opts).Run(mem, "/kind/postgres")
+	res, err := krusty.MakeKustomizer(opts).Run(mem, dir)
 	if err != nil {
 		return nil, fmt.Errorf("while building the PostgreSQL manifest: %w", err)
 	}
diff --git a/pkg/kagent/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go
index 84ca770fdd..c7a80ba18e 100644
--- a/pkg/kagent/install/tui/tui_test.go
+++ b/pkg/kagent/install/tui/tui_test.go
@@ -78,6 +78,11 @@ func TestHint(t *testing.T) {
 			c.Releases.All.ValuesFiles = []string{"/etc/gke.yaml"}
 			return c
 		}(),
+		func() install.Config {
+			c := install.DefaultConfig()
+			c.Profile, c.AKS.ACR = install.ProfileAKS, "r.azurecr.io"
+			return c
+		}(),
 	} {
 		hint, err := Hint(&cfg)
 		if err != nil {
@@ -87,7 +92,7 @@ func TestHint(t *testing.T) {
 		if !ok {
 			t.Fatalf("Hint() has no config:\n%s", hint)
 		}
-		got, err := install.ReadConfig(strings.NewReader(doc), "")
+		got, err := install.ReadConfig(strings.NewReader(doc), "", "")
 		if err != nil {
 			t.Fatalf("ReadConfig(Hint()): %v\n%s", err, doc)
 		}
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index d169513f73..71cc72a9ea 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -27,13 +27,30 @@ import (
 )
 
 // Wizard asks for the install choices, stores them in cfg, and prints the
-// matching config to w. It returns install.ErrAborted if the user declines or
-// quits.
-func Wizard(w io.Writer, cfg *install.Config) error {
+// matching config to w. askProfile asks for the profile, otherwise cfg's is
+// kept. It returns install.ErrAborted if the user declines or quits.
+func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 	store := &cfg.ObjectStore
 	external := func() bool { return store.Type != install.ObjectStoreBundled }
+	aks := func() bool { return cfg.Profile == install.ProfileAKS }
 	confirmed := true
 	form := huh.NewForm(
+		huh.NewGroup(
+			huh.NewSelect[install.Profile]().
+				Title("Cluster").
+				Options(
+					huh.NewOption("Any Kubernetes cluster", install.ProfileGeneric),
+					huh.NewOption("Azure Kubernetes Service (AKS)", install.ProfileAKS),
+				).
+				Value(&cfg.Profile),
+		).WithHideFunc(func() bool { return !askProfile }),
+		huh.NewGroup(
+			huh.NewInput().
+				Title("Azure Container Registry").
+				Description("Login server to pull Agent Substrate's images from, like myregistry.azurecr.io. It must hold\nthe images under their usual paths, and the cluster must be attached to it. Leave empty for the public registries.").
+				Value(&cfg.AKS.ACR).
+				Validate(install.ValidateACR),
+		).WithHideFunc(func() bool { return !aks() }),
 		huh.NewGroup(
 			huh.NewSelect[install.PostgresType]().
 				Title("PostgreSQL").
@@ -119,6 +136,9 @@ func Wizard(w io.Writer, cfg *install.Config) error {
 	if !external() || !cfg.MicroVM.Enabled {
 		cfg.MicroVM.AssetBucket = ""
 	}
+	if !aks() {
+		cfg.AKS = install.AKS{}
+	}
 	hint, err := Hint(cfg)
 	if err != nil {
 		return err
diff --git a/pkg/kagent/install/uninstall.go b/pkg/kagent/install/uninstall.go
index 375ba5a228..dd431b5f38 100644
--- a/pkg/kagent/install/uninstall.go
+++ b/pkg/kagent/install/uninstall.go
@@ -217,7 +217,7 @@ func (in *Installer) uninstallStep(r release) func(context.Context, func(string)
 // removePostgres deletes the bundled PostgreSQL objects that install applied,
 // identified by kubectl ate's field manager. Its volume stays until --purge.
 func (in *Installer) removePostgres(ctx context.Context, _ func(string)) (string, error) {
-	manifest, err := postgresManifest()
+	manifest, err := postgresManifest("")
 	if err != nil {
 		return "", err
 	}

From 73190078aed5976e5ecfc22cc7333a70e8a6b606 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 07:46:40 -0500
Subject: [PATCH 02/10] kubectl-ate: check NetworkPolicy enforcement on AKS

An AKS cluster created without --network-policy accepts NetworkPolicy
objects and ignores them, which leaves the ate-system policies and the
per-WorkerPool policies without effect. With profile: aks, install and
upgrade now fail unless a cilium, azure-npm or calico-node DaemonSet
exists.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 cmd/kubectl-ate/internal/cmd/install.go |  8 ++--
 pkg/kagent/install/aks.go               | 61 ++++++++++++++++++++++++
 pkg/kagent/install/aks_test.go          | 62 +++++++++++++++++++++++++
 pkg/kagent/install/steps.go             | 10 +++-
 pkg/kagent/install/upgrade.go           | 18 +++++--
 pkg/kagent/install/upgrade_test.go      | 20 ++++++++
 6 files changed, 171 insertions(+), 8 deletions(-)
 create mode 100644 pkg/kagent/install/aks.go
 create mode 100644 pkg/kagent/install/aks_test.go

diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index 155fb15a49..1eb5c0c794 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -98,8 +98,9 @@ To store snapshots in a bucket, set objectStore.type to s3 or gcs and set
 objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount.
 
 profile selects the kind of cluster, and --profile sets it from the command
-line. generic makes no assumptions. aks is for Azure Kubernetes Service:
-with aks.acr set, install pulls every image it installs, including the
+line. generic makes no assumptions. aks is for Azure Kubernetes Service. It
+checks that a NetworkPolicy engine is installed, since AKS ignores
+NetworkPolicy without one. With aks.acr set, install pulls every image it installs, including the
 bundled PostgreSQL's, from that Azure Container Registry. Import the images
 under their usual paths with az acr import and attach the registry with
 az aks update --attach-acr first.
@@ -143,7 +144,8 @@ release. It doesn't roll WorkerPool worker images, which
 docs/reference/version-skew-policy.md upgrades after atelet.
 
 Each release keeps the values it was installed with. The values from the
-config's profile and releases are layered on top. Upgrade also reads timeout and
+config's profile and releases are layered on top. With profile aks, upgrade
+first runs install's AKS checks. Upgrade also reads timeout and
 microvm.assetBucket from the config. It refuses downgrades, skipped minor
 releases, and CRD changes that would strand stored objects.
 
diff --git a/pkg/kagent/install/aks.go b/pkg/kagent/install/aks.go
new file mode 100644
index 0000000000..7e3e33a543
--- /dev/null
+++ b/pkg/kagent/install/aks.go
@@ -0,0 +1,61 @@
+// Copyright 2026 The Agent Substrate Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package install
+
+import (
+	"context"
+	"errors"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// networkPolicyEngines are the DaemonSets of the engines AKS installs with
+// az aks create --network-policy. Older clusters run Calico in kube-system.
+var networkPolicyEngines = []types.NamespacedName{
+	{Namespace: "kube-system", Name: "cilium"},
+	{Namespace: "kube-system", Name: "azure-npm"},
+	{Namespace: "calico-system", Name: "calico-node"},
+	{Namespace: "kube-system", Name: "calico-node"},
+}
+
+// checkAKS runs the aks profile's cluster checks. Upgrade runs them too, since
+// it applies the profile's values.
+func (in *Installer) checkAKS(ctx context.Context, progress func(string)) (string, error) {
+	progress("checking NetworkPolicy enforcement")
+	engine, err := in.checkNetworkPolicy(ctx)
+	if err != nil {
+		return "", err
+	}
+	summary := "NetworkPolicy by " + engine
+	return summary, nil
+}
+
+// checkNetworkPolicy returns the DaemonSet that enforces NetworkPolicy. An AKS
+// cluster without one accepts NetworkPolicy objects and ignores them, which
+// would leave Substrate's components and WorkerPools unisolated.
+func (in *Installer) checkNetworkPolicy(ctx context.Context) (string, error) {
+	for _, e := range networkPolicyEngines {
+		_, err := in.kube.AppsV1().DaemonSets(e.Namespace).Get(ctx, e.Name, metav1.GetOptions{})
+		if err == nil {
+			return e.String(), nil
+		}
+		if !apierrors.IsNotFound(err) {
+			return "", err
+		}
+	}
+	return "", errors.New("nothing in the cluster enforces NetworkPolicy; enable it with az aks update --network-policy, or set profile: generic to skip this check")
+}
diff --git a/pkg/kagent/install/aks_test.go b/pkg/kagent/install/aks_test.go
new file mode 100644
index 0000000000..db7507b2cb
--- /dev/null
+++ b/pkg/kagent/install/aks_test.go
@@ -0,0 +1,62 @@
+// Copyright 2026 The Agent Substrate Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package install
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+)
+
+func TestCheckNetworkPolicy(t *testing.T) {
+	ds := func(ns, name string) runtime.Object {
+		return &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: name}}
+	}
+	tests := []struct {
+		name    string
+		objs    []runtime.Object
+		want    string
+		wantErr string
+	}{
+		{name: "cilium", objs: []runtime.Object{ds("kube-system", "cilium")}, want: "kube-system/cilium"},
+		{name: "azure-npm", objs: []runtime.Object{ds("kube-system", "azure-npm")}, want: "kube-system/azure-npm"},
+		{name: "calico", objs: []runtime.Object{ds("calico-system", "calico-node")}, want: "calico-system/calico-node"},
+		{name: "calico in kube-system", objs: []runtime.Object{ds("kube-system", "calico-node")}, want: "kube-system/calico-node"},
+		{name: "none", objs: []runtime.Object{ds("kube-system", "kube-proxy")}, wantErr: "az aks update --network-policy"},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			in := &Installer{kube: fake.NewClientset(tc.objs...)}
+			got, err := in.checkNetworkPolicy(context.Background())
+			if tc.wantErr != "" {
+				if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
+					t.Fatalf("checkNetworkPolicy() error = %v, want it to contain %q", err, tc.wantErr)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tc.want {
+				t.Errorf("checkNetworkPolicy() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
diff --git a/pkg/kagent/install/steps.go b/pkg/kagent/install/steps.go
index 481255ef75..679f3a653f 100644
--- a/pkg/kagent/install/steps.go
+++ b/pkg/kagent/install/steps.go
@@ -48,6 +48,14 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri
 	if err := checkPodCertificateAPI(in); err != nil {
 		return "", err
 	}
+	summary := "Kubernetes " + sv.GitVersion
+	if in.opts.Profile == ProfileAKS {
+		aks, err := in.checkAKS(ctx, progress)
+		if err != nil {
+			return "", err
+		}
+		summary += ", " + aks
+	}
 
 	progress("checking for an existing install")
 	rel, err := in.helm.get(substrateRelease)
@@ -64,7 +72,7 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri
 			return "", err
 		}
 	}
-	return "Kubernetes " + sv.GitVersion, nil
+	return summary, nil
 }
 
 // checkPodCertificateAPI checks that the cluster serves PodCertificateRequest,
diff --git a/pkg/kagent/install/upgrade.go b/pkg/kagent/install/upgrade.go
index e57eaf01d8..3822ce8f3e 100644
--- a/pkg/kagent/install/upgrade.go
+++ b/pkg/kagent/install/upgrade.go
@@ -55,10 +55,7 @@ type UpgradeOptions struct {
 // Upgrade moves every release to the charts in this binary.
 func (in *Installer) Upgrade(ctx context.Context, opts UpgradeOptions) error {
 	in.upgrading = true
-	steps := []Step{
-		{"Check versions", in.checkVersions},
-		{"Check CRD storage versions", in.checkCRDStorage},
-	}
+	steps := in.upgradeChecks()
 	if opts.DryRun {
 		for _, u := range upgradeOrder {
 			steps = append(steps, Step{"Diff " + u.r.name, in.diffStep(u.r, in.valuesFor(u.r), opts.Out)})
@@ -89,6 +86,19 @@ func (in *Installer) Upgrade(ctx context.Context, opts UpgradeOptions) error {
 	return in.Run(ctx, steps)
 }
 
+// upgradeChecks returns the steps that check the cluster before Upgrade
+// changes it.
+func (in *Installer) upgradeChecks() []Step {
+	steps := []Step{
+		{"Check versions", in.checkVersions},
+		{"Check CRD storage versions", in.checkCRDStorage},
+	}
+	if in.opts.Profile == ProfileAKS {
+		steps = append(steps, Step{"Check AKS", in.checkAKS})
+	}
+	return steps
+}
+
 // hasMicroVMSandboxConfig reports whether the microvm SandboxConfig exists.
 func (in *Installer) hasMicroVMSandboxConfig(ctx context.Context) (bool, error) {
 	_, err := in.dyn.Resource(sandboxConfigsGVR).Get(ctx, microVMSandboxConfig, metav1.GetOptions{})
diff --git a/pkg/kagent/install/upgrade_test.go b/pkg/kagent/install/upgrade_test.go
index 852f840f19..f5da754489 100644
--- a/pkg/kagent/install/upgrade_test.go
+++ b/pkg/kagent/install/upgrade_test.go
@@ -20,6 +20,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -127,3 +128,22 @@ func TestUpgradeKeepsStorageValues(t *testing.T) {
 		}
 	}
 }
+
+func TestUpgradeChecks(t *testing.T) {
+	for _, tc := range []struct {
+		profile Profile
+		want    []string
+	}{
+		{profile: ProfileGeneric, want: []string{"Check versions", "Check CRD storage versions"}},
+		{profile: ProfileAKS, want: []string{"Check versions", "Check CRD storage versions", "Check AKS"}},
+	} {
+		in := &Installer{opts: Options{Config: Config{Profile: tc.profile}}}
+		var got []string
+		for _, s := range in.upgradeChecks() {
+			got = append(got, s.Name)
+		}
+		if diff := cmp.Diff(tc.want, got); diff != "" {
+			t.Errorf("profile %s: upgradeChecks() mismatch (-want +got):\n%s", tc.profile, diff)
+		}
+	}
+}

From 0715d60d874d50d21ba8215ba02c577294e86ec4 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 07:54:12 -0500
Subject: [PATCH 03/10] kubectl-ate: add a storageClass option for the bundled
 volumes

storageClass sets the StorageClass of the bundled PostgreSQL and RustFS
volumes, which otherwise use the cluster's default. The substrate chart
gets a rustfs.storageClassName value for it. On AKS the wizard suggests
managed-csi-premium.

An existing PostgreSQL keeps its StorageClass, since a StatefulSet's
volumeClaimTemplates can't change.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 charts/substrate/README.md                    |  1 +
 charts/substrate/templates/rustfs.yaml        |  3 +
 .../substrate/tests/control_plane_test.yaml   | 19 ++++
 charts/substrate/values.yaml                  |  2 +
 cmd/kubectl-ate/internal/cmd/install.go       |  4 +
 pkg/kagent/install/config.go                  | 23 +++++
 pkg/kagent/install/config_test.go             |  3 +
 pkg/kagent/install/install_test.go            | 99 ++++++++++++++++---
 pkg/kagent/install/postgres.go                | 63 +++++++++---
 pkg/kagent/install/steps.go                   |  6 +-
 pkg/kagent/install/tui/wizard.go              | 21 ++++
 pkg/kagent/install/uninstall.go               |  2 +-
 12 files changed, 217 insertions(+), 29 deletions(-)

diff --git a/charts/substrate/README.md b/charts/substrate/README.md
index 35d7bc96c8..20fe98412b 100644
--- a/charts/substrate/README.md
+++ b/charts/substrate/README.md
@@ -89,6 +89,7 @@ See `values.yaml` for the full set; the important keys:
 | `atenetEgress.extraCredentialProviders` | `[]` | Credential providers you deploy, added after the bundled `k8s.io` provider, each `{name, address}`. `name` is the host of the `ate-secret://` URIs it serves; `address` is the `host:port` the egress gateway dials over mTLS. Its serving certificate must chain to the servicedns CA. See [egress credential injection](../../docs/guides/egress-credential-injection.md#the-plugin-api) |
 | `atenetEgress.agentgateway.logLevel` | `info` | Egress proxy `RUST_LOG` filter, e.g. `debug` or `info,agentgateway=debug` when troubleshooting |
 | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots |
+| `rustfs.storageClassName` | `""` | StorageClass of the RustFS data volume; empty uses the cluster's default |
 | `ateApi.snapshotLocation` | `""` | Cluster-wide snapshot URI; defaults to `gs://<rustfs.bucket>/` with bundled RustFS |
 | `atelet.enabled` | `true` | Deprecated. Set to `false` and install atelet with the [substrate-atelet](../substrate-atelet/README.md) chart |
 | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml
index 95a5b10053..d8d39a6514 100644
--- a/charts/substrate/templates/rustfs.yaml
+++ b/charts/substrate/templates/rustfs.yaml
@@ -32,6 +32,9 @@ metadata:
 spec:
   accessModes:
   - ReadWriteOnce
+  {{- with .Values.rustfs.storageClassName }}
+  storageClassName: {{ . | quote }}
+  {{- end }}
   resources:
     requests:
       storage: {{ .Values.rustfs.storageSize }}
diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml
index 87646b96fc..fb8e9abf0e 100644
--- a/charts/substrate/tests/control_plane_test.yaml
+++ b/charts/substrate/tests/control_plane_test.yaml
@@ -265,6 +265,25 @@ tests:
       value:
       - protocol: TCP
         port: api
+- it: leaves the RustFS volume on the default StorageClass
+  template: rustfs.yaml
+  documentSelector:
+    path: kind
+    value: PersistentVolumeClaim
+  asserts:
+  - notExists:
+      path: spec.storageClassName
+- it: puts the RustFS volume on rustfs.storageClassName
+  template: rustfs.yaml
+  documentSelector:
+    path: kind
+    value: PersistentVolumeClaim
+  set:
+    rustfs.storageClassName: managed-csi-premium
+  asserts:
+  - equal:
+      path: spec.storageClassName
+      value: managed-csi-premium
 
 - it: supplies the required API config and bootstraps the release client
   template: ate-api-server.yaml
diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml
index 5d0f7a4dce..87e9c55c67 100644
--- a/charts/substrate/values.yaml
+++ b/charts/substrate/values.yaml
@@ -47,6 +47,8 @@ postgres:
 rustfs:
   enabled: true
   storageSize: 1Gi
+  # StorageClass of the data volume. Empty uses the cluster's default.
+  storageClassName: ""
   bucket: ate-snapshots
   accessKey: rustfsadmin
   secretKey: rustfsadmin
diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index 1eb5c0c794..2777baa9ef 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -62,6 +62,7 @@ const configHelp = `--config takes a YAML file, or - for stdin:
   microvm:
     enabled: true
     assetBucket: my-assets # for s3 or gcs
+  storageClass: fast       # bundled volumes; empty for the default
   timeout: 10m             # each wait for workloads
   releases:                # Helm values
     all:                   # every release except substrate-crds
@@ -97,6 +98,9 @@ described in charts/substrate/README.md and set postgres.type to external.
 To store snapshots in a bucket, set objectStore.type to s3 or gcs and set
 objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount.
 
+storageClass sets the StorageClass of the bundled PostgreSQL and RustFS
+volumes when they are created. On AKS, managed-csi-premium uses Premium SSDs.
+
 profile selects the kind of cluster, and --profile sets it from the command
 line. generic makes no assumptions. aks is for Azure Kubernetes Service. It
 checks that a NetworkPolicy engine is installed, since AKS ignores
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index 4870333eff..5bec1e4e2a 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -28,6 +28,7 @@ import (
 
 	"helm.sh/helm/v4/pkg/cli/values"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
 	"sigs.k8s.io/yaml"
 )
 
@@ -41,6 +42,9 @@ type Config struct {
 	Postgres    Postgres    `json:"postgres,omitzero"`
 	ObjectStore ObjectStore `json:"objectStore,omitzero"`
 	MicroVM     MicroVM     `json:"microvm,omitzero"`
+	// StorageClass is the StorageClass of the bundled PostgreSQL and RustFS
+	// volumes. Empty uses the cluster's default.
+	StorageClass string `json:"storageClass,omitempty"`
 	// Timeout bounds each wait for workloads to become ready.
 	Timeout metav1.Duration `json:"timeout,omitzero"`
 	// Releases holds Helm values for the releases.
@@ -267,6 +271,17 @@ func ValidateACR(s string) error {
 	return nil
 }
 
+// ValidateStorageClass checks that s is empty or a valid StorageClass name.
+func ValidateStorageClass(s string) error {
+	if s == "" {
+		return nil
+	}
+	if errs := validation.IsDNS1123Subdomain(s); len(errs) > 0 {
+		return fmt.Errorf("invalid StorageClass name %q: %s", s, strings.Join(errs, "; "))
+	}
+	return nil
+}
+
 func (c Config) validate() error {
 	if err := validateProfile(c.Profile); err != nil {
 		return err
@@ -285,6 +300,14 @@ func (c Config) validate() error {
 	if err := c.ObjectStore.validate(); err != nil {
 		return err
 	}
+	if c.StorageClass != "" {
+		if err := ValidateStorageClass(c.StorageClass); err != nil {
+			return fmt.Errorf("storageClass: %w", err)
+		}
+		if c.Postgres.Type != PostgresBundled && c.ObjectStore.Type != ObjectStoreBundled {
+			return fmt.Errorf("storageClass needs postgres.type: %s or objectStore.type: %s", PostgresBundled, ObjectStoreBundled)
+		}
+	}
 	if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" {
 		return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket")
 	}
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index 5a1feb6ab5..d84c2f0a47 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -83,6 +83,9 @@ releases:
 				c.Profile, c.AKS.ACR = ProfileAKS, "r.azurecr.io"
 			}),
 		},
+		{name: "storage class", in: "storageClass: managed-csi-premium\n", want: withDefaults(func(c *Config) { c.StorageClass = "managed-csi-premium" })},
+		{name: "bad storage class", in: "storageClass: Premium_SSD\n", wantErr: "invalid StorageClass name"},
+		{name: "storage class without bundled volumes", in: "postgres: external\nobjectStore: s3\nstorageClass: fast\n", wantErr: "storageClass needs"},
 		{name: "profile argument matches", in: "profile: aks\n", profile: ProfileAKS, want: withDefaults(func(c *Config) { c.Profile = ProfileAKS })},
 		{name: "empty", in: "", wantErr: "config is empty"},
 		{name: "bad profile", in: "profile: eks\n", wantErr: "invalid profile"},
diff --git a/pkg/kagent/install/install_test.go b/pkg/kagent/install/install_test.go
index 64111c08f9..e8dcd9b58c 100644
--- a/pkg/kagent/install/install_test.go
+++ b/pkg/kagent/install/install_test.go
@@ -24,9 +24,11 @@ import (
 	"testing"
 
 	chart "helm.sh/helm/v4/pkg/chart/v2"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/utils/ptr"
 
 	chartsfs "go.ate.dev/substrate/charts"
 )
@@ -180,7 +182,7 @@ func secretData(t *testing.T, kc *fake.Clientset) map[string]any {
 
 func TestApplyPostgresManifest(t *testing.T) {
 	ctx := context.Background()
-	manifest, err := postgresManifest("")
+	manifest, err := postgresManifest(Config{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -204,29 +206,35 @@ func TestApplyPostgresManifest(t *testing.T) {
 	}
 }
 
-func TestPostgresManifestRegistry(t *testing.T) {
+func postgresStatefulSetFor(t *testing.T, c Config) *appsv1.StatefulSet {
+	t.Helper()
 	ctx := context.Background()
-	images := func(registry string) []string {
+	manifest, err := postgresManifest(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	kc := fake.NewClientset()
+	if err := applyManifest(ctx, kc, manifest); err != nil {
+		t.Fatal(err)
+	}
+	ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return ss
+}
+
+func TestPostgresManifestRegistry(t *testing.T) {
+	images := func(c Config) []string {
 		t.Helper()
-		manifest, err := postgresManifest(registry)
-		if err != nil {
-			t.Fatal(err)
-		}
-		kc := fake.NewClientset()
-		if err := applyManifest(ctx, kc, manifest); err != nil {
-			t.Fatal(err)
-		}
-		ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{})
-		if err != nil {
-			t.Fatal(err)
-		}
+		ss := postgresStatefulSetFor(t, c)
 		var out []string
 		for _, c := range append(ss.Spec.Template.Spec.InitContainers, ss.Spec.Template.Spec.Containers...) {
 			out = append(out, c.Image)
 		}
 		return out
 	}
-	base, mirrored := images(""), images("r.azurecr.io")
+	base, mirrored := images(Config{}), images(Config{Profile: ProfileAKS, AKS: AKS{ACR: "r.azurecr.io"}})
 	if len(base) == 0 || len(base) != len(mirrored) {
 		t.Fatalf("got images %v and %v, want the same nonzero count", base, mirrored)
 	}
@@ -238,6 +246,65 @@ func TestPostgresManifestRegistry(t *testing.T) {
 	}
 }
 
+func TestPostgresManifestStorageClass(t *testing.T) {
+	for _, sc := range []string{"", "managed-csi-premium"} {
+		ss := postgresStatefulSetFor(t, Config{StorageClass: sc})
+		vct := ss.Spec.VolumeClaimTemplates[0].Spec
+		if got := ptr.Deref(vct.StorageClassName, ""); got != sc {
+			t.Errorf("StorageClass %q: storageClassName = %q", sc, got)
+		}
+		// The overlay keeps the kind overlay's patches.
+		if got := vct.Resources.Requests.Storage().String(); got != "1Gi" {
+			t.Errorf("StorageClass %q: volume size = %s, want 1Gi", sc, got)
+		}
+	}
+}
+
+func TestPostgresStorageClass(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		existing *string
+		want     string
+	}{
+		{name: "new install uses the config's", want: "managed-csi-premium"},
+		{name: "existing keeps the default", existing: ptr.To(""), want: ""},
+		{name: "existing keeps its own", existing: ptr.To("managed-csi"), want: "managed-csi"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			kc := fake.NewClientset()
+			if tc.existing != nil {
+				ss := postgresStatefulSetFor(t, Config{StorageClass: *tc.existing})
+				if _, err := kc.AppsV1().StatefulSets(SystemNamespace).Create(t.Context(), ss, metav1.CreateOptions{}); err != nil {
+					t.Fatal(err)
+				}
+			}
+			got, err := postgresStorageClass(t.Context(), kc, "managed-csi-premium")
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != tc.want {
+				t.Errorf("postgresStorageClass() = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestSubstrateValuesStorageClass(t *testing.T) {
+	kc := fake.NewClientset(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Namespace: SystemNamespace, Name: rustfsCredentialsSecret},
+		Data:       map[string][]byte{"AWS_ACCESS_KEY_ID": []byte("a"), "AWS_SECRET_ACCESS_KEY": []byte("s")},
+	})
+	in := &Installer{kube: kc, opts: Options{Config: Config{ObjectStore: ObjectStore{Type: ObjectStoreBundled}, Postgres: Postgres{Type: PostgresBundled}, StorageClass: "fast"}}}
+	defaults, _, err := in.substrateValues(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := map[string]any{"rustfs": map[string]any{"storageClassName": "fast"}}
+	if !reflect.DeepEqual(defaults, want) {
+		t.Errorf("substrate defaults = %v, want %v", defaults, want)
+	}
+}
+
 func TestCheckExternalObjectStore(t *testing.T) {
 	s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"}
 	gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}}
diff --git a/pkg/kagent/install/postgres.go b/pkg/kagent/install/postgres.go
index 3835d90406..755a0fd25a 100644
--- a/pkg/kagent/install/postgres.go
+++ b/pkg/kagent/install/postgres.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
@@ -34,9 +35,11 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/kustomize/api/krusty"
 	kusttypes "sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/filesys"
+	"sigs.k8s.io/kustomize/kyaml/resid"
 	"sigs.k8s.io/yaml"
 
 	ateinstall "go.ate.dev/substrate/manifests/ate-install"
@@ -58,7 +61,13 @@ const (
 // applyPostgres applies manifests/ate-install/kind/postgres and waits for it.
 // That overlay sizes the database for evaluation rather than production.
 func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (string, error) {
-	manifest, err := postgresManifest(in.opts.imageRegistry())
+	c := in.opts.Config
+	sc, err := postgresStorageClass(ctx, in.kube, c.StorageClass)
+	if err != nil {
+		return "", err
+	}
+	c.StorageClass = sc
+	manifest, err := postgresManifest(c)
 	if err != nil {
 		return "", err
 	}
@@ -72,9 +81,24 @@ func (in *Installer) applyPostgres(ctx context.Context, progress func(string)) (
 	return "statefulset/" + postgresStatefulSet + " ready", nil
 }
 
-// postgresManifest builds the bundled PostgreSQL manifest. A non-empty
-// registry replaces the image's, the way the charts' global.imageRegistry does.
-func postgresManifest(registry string) ([]byte, error) {
+// postgresStorageClass returns the StorageClass to apply the bundled
+// PostgreSQL with. An existing StatefulSet keeps its own, since its
+// volumeClaimTemplates can't change.
+func postgresStorageClass(ctx context.Context, kc kubernetes.Interface, sc string) (string, error) {
+	ss, err := kc.AppsV1().StatefulSets(SystemNamespace).Get(ctx, postgresStatefulSet, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		return sc, nil
+	}
+	if err != nil {
+		return "", err
+	}
+	return ptr.Deref(ss.Spec.VolumeClaimTemplates[0].Spec.StorageClassName, ""), nil
+}
+
+// postgresManifest builds the bundled PostgreSQL manifest. c's image registry
+// replaces the image's, the way the charts' global.imageRegistry does, and c's
+// StorageClass sets the volume's.
+func postgresManifest(c Config) ([]byte, error) {
 	mem := filesys.MakeFsInMemory()
 	err := fs.WalkDir(ateinstall.Postgres, ".", func(p string, d fs.DirEntry, err error) error {
 		if err != nil || d.IsDir() {
@@ -90,16 +114,33 @@ func postgresManifest(registry string) ([]byte, error) {
 		return nil, err
 	}
 	dir := "/kind/postgres"
-	if registry != "" {
-		k, err := yaml.Marshal(kusttypes.Kustomization{
-			TypeMeta:  kusttypes.TypeMeta{APIVersion: kusttypes.KustomizationVersion, Kind: kusttypes.KustomizationKind},
-			Resources: []string{"../kind/postgres"},
-			Images:    []kusttypes.Image{{Name: "postgres", NewName: registry + "/postgres"}},
-		})
+	overlay := kusttypes.Kustomization{
+		TypeMeta:  kusttypes.TypeMeta{APIVersion: kusttypes.KustomizationVersion, Kind: kusttypes.KustomizationKind},
+		Resources: []string{"../kind/postgres"},
+	}
+	if r := c.imageRegistry(); r != "" {
+		overlay.Images = []kusttypes.Image{{Name: "postgres", NewName: r + "/postgres"}}
+	}
+	if sc := c.StorageClass; sc != "" {
+		patch, err := yaml.Marshal([]map[string]any{{
+			"op":    "add",
+			"path":  "/spec/volumeClaimTemplates/0/spec/storageClassName",
+			"value": sc,
+		}})
+		if err != nil {
+			return nil, err
+		}
+		overlay.Patches = []kusttypes.Patch{{
+			Target: &kusttypes.Selector{ResId: resid.NewResIdWithNamespace(resid.NewGvk("apps", "v1", "StatefulSet"), postgresStatefulSet, SystemNamespace)},
+			Patch:  string(patch),
+		}}
+	}
+	if overlay.Images != nil || overlay.Patches != nil {
+		k, err := yaml.Marshal(overlay)
 		if err != nil {
 			return nil, err
 		}
-		dir = "/registry"
+		dir = "/overlay"
 		if err := mem.WriteFile(dir+"/kustomization.yaml", k); err != nil {
 			return nil, err
 		}
diff --git a/pkg/kagent/install/steps.go b/pkg/kagent/install/steps.go
index 679f3a653f..9b8baf61f2 100644
--- a/pkg/kagent/install/steps.go
+++ b/pkg/kagent/install/steps.go
@@ -199,12 +199,16 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s
 		// The release keeps its storage and database values.
 		return nil, forced, nil
 	}
+	var defaults map[string]any
 	if in.opts.ObjectStore.Type == ObjectStoreBundled {
 		access, secret, err := in.rustfsCredentials(ctx)
 		if err != nil {
 			return nil, nil, err
 		}
 		forced["rustfs"] = map[string]any{"accessKey": access, "secretKey": secret}
+		if sc := in.opts.StorageClass; sc != "" {
+			defaults = map[string]any{"rustfs": map[string]any{"storageClassName": sc}}
+		}
 	} else {
 		_, substrate := in.externalStoreValues()
 		forced = mergeValues(forced, substrate)
@@ -212,7 +216,7 @@ func (in *Installer) substrateValues(ctx context.Context) (map[string]any, map[s
 	if in.opts.Postgres.Type == PostgresBundled {
 		forced["postgres"] = map[string]any{"clientCertificates": map[string]any{"enabled": true}}
 	}
-	return nil, forced, nil
+	return defaults, forced, nil
 }
 
 // externalStoreValues returns the values Config.ObjectStore sets on the
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index 71cc72a9ea..f1b966aada 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -33,6 +33,7 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 	store := &cfg.ObjectStore
 	external := func() bool { return store.Type != install.ObjectStoreBundled }
 	aks := func() bool { return cfg.Profile == install.ProfileAKS }
+	bundledVolumes := func() bool { return cfg.Postgres.Type == install.PostgresBundled || !external() }
 	confirmed := true
 	form := huh.NewForm(
 		huh.NewGroup(
@@ -99,6 +100,15 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 				Value(&store.GCS.TransferServiceAccount).
 				Validate(notEmpty),
 		).WithHideFunc(func() bool { return store.Type != install.ObjectStoreGCS }),
+		huh.NewGroup(
+			huh.NewInput().
+				Title("StorageClass").
+				Description("For the bundled volumes. Leave empty for the cluster's default.").
+				PlaceholderFunc(func() string { return strings.Join(storageClassSuggestions(cfg.Profile), ", ") }, &cfg.Profile).
+				SuggestionsFunc(func() []string { return storageClassSuggestions(cfg.Profile) }, &cfg.Profile).
+				Value(&cfg.StorageClass).
+				Validate(install.ValidateStorageClass),
+		).WithHideFunc(func() bool { return !bundledVolumes() }),
 		huh.NewGroup(
 			huh.NewInput().
 				Title("MicroVM asset bucket").
@@ -136,6 +146,9 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 	if !external() || !cfg.MicroVM.Enabled {
 		cfg.MicroVM.AssetBucket = ""
 	}
+	if !bundledVolumes() {
+		cfg.StorageClass = ""
+	}
 	if !aks() {
 		cfg.AKS = install.AKS{}
 	}
@@ -156,6 +169,14 @@ func Hint(cfg *install.Config) (string, error) {
 	return "To skip these questions next time, save this as install.yaml and run kubectl ate install -c install.yaml:\n\n" + string(data) + "\n", nil
 }
 
+// storageClassSuggestions lists the StorageClasses to suggest on profile p.
+func storageClassSuggestions(p install.Profile) []string {
+	if p == install.ProfileAKS {
+		return []string{"managed-csi-premium"}
+	}
+	return nil
+}
+
 func notEmpty(s string) error {
 	if strings.TrimSpace(s) == "" {
 		return errors.New("required")
diff --git a/pkg/kagent/install/uninstall.go b/pkg/kagent/install/uninstall.go
index dd431b5f38..6c279731af 100644
--- a/pkg/kagent/install/uninstall.go
+++ b/pkg/kagent/install/uninstall.go
@@ -217,7 +217,7 @@ func (in *Installer) uninstallStep(r release) func(context.Context, func(string)
 // removePostgres deletes the bundled PostgreSQL objects that install applied,
 // identified by kubectl ate's field manager. Its volume stays until --purge.
 func (in *Installer) removePostgres(ctx context.Context, _ func(string)) (string, error) {
-	manifest, err := postgresManifest("")
+	manifest, err := postgresManifest(Config{})
 	if err != nil {
 		return "", err
 	}

From a835cc603463bac6c3f1a63e3eb2ab0fea0ca657 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 12:45:51 -0500
Subject: [PATCH 04/10] kubectl-ate: add an AKS node pool option

aks.nodePool restricts atelet to one node pool through its
kubernetes.azure.com/agentpool label. Install checks that the pool has
nodes, and with microvm enabled, that one of them advertises
ate.dev/kvm, which needs a VM size with nested virtualization.
WorkerPools need the same node selector, and install prints a note
saying so.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 cmd/kubectl-ate/internal/cmd/install.go |  6 +++
 pkg/kagent/install/aks.go               | 51 ++++++++++++++++++++
 pkg/kagent/install/aks_test.go          | 62 +++++++++++++++++++++++++
 pkg/kagent/install/config.go            | 45 +++++++++++++++---
 pkg/kagent/install/config_test.go       | 15 ++++--
 pkg/kagent/install/installer.go         |  7 +++
 pkg/kagent/install/tui/wizard.go        |  5 ++
 7 files changed, 182 insertions(+), 9 deletions(-)

diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index 2777baa9ef..dc772403fc 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -47,6 +47,7 @@ const configHelp = `--config takes a YAML file, or - for stdin:
   profile: aks             # or generic
   aks:                     # for profile: aks
     acr: my.azurecr.io     # pull the images from it
+    nodePool: agents       # run atelet only in this pool
   postgres:
     type: bundled          # or external
   objectStore:
@@ -109,6 +110,11 @@ bundled PostgreSQL's, from that Azure Container Registry. Import the images
 under their usual paths with az acr import and attach the registry with
 az aks update --attach-acr first.
 
+aks.nodePool runs atelet only on the nodes of that pool. Give each WorkerPool
+the nodeSelector kubernetes.azure.com/agentpool with the pool's name. With
+microvm.enabled, install checks that atelet's nodes have /dev/kvm, which needs
+a generation 2 VM size with nested virtualization.
+
 microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each
 node architecture, caches them under the user cache directory, uploads them to
 the object store, and applies the microvm SandboxConfig. Upgrade restages them.
diff --git a/pkg/kagent/install/aks.go b/pkg/kagent/install/aks.go
index 7e3e33a543..d4641a0279 100644
--- a/pkg/kagent/install/aks.go
+++ b/pkg/kagent/install/aks.go
@@ -17,10 +17,16 @@ package install
 import (
 	"context"
 	"errors"
+	"fmt"
+	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"go.ate.dev/substrate/internal/deviceplugin"
 )
 
 // networkPolicyEngines are the DaemonSets of the engines AKS installs with
@@ -41,6 +47,13 @@ func (in *Installer) checkAKS(ctx context.Context, progress func(string)) (strin
 		return "", err
 	}
 	summary := "NetworkPolicy by " + engine
+	if pool := in.opts.AKS.NodePool; pool != "" {
+		n, err := in.checkNodePool(ctx)
+		if err != nil {
+			return "", err
+		}
+		summary += fmt.Sprintf(", node pool %s with %d nodes", pool, n)
+	}
 	return summary, nil
 }
 
@@ -59,3 +72,41 @@ func (in *Installer) checkNetworkPolicy(ctx context.Context) (string, error) {
 	}
 	return "", errors.New("nothing in the cluster enforces NetworkPolicy; enable it with az aks update --network-policy, or set profile: generic to skip this check")
 }
+
+// checkNodePool returns how many nodes aks.nodePool has.
+func (in *Installer) checkNodePool(ctx context.Context) (int, error) {
+	pool := in.opts.AKS.NodePool
+	nodes, err := in.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: labels.SelectorFromSet(in.opts.nodeSelector()).String()})
+	if err != nil {
+		return 0, err
+	}
+	if len(nodes.Items) == 0 {
+		return 0, fmt.Errorf("node pool %s has no nodes; check aks.nodePool against az aks nodepool list", pool)
+	}
+	return len(nodes.Items), nil
+}
+
+// checkKVM waits for a node running atelet to advertise ate.dev/kvm, which
+// microVM workers request. atelet advertises it only on nodes with /dev/kvm.
+func (in *Installer) checkKVM(ctx context.Context, progress func(string)) (string, error) {
+	sel := labels.SelectorFromSet(in.opts.nodeSelector()).String()
+	var total int
+	err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) {
+		nodes, err := in.kube.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: sel})
+		if err != nil {
+			return false, err
+		}
+		total = len(nodes.Items)
+		for _, n := range nodes.Items {
+			if q, ok := n.Status.Allocatable[deviceplugin.ResourceKVM]; ok && !q.IsZero() {
+				return true, nil
+			}
+		}
+		progress(fmt.Sprintf("waiting for %s on %d nodes", deviceplugin.ResourceKVM, total))
+		return false, nil
+	})
+	if err != nil {
+		return "", fmt.Errorf("no node advertises %s, so microVM workers can't schedule; use a generation 2 VM size with nested virtualization, like Standard_D4s_v5: %w", deviceplugin.ResourceKVM, err)
+	}
+	return deviceplugin.ResourceKVM + " advertised", nil
+}
diff --git a/pkg/kagent/install/aks_test.go b/pkg/kagent/install/aks_test.go
index db7507b2cb..aff5c3149a 100644
--- a/pkg/kagent/install/aks_test.go
+++ b/pkg/kagent/install/aks_test.go
@@ -20,9 +20,13 @@ import (
 	"testing"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes/fake"
+
+	"go.ate.dev/substrate/internal/deviceplugin"
 )
 
 func TestCheckNetworkPolicy(t *testing.T) {
@@ -60,3 +64,61 @@ func TestCheckNetworkPolicy(t *testing.T) {
 		})
 	}
 }
+
+func node(name, pool string, kvm bool) runtime.Object {
+	n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{aksNodePoolLabel: pool}}}
+	if kvm {
+		n.Status.Allocatable = corev1.ResourceList{deviceplugin.ResourceKVM: resource.MustParse("4096")}
+	}
+	return n
+}
+
+func TestCheckNodePool(t *testing.T) {
+	kc := fake.NewClientset(node("a", "agents", false), node("b", "agents", false), node("c", "system", false))
+	for _, tc := range []struct {
+		pool    string
+		want    int
+		wantErr string
+	}{
+		{pool: "agents", want: 2},
+		{pool: "gpu", wantErr: "node pool gpu has no nodes"},
+	} {
+		in := &Installer{kube: kc, opts: Options{Config: Config{Profile: ProfileAKS, AKS: AKS{NodePool: tc.pool}}}}
+		got, err := in.checkNodePool(context.Background())
+		if tc.wantErr != "" {
+			if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
+				t.Errorf("pool %s: checkNodePool() error = %v, want it to contain %q", tc.pool, err, tc.wantErr)
+			}
+			continue
+		}
+		if err != nil || got != tc.want {
+			t.Errorf("pool %s: checkNodePool() = %d, %v, want %d", tc.pool, got, err, tc.want)
+		}
+	}
+}
+
+func TestCheckKVM(t *testing.T) {
+	tests := []struct {
+		name    string
+		pool    string
+		nodes   []runtime.Object
+		wantErr bool
+	}{
+		{name: "any node", nodes: []runtime.Object{node("a", "system", false), node("b", "agents", true)}},
+		{name: "in pool", pool: "agents", nodes: []runtime.Object{node("a", "agents", true)}},
+		{name: "outside pool", pool: "agents", nodes: []runtime.Object{node("a", "agents", false), node("b", "system", true)}, wantErr: true},
+		{name: "none", nodes: []runtime.Object{node("a", "system", false)}, wantErr: true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			in := &Installer{kube: fake.NewClientset(tc.nodes...), opts: Options{Config: Config{Profile: ProfileAKS, AKS: AKS{NodePool: tc.pool}}}}
+			// The first check runs before the poll sees the canceled context.
+			ctx, cancel := context.WithCancel(context.Background())
+			cancel()
+			_, err := in.checkKVM(ctx, func(string) {})
+			if gotErr := err != nil; gotErr != tc.wantErr {
+				t.Errorf("checkKVM() error = %v, want error %t", err, tc.wantErr)
+			}
+		})
+	}
+}
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index 5bec1e4e2a..24822dd49b 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -192,6 +192,9 @@ type AKS struct {
 	// registry must hold the images under their usual paths, and the cluster
 	// must be attached to it.
 	ACR string `json:"acr,omitempty"`
+	// NodePool restricts atelet to the nodes of one node pool. WorkerPools
+	// then need the same node selector.
+	NodePool string `json:"nodePool,omitempty"`
 }
 
 // MicroVM configures the microvm sandbox class.
@@ -271,6 +274,14 @@ func ValidateACR(s string) error {
 	return nil
 }
 
+// ValidateNodePool checks that s is empty or a valid node pool label value.
+func ValidateNodePool(s string) error {
+	if errs := validation.IsValidLabelValue(s); len(errs) > 0 {
+		return fmt.Errorf("invalid node pool %q: %s", s, strings.Join(errs, "; "))
+	}
+	return nil
+}
+
 // ValidateStorageClass checks that s is empty or a valid StorageClass name.
 func ValidateStorageClass(s string) error {
 	if s == "" {
@@ -292,6 +303,9 @@ func (c Config) validate() error {
 	if err := ValidateACR(c.AKS.ACR); err != nil {
 		return fmt.Errorf("aks.acr: %w", err)
 	}
+	if err := ValidateNodePool(c.AKS.NodePool); err != nil {
+		return fmt.Errorf("aks.nodePool: %w", err)
+	}
 	switch c.Postgres.Type {
 	case PostgresBundled, PostgresExternal:
 	default:
@@ -333,14 +347,34 @@ func (c Config) imageRegistry() string {
 	return ""
 }
 
-// profileValues returns Helm values for every release except substrate-crds.
-func (c Config) profileValues() map[string]any {
-	if r := c.imageRegistry(); r != "" {
-		return map[string]any{"global": map[string]any{"imageRegistry": r}}
+// aksNodePoolLabel is the node label AKS sets to the node's pool name.
+const aksNodePoolLabel = "kubernetes.azure.com/agentpool"
+
+// nodeSelector returns the node selector atelet runs with, or nil for every
+// node.
+func (c Config) nodeSelector() map[string]string {
+	if c.Profile == ProfileAKS && c.AKS.NodePool != "" {
+		return map[string]string{aksNodePoolLabel: c.AKS.NodePool}
 	}
 	return nil
 }
 
+// profileValues returns the profile's Helm values for r.
+func (c Config) profileValues(r release) map[string]any {
+	vals := map[string]any{}
+	if reg := c.imageRegistry(); reg != "" {
+		vals["global"] = map[string]any{"imageRegistry": reg}
+	}
+	if sel := c.nodeSelector(); sel != nil && r == ateletRelease {
+		nodeSelector := map[string]any{}
+		for k, v := range sel {
+			nodeSelector[k] = v
+		}
+		vals["atelet"] = map[string]any{"nodeSelector": nodeSelector}
+	}
+	return vals
+}
+
 func (r *Releases) all() []*ReleaseValues {
 	return []*ReleaseValues{&r.All, &r.PodCert, &r.Atelet, &r.Substrate}
 }
@@ -352,7 +386,6 @@ func (c Config) userValues() (map[string]map[string]any, error) {
 	if err != nil {
 		return nil, err
 	}
-	all = mergeValues(c.profileValues(), all)
 	user := map[string]map[string]any{}
 	for r, rv := range map[release]ReleaseValues{
 		podCertRelease:   c.Releases.PodCert,
@@ -363,7 +396,7 @@ func (c Config) userValues() (map[string]map[string]any, error) {
 		if err != nil {
 			return nil, err
 		}
-		user[r.name] = mergeValues(all, own)
+		user[r.name] = mergeValues(c.profileValues(r), all, own)
 	}
 	return user, nil
 }
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index d84c2f0a47..1cd88370df 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -94,6 +94,8 @@ releases:
 		{name: "aks needs profile", in: "aks: {acr: r.azurecr.io}\n", wantErr: "aks needs profile: aks"},
 		{name: "acr with path", in: "profile: aks\naks: {acr: r.azurecr.io/substrate}\n", wantErr: "invalid ACR"},
 		{name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"},
+		{name: "node pool", in: "profile: aks\naks: {nodePool: agents}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.NodePool = ProfileAKS, "agents" })},
+		{name: "bad node pool", in: "profile: aks\naks: {nodePool: 'agents pool'}\n", wantErr: "invalid node pool"},
 		{name: "acr with space", in: "profile: aks\naks: {acr: 'r.azurecr.io '}\n", wantErr: "invalid ACR"},
 		{name: "unknown field", in: "postgress: external\n", wantErr: "postgress"},
 		{name: "unknown release", in: "releases: {crds: {values: {a: 1}}}\n", wantErr: "crds"},
@@ -191,8 +193,9 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	}
 	c := Config{
 		Profile: ProfileAKS,
-		AKS:     AKS{ACR: "r.azurecr.io"},
+		AKS:     AKS{ACR: "r.azurecr.io", NodePool: "agents"},
 		Releases: Releases{
+			Atelet:    ReleaseValues{Values: map[string]any{"atelet": map[string]any{"nodeSelector": map[string]any{"disk": "ssd"}}}},
 			Substrate: ReleaseValues{Values: registry("mirror.example.com")},
 		},
 	}
@@ -200,10 +203,16 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	// The release's own values win over the profile's.
+	// The release's own values win over the profile's, and only atelet gets
+	// the node pool.
+	atelet := registry("r.azurecr.io")
+	atelet["atelet"] = map[string]any{"nodeSelector": map[string]any{
+		"kubernetes.azure.com/agentpool": "agents",
+		"disk":                           "ssd",
+	}}
 	want := map[string]map[string]any{
 		"substrate-podcert": registry("r.azurecr.io"),
-		"substrate-atelet":  registry("r.azurecr.io"),
+		"substrate-atelet":  atelet,
 		"substrate":         registry("mirror.example.com"),
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
diff --git a/pkg/kagent/install/installer.go b/pkg/kagent/install/installer.go
index 45ec7df55c..8a8c1a9033 100644
--- a/pkg/kagent/install/installer.go
+++ b/pkg/kagent/install/installer.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
@@ -182,6 +183,9 @@ func (in *Installer) InstallSteps() ([]Step, error) {
 	if in.opts.Postgres.Type == PostgresBundled {
 		in.report.Report(Event{Kind: Note, Msg: "Installing a single-replica PostgreSQL for evaluation only. It has no backups or high availability. Set postgres.type to external with a managed database for anything else."})
 	}
+	if sel := in.opts.nodeSelector(); sel != nil {
+		in.report.Report(Event{Kind: Note, Msg: fmt.Sprintf("atelet runs only on nodes labeled %s. Give each WorkerPool the same spec.template.nodeSelector, or its workers won't become ready.", labels.SelectorFromSet(sel))})
+	}
 	steps := []Step{
 		{"Check cluster", in.preflight},
 		{"Install CRDs", in.helmStep(crdsRelease, noValues)},
@@ -192,6 +196,9 @@ func (in *Installer) InstallSteps() ([]Step, error) {
 		{"Configure object storage", in.configureObjectStore},
 		{"Install atelet", in.helmStep(ateletRelease, in.ateletValues)},
 	}
+	if in.opts.Profile == ProfileAKS && in.opts.MicroVM.Enabled {
+		steps = append(steps, Step{"Check KVM", in.checkKVM})
+	}
 	if in.opts.Postgres.Type == PostgresBundled {
 		steps = append(steps,
 			Step{"Install PostgreSQL", in.applyPostgres},
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index f1b966aada..80debaad01 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -51,6 +51,11 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 				Description("Login server to pull Agent Substrate's images from, like myregistry.azurecr.io. It must hold\nthe images under their usual paths, and the cluster must be attached to it. Leave empty for the public registries.").
 				Value(&cfg.AKS.ACR).
 				Validate(install.ValidateACR),
+			huh.NewInput().
+				Title("Node pool").
+				Description("Runs atelet only in this pool. WorkerPools then need the nodeSelector\nkubernetes.azure.com/agentpool set to it. Leave empty for every node.").
+				Value(&cfg.AKS.NodePool).
+				Validate(install.ValidateNodePool),
 		).WithHideFunc(func() bool { return !aks() }),
 		huh.NewGroup(
 			huh.NewSelect[install.PostgresType]().

From 210b8147c1e0767ad64fa98a65400f3304efd473 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 14:07:05 -0500
Subject: [PATCH 05/10] kubectl-ate: add an AKS ACR credential provider option

aks.acrCredentialProvider lets atelet pull actor images from Azure
Container Registry. atelet runs the node's acr-credential-provider with
the kubelet's own config. The provider reads /etc/kubernetes/azure.json,
which the charts now mount through a new
atelet.imageCredentialProviderHostFiles value.

It is off by default, since atelet can then pull from every registry the
kubelet identity can. With profile aks, turning it off resets those
values, so an earlier install's access goes away.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 charts/substrate-atelet/templates/atelet.yaml | 14 ++++
 charts/substrate-atelet/values.yaml           |  3 +
 charts/substrate/README.md                    |  1 +
 charts/substrate/templates/atelet.yaml        | 14 ++++
 charts/substrate/values.yaml                  |  3 +
 cmd/atelet/helm_test.go                       | 33 +++++++++-
 cmd/kubectl-ate/internal/cmd/install.go       | 10 +++
 pkg/kagent/install/config.go                  | 41 ++++++++++--
 pkg/kagent/install/config_test.go             | 66 +++++++++++++++++--
 pkg/kagent/install/tui/wizard.go              |  4 ++
 10 files changed, 178 insertions(+), 11 deletions(-)

diff --git a/charts/substrate-atelet/templates/atelet.yaml b/charts/substrate-atelet/templates/atelet.yaml
index c8374bbf94..d172eb9e75 100644
--- a/charts/substrate-atelet/templates/atelet.yaml
+++ b/charts/substrate-atelet/templates/atelet.yaml
@@ -21,6 +21,9 @@ limitations under the License.
 {{- if ne (empty $credentialProviderConfig) (empty .Values.atelet.imageCredentialProviderBinDir) }}
 {{- fail "atelet.imageCredentialProviderConfig (or atelet.imageCredentialProviderConfigYAML) and atelet.imageCredentialProviderBinDir must be set together" }}
 {{- end }}
+{{- if and .Values.atelet.imageCredentialProviderHostFiles (not $credentialProviderConfig) }}
+{{- fail "atelet.imageCredentialProviderHostFiles needs atelet.imageCredentialProviderConfig or atelet.imageCredentialProviderConfigYAML" }}
+{{- end }}
 
 # atelet
 apiVersion: v1
@@ -308,6 +311,11 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
+{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
+        - name: image-credential-provider-file-{{ $i }}
+          mountPath: {{ $path | quote }}
+          readOnly: true
+{{- end }}
 {{- end }}
       volumes:
       - name: objectstore-plugin
@@ -345,6 +353,12 @@ spec:
           path: {{ .Values.atelet.imageCredentialProviderConfig | quote }}
           type: File
 {{- end }}
+{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
+      - name: image-credential-provider-file-{{ $i }}
+        hostPath:
+          path: {{ $path | quote }}
+          type: File
+{{- end }}
 {{- end }}
       - name: podidentity
         projected:
diff --git a/charts/substrate-atelet/values.yaml b/charts/substrate-atelet/values.yaml
index b68515745e..01ac4eb01a 100644
--- a/charts/substrate-atelet/values.yaml
+++ b/charts/substrate-atelet/values.yaml
@@ -56,6 +56,9 @@ atelet:
   # either one still needs the bin directory.
   imageCredentialProviderConfigYAML: ""
   imageCredentialProviderBinDir: ""
+  # Other host files the providers read, mounted read-only at the same path
+  # (AKS: /etc/kubernetes/azure.json).
+  imageCredentialProviderHostFiles: []
   # Restricts atelet to matching nodes. Every node that runs WorkerPool pods
   # needs an atelet, so cover each pool's nodeSelector here.
   nodeSelector: {}
diff --git a/charts/substrate/README.md b/charts/substrate/README.md
index 20fe98412b..30310d597f 100644
--- a/charts/substrate/README.md
+++ b/charts/substrate/README.md
@@ -97,6 +97,7 @@ See `values.yaml` for the full set; the important keys:
 | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path |
 | `atelet.nodeSelector` | `{}` | Node labels atelet pods must match; must cover the nodes of every WorkerPool |
 | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only |
+| `atelet.imageCredentialProviderHostFiles` | `[]` | Other host files the providers read, mounted read-only at the same path; needs a provider config |
 | `credentialProvider.enabled` | `true` | Deploy the Kubernetes Secret provider and route `k8s.io` credential URIs to it. With no provider, every credential injection fails closed |
 | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider |
 | `ateApi.actorJWTIssuer` | `https://<idp-service>.<namespace>.svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties |
diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml
index e8c3ecad3b..8c492a777b 100644
--- a/charts/substrate/templates/atelet.yaml
+++ b/charts/substrate/templates/atelet.yaml
@@ -22,6 +22,9 @@ limitations under the License.
 {{- if ne (empty $credentialProviderConfig) (empty .Values.atelet.imageCredentialProviderBinDir) }}
 {{- fail "atelet.imageCredentialProviderConfig (or atelet.imageCredentialProviderConfigYAML) and atelet.imageCredentialProviderBinDir must be set together" }}
 {{- end }}
+{{- if and .Values.atelet.imageCredentialProviderHostFiles (not $credentialProviderConfig) }}
+{{- fail "atelet.imageCredentialProviderHostFiles needs atelet.imageCredentialProviderConfig or atelet.imageCredentialProviderConfigYAML" }}
+{{- end }}
 
 # atelet
 apiVersion: v1
@@ -309,6 +312,11 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
+{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
+        - name: image-credential-provider-file-{{ $i }}
+          mountPath: {{ $path | quote }}
+          readOnly: true
+{{- end }}
 {{- end }}
       volumes:
       - name: objectstore-plugin
@@ -346,6 +354,12 @@ spec:
           path: {{ .Values.atelet.imageCredentialProviderConfig | quote }}
           type: File
 {{- end }}
+{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
+      - name: image-credential-provider-file-{{ $i }}
+        hostPath:
+          path: {{ $path | quote }}
+          type: File
+{{- end }}
 {{- end }}
       - name: podidentity
         projected:
diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml
index 87e9c55c67..30037c3ef3 100644
--- a/charts/substrate/values.yaml
+++ b/charts/substrate/values.yaml
@@ -90,6 +90,9 @@ atelet:
   # either one still needs the bin directory.
   imageCredentialProviderConfigYAML: ""
   imageCredentialProviderBinDir: ""
+  # Other host files the providers read, mounted read-only at the same path
+  # (AKS: /etc/kubernetes/azure.json).
+  imageCredentialProviderHostFiles: []
   storageBackend: s3
   # Restricts atelet to matching nodes. Every node that runs WorkerPool pods
   # needs an atelet, so cover each pool's nodeSelector here.
diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go
index 63fce1ddf7..a5ff1c803b 100644
--- a/cmd/atelet/helm_test.go
+++ b/cmd/atelet/helm_test.go
@@ -16,6 +16,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"io"
 	"os"
@@ -43,10 +44,18 @@ kind: CredentialProviderConfig
 providers: []`
 	for _, tc := range []struct {
 		name, config, configYAML, binDir string
+		hostFiles                        []string
 		wantError                        string
 	}{
 		{name: "public registries"},
 		{name: "node provider", config: "/etc/srv/kubernetes/cri_auth_config.yaml", binDir: "/home/kubernetes/bin"},
+		{
+			name:      "node provider with host files",
+			config:    "/var/lib/kubelet/credential-provider-config.yaml",
+			binDir:    "/var/lib/kubelet/credential-provider",
+			hostFiles: []string{"/etc/kubernetes/azure.json", "/etc/provider/ca.pem"},
+		},
+		{name: "host files without config", hostFiles: []string{"/etc/kubernetes/azure.json"}, wantError: "imageCredentialProviderHostFiles needs"},
 		{name: "chart provided config", configYAML: providerConfig, binDir: "/home/kubernetes/bin"},
 		{name: "missing bin directory", config: "/etc/provider.yaml", wantError: "must be set together"},
 		{name: "chart provided config without bin directory", configYAML: providerConfig, wantError: "must be set together"},
@@ -60,11 +69,16 @@ providers: []`
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			hostFiles, err := json.Marshal(append([]string{}, tc.hostFiles...))
+			if err != nil {
+				t.Fatal(err)
+			}
 			out, err := exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate",
 				"--show-only", "templates/atelet.yaml",
 				"--set-string", "atelet.imageCredentialProviderConfig="+tc.config,
 				"--set-string", "atelet.imageCredentialProviderConfigYAML="+tc.configYAML,
-				"--set-string", "atelet.imageCredentialProviderBinDir="+tc.binDir).CombinedOutput()
+				"--set-string", "atelet.imageCredentialProviderBinDir="+tc.binDir,
+				"--set-json", "atelet.imageCredentialProviderHostFiles="+string(hostFiles)).CombinedOutput()
 			if tc.wantError != "" {
 				if err == nil || !strings.Contains(string(out), tc.wantError) {
 					t.Fatalf("render rejected provider config with %q: %v\n%s", tc.wantError, err, out)
@@ -141,6 +155,20 @@ providers: []`
 					}
 				}
 			}
+			var gotFiles []string
+			for _, m := range container.VolumeMounts {
+				if !strings.HasPrefix(m.Name, "image-credential-provider-file-") {
+					continue
+				}
+				i := slices.IndexFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == m.Name })
+				if i < 0 || pod.Volumes[i].HostPath == nil || pod.Volumes[i].HostPath.Path != m.MountPath || !m.ReadOnly {
+					t.Errorf("%s must mount host file %s read-only at the same path", m.Name, m.MountPath)
+				}
+				gotFiles = append(gotFiles, m.MountPath)
+			}
+			if !slices.Equal(gotFiles, tc.hostFiles) {
+				t.Errorf("atelet mounts host files %v, want %v", gotFiles, tc.hostFiles)
+			}
 		})
 	}
 }
@@ -285,6 +313,9 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) {
 				"--set", "preview[0]=ALL",
 				"--set", "otel.endpoint=http://collector:4317",
 				"--set", "atelet.nodeSelector.pool=agents",
+				"--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml",
+				"--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider",
+				"--set", "atelet.imageCredentialProviderHostFiles[0]=/etc/kubernetes/azure.json",
 				"--set", "image.tag=test",
 			},
 			substrate: append([]string{"--set-json", `objectStorePlugin.extraEnv=[
diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index dc772403fc..76b87582b2 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -48,6 +48,7 @@ const configHelp = `--config takes a YAML file, or - for stdin:
   aks:                     # for profile: aks
     acr: my.azurecr.io     # pull the images from it
     nodePool: agents       # run atelet only in this pool
+    acrCredentialProvider: true # pull actor images from ACR
   postgres:
     type: bundled          # or external
   objectStore:
@@ -115,6 +116,15 @@ the nodeSelector kubernetes.azure.com/agentpool with the pool's name. With
 microvm.enabled, install checks that atelet's nodes have /dev/kvm, which needs
 a generation 2 VM size with nested virtualization.
 
+aks.acrCredentialProvider lets atelet pull actor images from Azure Container
+Registry. atelet runs the node's acr-credential-provider, which reads the
+node's /etc/kubernetes/azure.json, so atelet can pull from every registry the
+kubelet identity can. Grant that identity AcrPull, or attach the registry. On
+a cluster that uses a service principal instead of a managed identity, that
+file holds the service principal's secret, so atelet holds it too. With
+profile aks and the option off, install and upgrade reset atelet's
+imageCredentialProvider values, unless the config's releases set them.
+
 microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each
 node architecture, caches them under the user cache directory, uploads them to
 the object store, and applies the microvm SandboxConfig. Upgrade restages them.
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index 24822dd49b..31dd7d9491 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -195,6 +195,11 @@ type AKS struct {
 	// NodePool restricts atelet to the nodes of one node pool. WorkerPools
 	// then need the same node selector.
 	NodePool string `json:"nodePool,omitempty"`
+	// ACRCredentialProvider lets atelet pull actor images from ACR with the
+	// node's acr-credential-provider. atelet then reads the node's
+	// /etc/kubernetes/azure.json and holds the kubelet identity's registry
+	// credentials, or a service principal's secret on clusters that use one.
+	ACRCredentialProvider bool `json:"acrCredentialProvider,omitempty"`
 }
 
 // MicroVM configures the microvm sandbox class.
@@ -350,6 +355,14 @@ func (c Config) imageRegistry() string {
 // aksNodePoolLabel is the node label AKS sets to the node's pool name.
 const aksNodePoolLabel = "kubernetes.azure.com/agentpool"
 
+// The kubelet's credential provider files on AKS nodes. The provider reads
+// the kubelet identity from aksCloudConfig.
+const (
+	aksCredentialProviderConfig = "/var/lib/kubelet/credential-provider-config.yaml"
+	aksCredentialProviderBinDir = "/var/lib/kubelet/credential-provider"
+	aksCloudConfig              = "/etc/kubernetes/azure.json"
+)
+
 // nodeSelector returns the node selector atelet runs with, or nil for every
 // node.
 func (c Config) nodeSelector() map[string]string {
@@ -365,12 +378,30 @@ func (c Config) profileValues(r release) map[string]any {
 	if reg := c.imageRegistry(); reg != "" {
 		vals["global"] = map[string]any{"imageRegistry": reg}
 	}
-	if sel := c.nodeSelector(); sel != nil && r == ateletRelease {
-		nodeSelector := map[string]any{}
-		for k, v := range sel {
-			nodeSelector[k] = v
+	if r == ateletRelease {
+		atelet := map[string]any{}
+		if sel := c.nodeSelector(); sel != nil {
+			nodeSelector := map[string]any{}
+			for k, v := range sel {
+				nodeSelector[k] = v
+			}
+			atelet["nodeSelector"] = nodeSelector
+		}
+		if c.Profile == ProfileAKS {
+			// Off resets the chart defaults, so atelet loses the kubelet
+			// identity's registry access that an earlier install gave it.
+			config, binDir, hostFiles := "", "", []any{}
+			if c.AKS.ACRCredentialProvider {
+				config, binDir, hostFiles = aksCredentialProviderConfig, aksCredentialProviderBinDir, []any{aksCloudConfig}
+			}
+			atelet["imageCredentialProviderConfig"] = config
+			atelet["imageCredentialProviderConfigYAML"] = ""
+			atelet["imageCredentialProviderBinDir"] = binDir
+			atelet["imageCredentialProviderHostFiles"] = hostFiles
+		}
+		if len(atelet) > 0 {
+			vals["atelet"] = atelet
 		}
-		vals["atelet"] = map[string]any{"nodeSelector": nodeSelector}
 	}
 	return vals
 }
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index 1cd88370df..cb3162eb6d 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -95,6 +95,7 @@ releases:
 		{name: "acr with path", in: "profile: aks\naks: {acr: r.azurecr.io/substrate}\n", wantErr: "invalid ACR"},
 		{name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"},
 		{name: "node pool", in: "profile: aks\naks: {nodePool: agents}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.NodePool = ProfileAKS, "agents" })},
+		{name: "acr credential provider", in: "profile: aks\naks: {acrCredentialProvider: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ACRCredentialProvider = ProfileAKS, true })},
 		{name: "bad node pool", in: "profile: aks\naks: {nodePool: 'agents pool'}\n", wantErr: "invalid node pool"},
 		{name: "acr with space", in: "profile: aks\naks: {acr: 'r.azurecr.io '}\n", wantErr: "invalid ACR"},
 		{name: "unknown field", in: "postgress: external\n", wantErr: "postgress"},
@@ -193,7 +194,7 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	}
 	c := Config{
 		Profile: ProfileAKS,
-		AKS:     AKS{ACR: "r.azurecr.io", NodePool: "agents"},
+		AKS:     AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true},
 		Releases: Releases{
 			Atelet:    ReleaseValues{Values: map[string]any{"atelet": map[string]any{"nodeSelector": map[string]any{"disk": "ssd"}}}},
 			Substrate: ReleaseValues{Values: registry("mirror.example.com")},
@@ -206,10 +207,16 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	// The release's own values win over the profile's, and only atelet gets
 	// the node pool.
 	atelet := registry("r.azurecr.io")
-	atelet["atelet"] = map[string]any{"nodeSelector": map[string]any{
-		"kubernetes.azure.com/agentpool": "agents",
-		"disk":                           "ssd",
-	}}
+	atelet["atelet"] = map[string]any{
+		"nodeSelector": map[string]any{
+			"kubernetes.azure.com/agentpool": "agents",
+			"disk":                           "ssd",
+		},
+		"imageCredentialProviderConfig":     "/var/lib/kubelet/credential-provider-config.yaml",
+		"imageCredentialProviderConfigYAML": "",
+		"imageCredentialProviderBinDir":     "/var/lib/kubelet/credential-provider",
+		"imageCredentialProviderHostFiles":  []any{"/etc/kubernetes/azure.json"},
+	}
 	want := map[string]map[string]any{
 		"substrate-podcert": registry("r.azurecr.io"),
 		"substrate-atelet":  atelet,
@@ -219,3 +226,52 @@ func TestConfigUserValuesProfile(t *testing.T) {
 		t.Errorf("userValues() mismatch (-want +got):\n%s", diff)
 	}
 }
+
+func TestConfigUserValuesACRCredentialProvider(t *testing.T) {
+	provider := func(config, binDir string, hostFiles ...any) map[string]any {
+		if hostFiles == nil {
+			hostFiles = []any{}
+		}
+		return map[string]any{
+			"imageCredentialProviderConfig":     config,
+			"imageCredentialProviderConfigYAML": "",
+			"imageCredentialProviderBinDir":     binDir,
+			"imageCredentialProviderHostFiles":  hostFiles,
+		}
+	}
+	for _, tc := range []struct {
+		name string
+		c    Config
+		want map[string]any
+	}{
+		{name: "generic leaves it alone", c: Config{Profile: ProfileGeneric}},
+		{
+			name: "aks on",
+			c:    Config{Profile: ProfileAKS, AKS: AKS{ACRCredentialProvider: true}},
+			want: provider("/var/lib/kubelet/credential-provider-config.yaml", "/var/lib/kubelet/credential-provider", "/etc/kubernetes/azure.json"),
+		},
+		{
+			name: "aks off resets it",
+			c:    Config{Profile: ProfileAKS},
+			want: provider("", ""),
+		},
+		{
+			name: "aks off keeps the release's own",
+			c: Config{Profile: ProfileAKS, Releases: Releases{Atelet: ReleaseValues{Values: map[string]any{
+				"atelet": provider("/etc/provider.yaml", "/opt/provider"),
+			}}}},
+			want: provider("/etc/provider.yaml", "/opt/provider"),
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := tc.c.userValues()
+			if err != nil {
+				t.Fatal(err)
+			}
+			atelet, _ := got[ateletRelease.name]["atelet"].(map[string]any)
+			if diff := cmp.Diff(tc.want, atelet); diff != "" {
+				t.Errorf("atelet values mismatch (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index 80debaad01..a80f187863 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -56,6 +56,10 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 				Description("Runs atelet only in this pool. WorkerPools then need the nodeSelector\nkubernetes.azure.com/agentpool set to it. Leave empty for every node.").
 				Value(&cfg.AKS.NodePool).
 				Validate(install.ValidateNodePool),
+			huh.NewConfirm().
+				Title("Pull actor images from ACR?").
+				Description("Runs the node's acr-credential-provider in atelet. atelet reads the node's\n/etc/kubernetes/azure.json and gets every registry the kubelet identity can pull from.\nOn a service principal cluster, that file holds the service principal's secret.").
+				Value(&cfg.AKS.ACRCredentialProvider),
 		).WithHideFunc(func() bool { return !aks() }),
 		huh.NewGroup(
 			huh.NewSelect[install.PostgresType]().

From c4e47e73f2daf5df9c554ee3c63f4cc2f20493dd Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 14:20:44 -0500
Subject: [PATCH 06/10] kubectl-ate: add an otelEndpoint install option

otelEndpoint points atelet and the control plane at an existing OTLP
collector by setting otel.endpoint on the atelet and substrate releases.
The wizard asks for it. Install doesn't deploy a collector.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 cmd/kubectl-ate/internal/cmd/install.go | 11 ++++++---
 pkg/kagent/install/config.go            | 33 +++++++++++++++++++++++--
 pkg/kagent/install/config_test.go       | 23 ++++++++++++-----
 pkg/kagent/install/tui/tui_test.go      |  1 +
 pkg/kagent/install/tui/wizard.go        |  7 ++++++
 5 files changed, 64 insertions(+), 11 deletions(-)

diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index 76b87582b2..c646b57a20 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -65,6 +65,7 @@ const configHelp = `--config takes a YAML file, or - for stdin:
     enabled: true
     assetBucket: my-assets # for s3 or gcs
   storageClass: fast       # bundled volumes; empty for the default
+  otelEndpoint: http://otel-collector:4317 # your OTLP collector
   timeout: 10m             # each wait for workloads
   releases:                # Helm values
     all:                   # every release except substrate-crds
@@ -78,7 +79,8 @@ const configHelp = `--config takes a YAML file, or - for stdin:
     substrate:             # substrate
       values:
         otel:
-          endpoint: http://otel-collector:4317
+          traces:
+            samplingRatio: 0.1
 
 A release's own valuesFiles and values win over the ones in all.`
 
@@ -103,6 +105,9 @@ objectStore.bucket. GCS also needs objectStore.gcs.transferServiceAccount.
 storageClass sets the StorageClass of the bundled PostgreSQL and RustFS
 volumes when they are created. On AKS, managed-csi-premium uses Premium SSDs.
 
+otelEndpoint points atelet and the control plane at an OTLP collector you
+already run, for traces, metrics and logs. Install doesn't deploy one.
+
 profile selects the kind of cluster, and --profile sets it from the command
 line. generic makes no assumptions. aks is for Azure Kubernetes Service. It
 checks that a NetworkPolicy engine is installed, since AKS ignores
@@ -164,8 +169,8 @@ release. It doesn't roll WorkerPool worker images, which
 docs/reference/version-skew-policy.md upgrades after atelet.
 
 Each release keeps the values it was installed with. The values from the
-config's profile and releases are layered on top. With profile aks, upgrade
-first runs install's AKS checks. Upgrade also reads timeout and
+config's profile, otelEndpoint and releases are layered on top. With profile
+aks, upgrade first runs install's AKS checks. Upgrade also reads timeout and
 microvm.assetBucket from the config. It refuses downgrades, skipped minor
 releases, and CRD changes that would strand stored objects.
 
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index 31dd7d9491..b769e2bef9 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -45,6 +46,10 @@ type Config struct {
 	// StorageClass is the StorageClass of the bundled PostgreSQL and RustFS
 	// volumes. Empty uses the cluster's default.
 	StorageClass string `json:"storageClass,omitempty"`
+	// OTelEndpoint is an existing OTLP collector that atelet and the control
+	// plane export traces, metrics and logs to. Empty leaves the charts' otel
+	// values as they are.
+	OTelEndpoint string `json:"otelEndpoint,omitempty"`
 	// Timeout bounds each wait for workloads to become ready.
 	Timeout metav1.Duration `json:"timeout,omitzero"`
 	// Releases holds Helm values for the releases.
@@ -298,6 +303,18 @@ func ValidateStorageClass(s string) error {
 	return nil
 }
 
+// ValidateOTelEndpoint checks that s is empty or an http or https URL.
+func ValidateOTelEndpoint(s string) error {
+	if s == "" {
+		return nil
+	}
+	u, err := url.Parse(s)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+		return fmt.Errorf("invalid OTLP endpoint %q: must be a URL like http://otel-collector:4317", s)
+	}
+	return nil
+}
+
 func (c Config) validate() error {
 	if err := validateProfile(c.Profile); err != nil {
 		return err
@@ -327,6 +344,9 @@ func (c Config) validate() error {
 			return fmt.Errorf("storageClass needs postgres.type: %s or objectStore.type: %s", PostgresBundled, ObjectStoreBundled)
 		}
 	}
+	if err := ValidateOTelEndpoint(c.OTelEndpoint); err != nil {
+		return fmt.Errorf("otelEndpoint: %w", err)
+	}
 	if c.MicroVM.Enabled && c.ObjectStore.Type != ObjectStoreBundled && c.MicroVM.AssetBucket == "" {
 		return errors.New("microvm with an s3 or gcs objectStore needs microvm.assetBucket")
 	}
@@ -410,8 +430,17 @@ func (r *Releases) all() []*ReleaseValues {
 	return []*ReleaseValues{&r.All, &r.PodCert, &r.Atelet, &r.Substrate}
 }
 
+// otelValues returns the Helm values that point r at OTelEndpoint.
+func (c Config) otelValues(r release) map[string]any {
+	if c.OTelEndpoint == "" || (r != ateletRelease && r != substrateRelease) {
+		return nil
+	}
+	return map[string]any{"otel": map[string]any{"endpoint": c.OTelEndpoint}}
+}
+
 // userValues returns each release's values, keyed by release name: the
-// profile's, All's files and values, then the release's own.
+// profile's and OTelEndpoint's, All's files and values, then the release's
+// own.
 func (c Config) userValues() (map[string]map[string]any, error) {
 	all, err := c.Releases.All.read()
 	if err != nil {
@@ -427,7 +456,7 @@ func (c Config) userValues() (map[string]map[string]any, error) {
 		if err != nil {
 			return nil, err
 		}
-		user[r.name] = mergeValues(c.profileValues(r), all, own)
+		user[r.name] = mergeValues(c.profileValues(r), c.otelValues(r), all, own)
 	}
 	return user, nil
 }
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index cb3162eb6d..a0e7206d37 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -96,6 +96,8 @@ releases:
 		{name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"},
 		{name: "node pool", in: "profile: aks\naks: {nodePool: agents}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.NodePool = ProfileAKS, "agents" })},
 		{name: "acr credential provider", in: "profile: aks\naks: {acrCredentialProvider: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ACRCredentialProvider = ProfileAKS, true })},
+		{name: "otel endpoint", in: "otelEndpoint: http://collector:4317\n", want: withDefaults(func(c *Config) { c.OTelEndpoint = "http://collector:4317" })},
+		{name: "otel endpoint without scheme", in: "otelEndpoint: collector:4317\n", wantErr: "invalid OTLP endpoint"},
 		{name: "bad node pool", in: "profile: aks\naks: {nodePool: 'agents pool'}\n", wantErr: "invalid node pool"},
 		{name: "acr with space", in: "profile: aks\naks: {acr: 'r.azurecr.io '}\n", wantErr: "invalid ACR"},
 		{name: "unknown field", in: "postgress: external\n", wantErr: "postgress"},
@@ -192,21 +194,28 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	registry := func(r string) map[string]any {
 		return map[string]any{"global": map[string]any{"imageRegistry": r}}
 	}
+	otel := func(endpoint string) map[string]any {
+		return map[string]any{"endpoint": endpoint}
+	}
+	substrateValues := registry("mirror.example.com")
+	substrateValues["otel"] = otel("http://other:4317")
 	c := Config{
-		Profile: ProfileAKS,
-		AKS:     AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true},
+		Profile:      ProfileAKS,
+		AKS:          AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true},
+		OTelEndpoint: "http://collector:4317",
 		Releases: Releases{
 			Atelet:    ReleaseValues{Values: map[string]any{"atelet": map[string]any{"nodeSelector": map[string]any{"disk": "ssd"}}}},
-			Substrate: ReleaseValues{Values: registry("mirror.example.com")},
+			Substrate: ReleaseValues{Values: substrateValues},
 		},
 	}
 	got, err := c.userValues()
 	if err != nil {
 		t.Fatal(err)
 	}
-	// The release's own values win over the profile's, and only atelet gets
-	// the node pool.
+	// The release's own values win over the profile's and otelEndpoint's.
+	// Only atelet gets the node pool, and podcert gets no otel.
 	atelet := registry("r.azurecr.io")
+	atelet["otel"] = otel("http://collector:4317")
 	atelet["atelet"] = map[string]any{
 		"nodeSelector": map[string]any{
 			"kubernetes.azure.com/agentpool": "agents",
@@ -217,10 +226,12 @@ func TestConfigUserValuesProfile(t *testing.T) {
 		"imageCredentialProviderBinDir":     "/var/lib/kubelet/credential-provider",
 		"imageCredentialProviderHostFiles":  []any{"/etc/kubernetes/azure.json"},
 	}
+	substrate := registry("mirror.example.com")
+	substrate["otel"] = otel("http://other:4317")
 	want := map[string]map[string]any{
 		"substrate-podcert": registry("r.azurecr.io"),
 		"substrate-atelet":  atelet,
-		"substrate":         registry("mirror.example.com"),
+		"substrate":         substrate,
 	}
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("userValues() mismatch (-want +got):\n%s", diff)
diff --git a/pkg/kagent/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go
index c7a80ba18e..f58ac2f386 100644
--- a/pkg/kagent/install/tui/tui_test.go
+++ b/pkg/kagent/install/tui/tui_test.go
@@ -81,6 +81,7 @@ func TestHint(t *testing.T) {
 		func() install.Config {
 			c := install.DefaultConfig()
 			c.Profile, c.AKS.ACR = install.ProfileAKS, "r.azurecr.io"
+			c.OTelEndpoint = "http://collector:4317"
 			return c
 		}(),
 	} {
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index a80f187863..ea4b006a38 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -125,6 +125,13 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 				Value(&cfg.MicroVM.AssetBucket).
 				Validate(notEmpty),
 		).WithHideFunc(func() bool { return !external() || !cfg.MicroVM.Enabled }),
+		huh.NewGroup(
+			huh.NewInput().
+				Title("OTLP endpoint").
+				Description("An OTLP collector you already run, for traces, metrics and logs, like\nhttp://otel-collector.observability:4317. Install doesn't deploy one. Leave empty for none.").
+				Value(&cfg.OTelEndpoint).
+				Validate(install.ValidateOTelEndpoint),
+		),
 		huh.NewGroup(
 			huh.NewConfirm().
 				Title("Install Agent Substrate?").

From a984efac8975fcb28d035feb6746ac83a889fc72 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 14:32:17 -0500
Subject: [PATCH 07/10] kubectl-ate: add an AKS managed Prometheus option

aks.managedPrometheus has Azure Managed Prometheus scrape atelet and the
control plane. It enables a new podMonitors chart value with the
azmonitoring.coreos.com/v1 API. The charts then render PodMonitors for the
/metrics endpoints of atelet, ate-api-server, ate-controller,
atenet-router and atenet-egress.

Preflight fails if the cluster doesn't serve that API.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 charts/substrate-atelet/README.md             |   2 +-
 charts/substrate-atelet/templates/atelet.yaml |  15 +++
 charts/substrate-atelet/values.yaml           |   7 ++
 charts/substrate/README.md                    |   2 +
 charts/substrate/templates/atelet.yaml        |  15 +++
 charts/substrate/templates/podmonitors.yaml   |  39 +++++++
 charts/substrate/tests/podmonitors_test.yaml  | 106 ++++++++++++++++++
 charts/substrate/values.yaml                  |   7 ++
 cmd/atelet/helm_test.go                       |   2 +
 cmd/kubectl-ate/internal/cmd/install.go       |   5 +
 pkg/kagent/install/aks.go                     |  27 +++++
 pkg/kagent/install/aks_test.go                |  26 +++++
 pkg/kagent/install/config.go                  |   6 +
 pkg/kagent/install/config_test.go             |   9 +-
 pkg/kagent/install/tui/tui_test.go            |   1 +
 pkg/kagent/install/tui/wizard.go              |   4 +
 16 files changed, 270 insertions(+), 3 deletions(-)
 create mode 100644 charts/substrate/templates/podmonitors.yaml
 create mode 100644 charts/substrate/tests/podmonitors_test.yaml

diff --git a/charts/substrate-atelet/README.md b/charts/substrate-atelet/README.md
index 4f756e6b0b..4080a84eb2 100644
--- a/charts/substrate-atelet/README.md
+++ b/charts/substrate-atelet/README.md
@@ -41,7 +41,7 @@ Use the credentials from the substrate release's `rustfs.accessKey` and
 
 This chart reads the same value paths as the substrate chart for `atelet`,
 `objectStorePlugin.node`, `objectStorePlugin.extraEnv`, `preview`, `otel`,
-`image`, `imagePullSecrets` and `global`, so the substrate release's values
+`podMonitors`, `image`, `imagePullSecrets` and `global`, so the substrate release's values
 file can configure both releases.
 
 Object storage is set under `objectStore` instead of `rustfs` and
diff --git a/charts/substrate-atelet/templates/atelet.yaml b/charts/substrate-atelet/templates/atelet.yaml
index d172eb9e75..51d89e0419 100644
--- a/charts/substrate-atelet/templates/atelet.yaml
+++ b/charts/substrate-atelet/templates/atelet.yaml
@@ -382,3 +382,18 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+{{- if .Values.podMonitors.enabled }}
+---
+apiVersion: {{ .Values.podMonitors.apiVersion }}
+kind: PodMonitor
+metadata:
+  name: {{ include "substrate.fullname" (list "atelet" .) }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: atelet
+  podMetricsEndpoints:
+  - port: prometheus
+    path: /metrics
+{{- end }}
diff --git a/charts/substrate-atelet/values.yaml b/charts/substrate-atelet/values.yaml
index 01ac4eb01a..7959038f28 100644
--- a/charts/substrate-atelet/values.yaml
+++ b/charts/substrate-atelet/values.yaml
@@ -97,6 +97,13 @@ otel:
     enabled: true
     endpoint: ""
 
+# A PodMonitor that scrapes atelet's /metrics endpoint. It needs the
+# Prometheus Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure
+# Managed Prometheus.
+podMonitors:
+  enabled: false
+  apiVersion: monitoring.coreos.com/v1
+
 # Values under `global` are visible to this chart and to every subchart.
 global:
   # -- Mirror registry that overrides where every image is pulled from. It wins
diff --git a/charts/substrate/README.md b/charts/substrate/README.md
index 30310d597f..449588923f 100644
--- a/charts/substrate/README.md
+++ b/charts/substrate/README.md
@@ -114,6 +114,8 @@ See `values.yaml` for the full set; the important keys:
 | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` |
 | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateworkers, plus the router access log. Actor events go to stdout when OTLP logs are disabled |
 | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` |
+| `podMonitors.enabled` | `false` | Render PodMonitors that scrape `/metrics` on atelet, ate-api-server, ate-controller, atenet-router and atenet-egress; ate-controller's only while `otel.metrics.enabled`, since it serves no `/metrics` otherwise |
+| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus |
 
 ## Router timeouts
 
diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml
index 8c492a777b..3a7cb8cf41 100644
--- a/charts/substrate/templates/atelet.yaml
+++ b/charts/substrate/templates/atelet.yaml
@@ -383,4 +383,19 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+{{- if .Values.podMonitors.enabled }}
+---
+apiVersion: {{ .Values.podMonitors.apiVersion }}
+kind: PodMonitor
+metadata:
+  name: {{ include "substrate.fullname" (list "atelet" .) }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: atelet
+  podMetricsEndpoints:
+  - port: prometheus
+    path: /metrics
+{{- end }}
 {{- end }}
diff --git a/charts/substrate/templates/podmonitors.yaml b/charts/substrate/templates/podmonitors.yaml
new file mode 100644
index 0000000000..302fbc7e84
--- /dev/null
+++ b/charts/substrate/templates/podmonitors.yaml
@@ -0,0 +1,39 @@
+{{/*
+Copyright 2026 The Agent Substrate Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- /* atelet's PodMonitor is in atelet.yaml, so substrate-atelet renders it too. */}}
+{{- if .Values.podMonitors.enabled }}
+{{- $targets := list (list "ate-api-server" "prometheus") (list "atenet-router" "gw-metrics") (list "atenet-egress" "stats") }}
+{{- /* ate-controller closes its metrics port when OTEL_METRICS_EXPORTER is none. */}}
+{{- if .Values.otel.metrics.enabled }}
+{{- $targets = append $targets (list "ate-controller" "metrics") }}
+{{- end }}
+{{- range $targets }}
+---
+apiVersion: {{ $.Values.podMonitors.apiVersion }}
+kind: PodMonitor
+metadata:
+  name: {{ include "substrate.fullname" (list (index . 0) $) }}
+  namespace: {{ $.Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ index . 0 }}
+  podMetricsEndpoints:
+  - port: {{ index . 1 }}
+    path: /metrics
+{{- end }}
+{{- end }}
diff --git a/charts/substrate/tests/podmonitors_test.yaml b/charts/substrate/tests/podmonitors_test.yaml
new file mode 100644
index 0000000000..e6caf009f1
--- /dev/null
+++ b/charts/substrate/tests/podmonitors_test.yaml
@@ -0,0 +1,106 @@
+# Copyright 2026 The Agent Substrate Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+suite: PodMonitors
+templates:
+- podmonitors.yaml
+- atelet.yaml
+release:
+  name: custom
+  namespace: custom-system
+tests:
+- it: renders no PodMonitors by default
+  template: podmonitors.yaml
+  asserts:
+  - hasDocuments:
+      count: 0
+- it: scrapes each control plane component's metrics port
+  template: podmonitors.yaml
+  set:
+    podMonitors:
+      enabled: true
+      apiVersion: azmonitoring.coreos.com/v1
+  asserts:
+  - hasDocuments:
+      count: 4
+  - isAPIVersion:
+      of: azmonitoring.coreos.com/v1
+  - isKind:
+      of: PodMonitor
+  - equal:
+      path: metadata.namespace
+      value: custom-system
+  - equal:
+      path: metadata.name
+      value: custom-ate-api-server
+    documentIndex: 0
+  - equal:
+      path: spec
+      value:
+        selector:
+          matchLabels:
+            app: ate-api-server
+        podMetricsEndpoints:
+        - port: prometheus
+          path: /metrics
+    documentIndex: 0
+  - equal:
+      path: spec.podMetricsEndpoints[0].port
+      value: gw-metrics
+    documentIndex: 1
+  - equal:
+      path: spec.podMetricsEndpoints[0].port
+      value: stats
+    documentIndex: 2
+  - equal:
+      path: spec.selector.matchLabels.app
+      value: ate-controller
+    documentIndex: 3
+  - equal:
+      path: spec.podMetricsEndpoints[0].port
+      value: metrics
+    documentIndex: 3
+- it: skips ate-controller when it serves no metrics
+  template: podmonitors.yaml
+  set:
+    podMonitors.enabled: true
+    otel.metrics.enabled: false
+  asserts:
+  - hasDocuments:
+      count: 3
+  - notEqual:
+      path: spec.selector.matchLabels.app
+      value: ate-controller
+- it: scrapes atelet's metrics port
+  template: atelet.yaml
+  set:
+    podMonitors.enabled: true
+  documentSelector:
+    path: kind
+    value: PodMonitor
+  asserts:
+  - isAPIVersion:
+      of: monitoring.coreos.com/v1
+  - equal:
+      path: metadata.name
+      value: custom-atelet
+  - equal:
+      path: spec
+      value:
+        selector:
+          matchLabels:
+            app: atelet
+        podMetricsEndpoints:
+        - port: prometheus
+          path: /metrics
diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml
index 30037c3ef3..8725c39f01 100644
--- a/charts/substrate/values.yaml
+++ b/charts/substrate/values.yaml
@@ -234,6 +234,13 @@ otel:
     enabled: true
     endpoint: ""
 
+# PodMonitors that scrape the /metrics endpoints of atelet and the control
+# plane. They need the Prometheus Operator CRDs, or apiVersion
+# azmonitoring.coreos.com/v1 for Azure Managed Prometheus.
+podMonitors:
+  enabled: false
+  apiVersion: monitoring.coreos.com/v1
+
 # Values under `global` are visible to this chart and to every subchart. A parent
 # chart or an operator sets one value here instead of one value per chart.
 global:
diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go
index a5ff1c803b..5b3e25b933 100644
--- a/cmd/atelet/helm_test.go
+++ b/cmd/atelet/helm_test.go
@@ -316,6 +316,8 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) {
 				"--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml",
 				"--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider",
 				"--set", "atelet.imageCredentialProviderHostFiles[0]=/etc/kubernetes/azure.json",
+				"--set", "podMonitors.enabled=true",
+				"--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1",
 				"--set", "image.tag=test",
 			},
 			substrate: append([]string{"--set-json", `objectStorePlugin.extraEnv=[
diff --git a/cmd/kubectl-ate/internal/cmd/install.go b/cmd/kubectl-ate/internal/cmd/install.go
index c646b57a20..888c4aa459 100644
--- a/cmd/kubectl-ate/internal/cmd/install.go
+++ b/cmd/kubectl-ate/internal/cmd/install.go
@@ -49,6 +49,7 @@ const configHelp = `--config takes a YAML file, or - for stdin:
     acr: my.azurecr.io     # pull the images from it
     nodePool: agents       # run atelet only in this pool
     acrCredentialProvider: true # pull actor images from ACR
+    managedPrometheus: true # scrape metrics with Azure Managed Prometheus
   postgres:
     type: bundled          # or external
   objectStore:
@@ -130,6 +131,10 @@ file holds the service principal's secret, so atelet holds it too. With
 profile aks and the option off, install and upgrade reset atelet's
 imageCredentialProvider values, unless the config's releases set them.
 
+aks.managedPrometheus creates PodMonitors so Azure Managed Prometheus scrapes
+the metrics of atelet and the control plane. Enable it on the cluster first
+with az aks update --enable-azure-monitor-metrics.
+
 microvm.enabled downloads the Kata and Cloud Hypervisor guest assets for each
 node architecture, caches them under the user cache directory, uploads them to
 the object store, and applies the microvm SandboxConfig. Upgrade restages them.
diff --git a/pkg/kagent/install/aks.go b/pkg/kagent/install/aks.go
index d4641a0279..dfa6e51cb7 100644
--- a/pkg/kagent/install/aks.go
+++ b/pkg/kagent/install/aks.go
@@ -54,6 +54,12 @@ func (in *Installer) checkAKS(ctx context.Context, progress func(string)) (strin
 		}
 		summary += fmt.Sprintf(", node pool %s with %d nodes", pool, n)
 	}
+	if in.opts.AKS.ManagedPrometheus {
+		if err := in.checkManagedPrometheus(); err != nil {
+			return "", err
+		}
+		summary += ", Managed Prometheus"
+	}
 	return summary, nil
 }
 
@@ -73,6 +79,27 @@ func (in *Installer) checkNetworkPolicy(ctx context.Context) (string, error) {
 	return "", errors.New("nothing in the cluster enforces NetworkPolicy; enable it with az aks update --network-policy, or set profile: generic to skip this check")
 }
 
+// managedPrometheusAPI is the PodMonitor API version of Azure Managed
+// Prometheus.
+const managedPrometheusAPI = "azmonitoring.coreos.com/v1"
+
+// checkManagedPrometheus checks that the cluster serves the PodMonitors that
+// aks.managedPrometheus creates.
+func (in *Installer) checkManagedPrometheus() error {
+	res, err := in.kube.Discovery().ServerResourcesForGroupVersion(managedPrometheusAPI)
+	if err != nil && !apierrors.IsNotFound(err) {
+		return fmt.Errorf("while discovering %s: %w", managedPrometheusAPI, err)
+	}
+	if err == nil {
+		for _, r := range res.APIResources {
+			if r.Name == "podmonitors" {
+				return nil
+			}
+		}
+	}
+	return errors.New("aks.managedPrometheus needs Azure Managed Prometheus; enable it with az aks update --enable-azure-monitor-metrics")
+}
+
 // checkNodePool returns how many nodes aks.nodePool has.
 func (in *Installer) checkNodePool(ctx context.Context) (int, error) {
 	pool := in.opts.AKS.NodePool
diff --git a/pkg/kagent/install/aks_test.go b/pkg/kagent/install/aks_test.go
index aff5c3149a..0b5b2feb53 100644
--- a/pkg/kagent/install/aks_test.go
+++ b/pkg/kagent/install/aks_test.go
@@ -65,6 +65,32 @@ func TestCheckNetworkPolicy(t *testing.T) {
 	}
 }
 
+func TestCheckManagedPrometheus(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		resources []*metav1.APIResourceList
+		wantErr   bool
+	}{
+		{name: "not enabled", wantErr: true},
+		{name: "no podmonitors", resources: []*metav1.APIResourceList{{GroupVersion: managedPrometheusAPI, APIResources: []metav1.APIResource{{Name: "servicemonitors"}}}}, wantErr: true},
+		{name: "upstream podmonitors", resources: []*metav1.APIResourceList{{GroupVersion: "monitoring.coreos.com/v1", APIResources: []metav1.APIResource{{Name: "podmonitors"}}}}, wantErr: true},
+		{name: "enabled", resources: []*metav1.APIResourceList{{GroupVersion: managedPrometheusAPI, APIResources: []metav1.APIResource{{Name: "podmonitors"}}}}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			kc := fake.NewClientset()
+			kc.Resources = tc.resources
+			err := (&Installer{kube: kc}).checkManagedPrometheus()
+			if tc.wantErr {
+				if err == nil || !strings.Contains(err.Error(), "--enable-azure-monitor-metrics") {
+					t.Errorf("checkManagedPrometheus() = %v, want an error naming --enable-azure-monitor-metrics", err)
+				}
+			} else if err != nil {
+				t.Errorf("checkManagedPrometheus() = %v", err)
+			}
+		})
+	}
+}
+
 func node(name, pool string, kvm bool) runtime.Object {
 	n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{aksNodePoolLabel: pool}}}
 	if kvm {
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index b769e2bef9..8ec4c7306f 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -205,6 +205,9 @@ type AKS struct {
 	// /etc/kubernetes/azure.json and holds the kubelet identity's registry
 	// credentials, or a service principal's secret on clusters that use one.
 	ACRCredentialProvider bool `json:"acrCredentialProvider,omitempty"`
+	// ManagedPrometheus has Azure Managed Prometheus scrape atelet and the
+	// control plane, through PodMonitors.
+	ManagedPrometheus bool `json:"managedPrometheus,omitempty"`
 }
 
 // MicroVM configures the microvm sandbox class.
@@ -398,6 +401,9 @@ func (c Config) profileValues(r release) map[string]any {
 	if reg := c.imageRegistry(); reg != "" {
 		vals["global"] = map[string]any{"imageRegistry": reg}
 	}
+	if c.Profile == ProfileAKS && c.AKS.ManagedPrometheus && (r == ateletRelease || r == substrateRelease) {
+		vals["podMonitors"] = map[string]any{"enabled": true, "apiVersion": managedPrometheusAPI}
+	}
 	if r == ateletRelease {
 		atelet := map[string]any{}
 		if sel := c.nodeSelector(); sel != nil {
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index a0e7206d37..11afd20124 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -96,6 +96,7 @@ releases:
 		{name: "acr without domain", in: "profile: aks\naks: {acr: myregistry}\n", wantErr: "invalid ACR"},
 		{name: "node pool", in: "profile: aks\naks: {nodePool: agents}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.NodePool = ProfileAKS, "agents" })},
 		{name: "acr credential provider", in: "profile: aks\naks: {acrCredentialProvider: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ACRCredentialProvider = ProfileAKS, true })},
+		{name: "managed prometheus", in: "profile: aks\naks: {managedPrometheus: true}\n", want: withDefaults(func(c *Config) { c.Profile, c.AKS.ManagedPrometheus = ProfileAKS, true })},
 		{name: "otel endpoint", in: "otelEndpoint: http://collector:4317\n", want: withDefaults(func(c *Config) { c.OTelEndpoint = "http://collector:4317" })},
 		{name: "otel endpoint without scheme", in: "otelEndpoint: collector:4317\n", wantErr: "invalid OTLP endpoint"},
 		{name: "bad node pool", in: "profile: aks\naks: {nodePool: 'agents pool'}\n", wantErr: "invalid node pool"},
@@ -201,7 +202,7 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	substrateValues["otel"] = otel("http://other:4317")
 	c := Config{
 		Profile:      ProfileAKS,
-		AKS:          AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true},
+		AKS:          AKS{ACR: "r.azurecr.io", NodePool: "agents", ACRCredentialProvider: true, ManagedPrometheus: true},
 		OTelEndpoint: "http://collector:4317",
 		Releases: Releases{
 			Atelet:    ReleaseValues{Values: map[string]any{"atelet": map[string]any{"nodeSelector": map[string]any{"disk": "ssd"}}}},
@@ -213,9 +214,12 @@ func TestConfigUserValuesProfile(t *testing.T) {
 		t.Fatal(err)
 	}
 	// The release's own values win over the profile's and otelEndpoint's.
-	// Only atelet gets the node pool, and podcert gets no otel.
+	// Only atelet gets the node pool, and podcert gets no otel or
+	// PodMonitors.
+	podMonitors := map[string]any{"enabled": true, "apiVersion": "azmonitoring.coreos.com/v1"}
 	atelet := registry("r.azurecr.io")
 	atelet["otel"] = otel("http://collector:4317")
+	atelet["podMonitors"] = podMonitors
 	atelet["atelet"] = map[string]any{
 		"nodeSelector": map[string]any{
 			"kubernetes.azure.com/agentpool": "agents",
@@ -228,6 +232,7 @@ func TestConfigUserValuesProfile(t *testing.T) {
 	}
 	substrate := registry("mirror.example.com")
 	substrate["otel"] = otel("http://other:4317")
+	substrate["podMonitors"] = podMonitors
 	want := map[string]map[string]any{
 		"substrate-podcert": registry("r.azurecr.io"),
 		"substrate-atelet":  atelet,
diff --git a/pkg/kagent/install/tui/tui_test.go b/pkg/kagent/install/tui/tui_test.go
index f58ac2f386..cca4261a99 100644
--- a/pkg/kagent/install/tui/tui_test.go
+++ b/pkg/kagent/install/tui/tui_test.go
@@ -81,6 +81,7 @@ func TestHint(t *testing.T) {
 		func() install.Config {
 			c := install.DefaultConfig()
 			c.Profile, c.AKS.ACR = install.ProfileAKS, "r.azurecr.io"
+			c.AKS.ManagedPrometheus = true
 			c.OTelEndpoint = "http://collector:4317"
 			return c
 		}(),
diff --git a/pkg/kagent/install/tui/wizard.go b/pkg/kagent/install/tui/wizard.go
index ea4b006a38..303dd530f0 100644
--- a/pkg/kagent/install/tui/wizard.go
+++ b/pkg/kagent/install/tui/wizard.go
@@ -60,6 +60,10 @@ func Wizard(w io.Writer, cfg *install.Config, askProfile bool) error {
 				Title("Pull actor images from ACR?").
 				Description("Runs the node's acr-credential-provider in atelet. atelet reads the node's\n/etc/kubernetes/azure.json and gets every registry the kubelet identity can pull from.\nOn a service principal cluster, that file holds the service principal's secret.").
 				Value(&cfg.AKS.ACRCredentialProvider),
+			huh.NewConfirm().
+				Title("Scrape metrics with Azure Managed Prometheus?").
+				Description("Creates PodMonitors for atelet and the control plane.\nNeeds az aks update --enable-azure-monitor-metrics.").
+				Value(&cfg.AKS.ManagedPrometheus),
 		).WithHideFunc(func() bool { return !aks() }),
 		huh.NewGroup(
 			huh.NewSelect[install.PostgresType]().

From e8a1d5deb047d231de42f1e8fe7fe0f636396a50 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 16:49:12 -0500
Subject: [PATCH 08/10] kubectl-ate: accept the v1 PodCertificateRequest API

Install's cluster check only looked for podcertificaterequests in
certificates.k8s.io/v1beta1. Kubernetes 1.37 clusters like AKS serve it
only at v1, so install failed there, though the certificate controller
already prefers v1. The check now accepts either version.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 pkg/kagent/install/install_test.go | 22 ++++++++++++++++++++++
 pkg/kagent/install/steps.go        | 19 +++++++++++--------
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/pkg/kagent/install/install_test.go b/pkg/kagent/install/install_test.go
index e8dcd9b58c..fef8a0aab9 100644
--- a/pkg/kagent/install/install_test.go
+++ b/pkg/kagent/install/install_test.go
@@ -305,6 +305,28 @@ func TestSubstrateValuesStorageClass(t *testing.T) {
 	}
 }
 
+func TestCheckPodCertificateAPI(t *testing.T) {
+	pcr := []metav1.APIResource{{Name: "podcertificaterequests"}}
+	for _, tc := range []struct {
+		name      string
+		resources []*metav1.APIResourceList
+		wantErr   bool
+	}{
+		{name: "v1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: pcr}}},
+		{name: "v1beta1", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1beta1", APIResources: pcr}}},
+		{name: "v1 without podcertificaterequests", resources: []*metav1.APIResourceList{{GroupVersion: "certificates.k8s.io/v1", APIResources: []metav1.APIResource{{Name: "clustertrustbundles"}}}}, wantErr: true},
+		{name: "neither", wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			kc := fake.NewClientset()
+			kc.Resources = tc.resources
+			if err := checkPodCertificateAPI(&Installer{kube: kc}); (err != nil) != tc.wantErr {
+				t.Errorf("checkPodCertificateAPI() = %v, wantErr %v", err, tc.wantErr)
+			}
+		})
+	}
+}
+
 func TestCheckExternalObjectStore(t *testing.T) {
 	s3 := ObjectStore{Type: ObjectStoreS3, Bucket: "b"}
 	gcs := ObjectStore{Type: ObjectStoreGCS, Bucket: "b", GCS: GCSObjectStore{TransferServiceAccount: "sa@p.iam.gserviceaccount.com"}}
diff --git a/pkg/kagent/install/steps.go b/pkg/kagent/install/steps.go
index 9b8baf61f2..82a86ed18f 100644
--- a/pkg/kagent/install/steps.go
+++ b/pkg/kagent/install/steps.go
@@ -16,6 +16,7 @@ package install
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -76,21 +77,23 @@ func (in *Installer) preflight(ctx context.Context, progress func(string)) (stri
 }
 
 // checkPodCertificateAPI checks that the cluster serves PodCertificateRequest,
-// which the certificate controller signs.
+// which the certificate controller signs, at v1 or v1beta1.
 func checkPodCertificateAPI(in *Installer) error {
-	const gv = "certificates.k8s.io/v1beta1"
-	res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv)
-	if err == nil {
+	for _, gv := range []string{"certificates.k8s.io/v1", "certificates.k8s.io/v1beta1"} {
+		res, err := in.kube.Discovery().ServerResourcesForGroupVersion(gv)
+		if apierrors.IsNotFound(err) {
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("while discovering %s: %w", gv, err)
+		}
 		for _, r := range res.APIResources {
 			if r.Name == "podcertificaterequests" {
 				return nil
 			}
 		}
 	}
-	if err != nil && !apierrors.IsNotFound(err) {
-		return fmt.Errorf("while discovering %s: %w", gv, err)
-	}
-	return fmt.Errorf("cluster doesn't serve %s podcertificaterequests; enable the PodCertificateRequest feature gate and API", gv)
+	return errors.New("cluster serves neither certificates.k8s.io/v1 nor v1beta1 podcertificaterequests; enable the PodCertificateRequest feature gate and API")
 }
 
 // valuesFunc returns installer defaults, which user values override, and

From 1744d3ca6171819d018cfc99777b95532eda57c8 Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 20:40:06 -0500
Subject: [PATCH 09/10] charts: replace imageCredentialProviderHostFiles with
 extraVolumes

atelet.extraVolumes and atelet.extraVolumeMounts add any volume to the
atelet pod, so the charts don't need a value just for credential
provider files. With aks.acrCredentialProvider on, install now mounts
/etc/kubernetes/azure.json through them, after any the config sets.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 charts/substrate-atelet/templates/atelet.yaml | 18 ++----
 charts/substrate-atelet/values.yaml           |  7 +-
 charts/substrate/README.md                    |  3 +-
 charts/substrate/templates/atelet.yaml        | 18 ++----
 charts/substrate/values.yaml                  |  7 +-
 cmd/atelet/helm_test.go                       | 64 ++++++++++---------
 pkg/kagent/install/config.go                  | 35 +++++++++-
 pkg/kagent/install/config_test.go             | 34 +++++++---
 8 files changed, 110 insertions(+), 76 deletions(-)

diff --git a/charts/substrate-atelet/templates/atelet.yaml b/charts/substrate-atelet/templates/atelet.yaml
index 51d89e0419..0d8f89aa22 100644
--- a/charts/substrate-atelet/templates/atelet.yaml
+++ b/charts/substrate-atelet/templates/atelet.yaml
@@ -21,9 +21,6 @@ limitations under the License.
 {{- if ne (empty $credentialProviderConfig) (empty .Values.atelet.imageCredentialProviderBinDir) }}
 {{- fail "atelet.imageCredentialProviderConfig (or atelet.imageCredentialProviderConfigYAML) and atelet.imageCredentialProviderBinDir must be set together" }}
 {{- end }}
-{{- if and .Values.atelet.imageCredentialProviderHostFiles (not $credentialProviderConfig) }}
-{{- fail "atelet.imageCredentialProviderHostFiles needs atelet.imageCredentialProviderConfig or atelet.imageCredentialProviderConfigYAML" }}
-{{- end }}
 
 # atelet
 apiVersion: v1
@@ -311,11 +308,9 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
-{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
-        - name: image-credential-provider-file-{{ $i }}
-          mountPath: {{ $path | quote }}
-          readOnly: true
 {{- end }}
+{{- with .Values.atelet.extraVolumeMounts }}
+{{- toYaml . | nindent 8 }}
 {{- end }}
       volumes:
       - name: objectstore-plugin
@@ -353,12 +348,6 @@ spec:
           path: {{ .Values.atelet.imageCredentialProviderConfig | quote }}
           type: File
 {{- end }}
-{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
-      - name: image-credential-provider-file-{{ $i }}
-        hostPath:
-          path: {{ $path | quote }}
-          type: File
-{{- end }}
 {{- end }}
       - name: podidentity
         projected:
@@ -382,6 +371,9 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+{{- with .Values.atelet.extraVolumes }}
+{{- toYaml . | nindent 6 }}
+{{- end }}
 {{- if .Values.podMonitors.enabled }}
 ---
 apiVersion: {{ .Values.podMonitors.apiVersion }}
diff --git a/charts/substrate-atelet/values.yaml b/charts/substrate-atelet/values.yaml
index 7959038f28..684cee37c6 100644
--- a/charts/substrate-atelet/values.yaml
+++ b/charts/substrate-atelet/values.yaml
@@ -56,14 +56,15 @@ atelet:
   # either one still needs the bin directory.
   imageCredentialProviderConfigYAML: ""
   imageCredentialProviderBinDir: ""
-  # Other host files the providers read, mounted read-only at the same path
-  # (AKS: /etc/kubernetes/azure.json).
-  imageCredentialProviderHostFiles: []
   # Restricts atelet to matching nodes. Every node that runs WorkerPool pods
   # needs an atelet, so cover each pool's nodeSelector here.
   nodeSelector: {}
   extraArgs: []
   extraEnv: []
+  # Added to the atelet pod's volumes and its container's volumeMounts, such
+  # as a host file a credential provider reads.
+  extraVolumes: []
+  extraVolumeMounts: []
   resources:
     requests:
       cpu: 50m
diff --git a/charts/substrate/README.md b/charts/substrate/README.md
index 449588923f..9975f61b2d 100644
--- a/charts/substrate/README.md
+++ b/charts/substrate/README.md
@@ -97,7 +97,8 @@ See `values.yaml` for the full set; the important keys:
 | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path |
 | `atelet.nodeSelector` | `{}` | Node labels atelet pods must match; must cover the nodes of every WorkerPool |
 | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only |
-| `atelet.imageCredentialProviderHostFiles` | `[]` | Other host files the providers read, mounted read-only at the same path; needs a provider config |
+| `atelet.extraVolumes` | `[]` | Volumes added to the atelet pod, such as a host file a credential provider reads |
+| `atelet.extraVolumeMounts` | `[]` | Volume mounts added to the atelet container |
 | `credentialProvider.enabled` | `true` | Deploy the Kubernetes Secret provider and route `k8s.io` credential URIs to it. With no provider, every credential injection fails closed |
 | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider |
 | `ateApi.actorJWTIssuer` | `https://<idp-service>.<namespace>.svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties |
diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml
index 3a7cb8cf41..ca0bc9d140 100644
--- a/charts/substrate/templates/atelet.yaml
+++ b/charts/substrate/templates/atelet.yaml
@@ -22,9 +22,6 @@ limitations under the License.
 {{- if ne (empty $credentialProviderConfig) (empty .Values.atelet.imageCredentialProviderBinDir) }}
 {{- fail "atelet.imageCredentialProviderConfig (or atelet.imageCredentialProviderConfigYAML) and atelet.imageCredentialProviderBinDir must be set together" }}
 {{- end }}
-{{- if and .Values.atelet.imageCredentialProviderHostFiles (not $credentialProviderConfig) }}
-{{- fail "atelet.imageCredentialProviderHostFiles needs atelet.imageCredentialProviderConfig or atelet.imageCredentialProviderConfigYAML" }}
-{{- end }}
 
 # atelet
 apiVersion: v1
@@ -312,11 +309,9 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
-{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
-        - name: image-credential-provider-file-{{ $i }}
-          mountPath: {{ $path | quote }}
-          readOnly: true
 {{- end }}
+{{- with .Values.atelet.extraVolumeMounts }}
+{{- toYaml . | nindent 8 }}
 {{- end }}
       volumes:
       - name: objectstore-plugin
@@ -354,12 +349,6 @@ spec:
           path: {{ .Values.atelet.imageCredentialProviderConfig | quote }}
           type: File
 {{- end }}
-{{- range $i, $path := .Values.atelet.imageCredentialProviderHostFiles }}
-      - name: image-credential-provider-file-{{ $i }}
-        hostPath:
-          path: {{ $path | quote }}
-          type: File
-{{- end }}
 {{- end }}
       - name: podidentity
         projected:
@@ -383,6 +372,9 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
+{{- with .Values.atelet.extraVolumes }}
+{{- toYaml . | nindent 6 }}
+{{- end }}
 {{- if .Values.podMonitors.enabled }}
 ---
 apiVersion: {{ .Values.podMonitors.apiVersion }}
diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml
index 8725c39f01..f6f57c7448 100644
--- a/charts/substrate/values.yaml
+++ b/charts/substrate/values.yaml
@@ -90,15 +90,16 @@ atelet:
   # either one still needs the bin directory.
   imageCredentialProviderConfigYAML: ""
   imageCredentialProviderBinDir: ""
-  # Other host files the providers read, mounted read-only at the same path
-  # (AKS: /etc/kubernetes/azure.json).
-  imageCredentialProviderHostFiles: []
   storageBackend: s3
   # Restricts atelet to matching nodes. Every node that runs WorkerPool pods
   # needs an atelet, so cover each pool's nodeSelector here.
   nodeSelector: {}
   extraArgs: []
   extraEnv: []
+  # Added to the atelet pod's volumes and its container's volumeMounts, such
+  # as a host file a credential provider reads.
+  extraVolumes: []
+  extraVolumeMounts: []
   resources:
     requests:
       cpu: 50m
diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go
index 5b3e25b933..53d9fbc00d 100644
--- a/cmd/atelet/helm_test.go
+++ b/cmd/atelet/helm_test.go
@@ -16,7 +16,6 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"io"
 	"os"
@@ -31,7 +30,9 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/yaml"
 )
 
@@ -44,18 +45,10 @@ kind: CredentialProviderConfig
 providers: []`
 	for _, tc := range []struct {
 		name, config, configYAML, binDir string
-		hostFiles                        []string
 		wantError                        string
 	}{
 		{name: "public registries"},
 		{name: "node provider", config: "/etc/srv/kubernetes/cri_auth_config.yaml", binDir: "/home/kubernetes/bin"},
-		{
-			name:      "node provider with host files",
-			config:    "/var/lib/kubelet/credential-provider-config.yaml",
-			binDir:    "/var/lib/kubelet/credential-provider",
-			hostFiles: []string{"/etc/kubernetes/azure.json", "/etc/provider/ca.pem"},
-		},
-		{name: "host files without config", hostFiles: []string{"/etc/kubernetes/azure.json"}, wantError: "imageCredentialProviderHostFiles needs"},
 		{name: "chart provided config", configYAML: providerConfig, binDir: "/home/kubernetes/bin"},
 		{name: "missing bin directory", config: "/etc/provider.yaml", wantError: "must be set together"},
 		{name: "chart provided config without bin directory", configYAML: providerConfig, wantError: "must be set together"},
@@ -69,16 +62,11 @@ providers: []`
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			hostFiles, err := json.Marshal(append([]string{}, tc.hostFiles...))
-			if err != nil {
-				t.Fatal(err)
-			}
 			out, err := exec.CommandContext(t.Context(), "helm", "template", "substrate", "../../charts/substrate",
 				"--show-only", "templates/atelet.yaml",
 				"--set-string", "atelet.imageCredentialProviderConfig="+tc.config,
 				"--set-string", "atelet.imageCredentialProviderConfigYAML="+tc.configYAML,
-				"--set-string", "atelet.imageCredentialProviderBinDir="+tc.binDir,
-				"--set-json", "atelet.imageCredentialProviderHostFiles="+string(hostFiles)).CombinedOutput()
+				"--set-string", "atelet.imageCredentialProviderBinDir="+tc.binDir).CombinedOutput()
 			if tc.wantError != "" {
 				if err == nil || !strings.Contains(string(out), tc.wantError) {
 					t.Fatalf("render rejected provider config with %q: %v\n%s", tc.wantError, err, out)
@@ -155,24 +143,39 @@ providers: []`
 					}
 				}
 			}
-			var gotFiles []string
-			for _, m := range container.VolumeMounts {
-				if !strings.HasPrefix(m.Name, "image-credential-provider-file-") {
-					continue
-				}
-				i := slices.IndexFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == m.Name })
-				if i < 0 || pod.Volumes[i].HostPath == nil || pod.Volumes[i].HostPath.Path != m.MountPath || !m.ReadOnly {
-					t.Errorf("%s must mount host file %s read-only at the same path", m.Name, m.MountPath)
-				}
-				gotFiles = append(gotFiles, m.MountPath)
-			}
-			if !slices.Equal(gotFiles, tc.hostFiles) {
-				t.Errorf("atelet mounts host files %v, want %v", gotFiles, tc.hostFiles)
-			}
 		})
 	}
 }
 
+func TestHelmAteletExtraVolumes(t *testing.T) {
+	if _, err := exec.LookPath("helm"); err != nil {
+		t.Skip("helm is not installed")
+	}
+	objs := helmTemplate(t, "substrate", "../../charts/substrate", "--show-only", "templates/atelet.yaml",
+		"--set-json", `atelet.extraVolumes=[{"name": "extra", "hostPath": {"path": "/etc/extra.json", "type": "File"}}]`,
+		"--set-json", `atelet.extraVolumeMounts=[{"name": "extra", "mountPath": "/etc/extra.json", "readOnly": true}]`)
+	for _, obj := range objs {
+		if obj.GetKind() != "DaemonSet" {
+			continue
+		}
+		var ds appsv1.DaemonSet
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &ds); err != nil {
+			t.Fatal(err)
+		}
+		pod := ds.Spec.Template.Spec
+		wantVolume := corev1.Volume{Name: "extra", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/etc/extra.json", Type: ptr.To(corev1.HostPathFile)}}}
+		if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return reflect.DeepEqual(v, wantVolume) }) {
+			t.Errorf("atelet volumes %v don't include %v", pod.Volumes, wantVolume)
+		}
+		wantMount := corev1.VolumeMount{Name: "extra", MountPath: "/etc/extra.json", ReadOnly: true}
+		if !slices.ContainsFunc(pod.Containers[0].VolumeMounts, func(m corev1.VolumeMount) bool { return reflect.DeepEqual(m, wantMount) }) {
+			t.Errorf("atelet volume mounts %v don't include %v", pod.Containers[0].VolumeMounts, wantMount)
+		}
+		return
+	}
+	t.Fatal("no atelet DaemonSet rendered")
+}
+
 func TestHelmSandboxAssetStorage(t *testing.T) {
 	for _, tc := range []struct {
 		name, backend string
@@ -315,7 +318,8 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) {
 				"--set", "atelet.nodeSelector.pool=agents",
 				"--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml",
 				"--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider",
-				"--set", "atelet.imageCredentialProviderHostFiles[0]=/etc/kubernetes/azure.json",
+				"--set-json", `atelet.extraVolumes=[{"name": "azure-json", "hostPath": {"path": "/etc/kubernetes/azure.json", "type": "File"}}]`,
+				"--set-json", `atelet.extraVolumeMounts=[{"name": "azure-json", "mountPath": "/etc/kubernetes/azure.json", "readOnly": true}]`,
 				"--set", "podMonitors.enabled=true",
 				"--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1",
 				"--set", "image.tag=test",
diff --git a/pkg/kagent/install/config.go b/pkg/kagent/install/config.go
index 8ec4c7306f..04f623f828 100644
--- a/pkg/kagent/install/config.go
+++ b/pkg/kagent/install/config.go
@@ -384,6 +384,7 @@ const (
 	aksCredentialProviderConfig = "/var/lib/kubelet/credential-provider-config.yaml"
 	aksCredentialProviderBinDir = "/var/lib/kubelet/credential-provider"
 	aksCloudConfig              = "/etc/kubernetes/azure.json"
+	aksCloudConfigVolume        = "aks-cloud-config"
 )
 
 // nodeSelector returns the node selector atelet runs with, or nil for every
@@ -416,14 +417,16 @@ func (c Config) profileValues(r release) map[string]any {
 		if c.Profile == ProfileAKS {
 			// Off resets the chart defaults, so atelet loses the kubelet
 			// identity's registry access that an earlier install gave it.
-			config, binDir, hostFiles := "", "", []any{}
+			// userValues adds the cloud config mount when it's on.
+			config, binDir := "", ""
 			if c.AKS.ACRCredentialProvider {
-				config, binDir, hostFiles = aksCredentialProviderConfig, aksCredentialProviderBinDir, []any{aksCloudConfig}
+				config, binDir = aksCredentialProviderConfig, aksCredentialProviderBinDir
 			}
 			atelet["imageCredentialProviderConfig"] = config
 			atelet["imageCredentialProviderConfigYAML"] = ""
 			atelet["imageCredentialProviderBinDir"] = binDir
-			atelet["imageCredentialProviderHostFiles"] = hostFiles
+			atelet["extraVolumes"] = []any{}
+			atelet["extraVolumeMounts"] = []any{}
 		}
 		if len(atelet) > 0 {
 			vals["atelet"] = atelet
@@ -464,9 +467,35 @@ func (c Config) userValues() (map[string]map[string]any, error) {
 		}
 		user[r.name] = mergeValues(c.profileValues(r), c.otelValues(r), all, own)
 	}
+	c.addCloudConfigMount(user[ateletRelease.name])
 	return user, nil
 }
 
+// addCloudConfigMount mounts the AKS cloud config, which the ACR credential
+// provider reads, into atelet. It appends to the release's own extraVolumes
+// and extraVolumeMounts, since lists replace in a merge.
+func (c Config) addCloudConfigMount(vals map[string]any) {
+	if c.Profile != ProfileAKS || !c.AKS.ACRCredentialProvider {
+		return
+	}
+	atelet, ok := vals["atelet"].(map[string]any)
+	if !ok {
+		atelet = map[string]any{}
+		vals["atelet"] = atelet
+	}
+	volumes, _ := atelet["extraVolumes"].([]any)
+	mounts, _ := atelet["extraVolumeMounts"].([]any)
+	atelet["extraVolumes"] = append(slices.Clone(volumes), map[string]any{
+		"name":     aksCloudConfigVolume,
+		"hostPath": map[string]any{"path": aksCloudConfig, "type": "File"},
+	})
+	atelet["extraVolumeMounts"] = append(slices.Clone(mounts), map[string]any{
+		"name":      aksCloudConfigVolume,
+		"mountPath": aksCloudConfig,
+		"readOnly":  true,
+	})
+}
+
 func (rv ReleaseValues) read() (map[string]any, error) {
 	vo := values.Options{ValueFiles: rv.ValuesFiles}
 	files, err := vo.MergeValues(nil)
diff --git a/pkg/kagent/install/config_test.go b/pkg/kagent/install/config_test.go
index 11afd20124..db297d5c64 100644
--- a/pkg/kagent/install/config_test.go
+++ b/pkg/kagent/install/config_test.go
@@ -191,6 +191,11 @@ func TestConfigUserValues(t *testing.T) {
 	}
 }
 
+var (
+	cloudConfigVolume = map[string]any{"name": "aks-cloud-config", "hostPath": map[string]any{"path": "/etc/kubernetes/azure.json", "type": "File"}}
+	cloudConfigMount  = map[string]any{"name": "aks-cloud-config", "mountPath": "/etc/kubernetes/azure.json", "readOnly": true}
+)
+
 func TestConfigUserValuesProfile(t *testing.T) {
 	registry := func(r string) map[string]any {
 		return map[string]any{"global": map[string]any{"imageRegistry": r}}
@@ -228,7 +233,8 @@ func TestConfigUserValuesProfile(t *testing.T) {
 		"imageCredentialProviderConfig":     "/var/lib/kubelet/credential-provider-config.yaml",
 		"imageCredentialProviderConfigYAML": "",
 		"imageCredentialProviderBinDir":     "/var/lib/kubelet/credential-provider",
-		"imageCredentialProviderHostFiles":  []any{"/etc/kubernetes/azure.json"},
+		"extraVolumes":                      []any{cloudConfigVolume},
+		"extraVolumeMounts":                 []any{cloudConfigMount},
 	}
 	substrate := registry("mirror.example.com")
 	substrate["otel"] = otel("http://other:4317")
@@ -244,17 +250,18 @@ func TestConfigUserValuesProfile(t *testing.T) {
 }
 
 func TestConfigUserValuesACRCredentialProvider(t *testing.T) {
-	provider := func(config, binDir string, hostFiles ...any) map[string]any {
-		if hostFiles == nil {
-			hostFiles = []any{}
-		}
+	provider := func(config, binDir string, volumes, mounts []any) map[string]any {
 		return map[string]any{
 			"imageCredentialProviderConfig":     config,
 			"imageCredentialProviderConfigYAML": "",
 			"imageCredentialProviderBinDir":     binDir,
-			"imageCredentialProviderHostFiles":  hostFiles,
+			"extraVolumes":                      volumes,
+			"extraVolumeMounts":                 mounts,
 		}
 	}
+	const config, binDir = "/var/lib/kubelet/credential-provider-config.yaml", "/var/lib/kubelet/credential-provider"
+	userVolume := map[string]any{"name": "extra", "emptyDir": map[string]any{}}
+	userMount := map[string]any{"name": "extra", "mountPath": "/extra"}
 	for _, tc := range []struct {
 		name string
 		c    Config
@@ -264,19 +271,26 @@ func TestConfigUserValuesACRCredentialProvider(t *testing.T) {
 		{
 			name: "aks on",
 			c:    Config{Profile: ProfileAKS, AKS: AKS{ACRCredentialProvider: true}},
-			want: provider("/var/lib/kubelet/credential-provider-config.yaml", "/var/lib/kubelet/credential-provider", "/etc/kubernetes/azure.json"),
+			want: provider(config, binDir, []any{cloudConfigVolume}, []any{cloudConfigMount}),
+		},
+		{
+			name: "aks on keeps the release's own volumes",
+			c: Config{Profile: ProfileAKS, AKS: AKS{ACRCredentialProvider: true}, Releases: Releases{Atelet: ReleaseValues{Values: map[string]any{
+				"atelet": map[string]any{"extraVolumes": []any{userVolume}, "extraVolumeMounts": []any{userMount}},
+			}}}},
+			want: provider(config, binDir, []any{userVolume, cloudConfigVolume}, []any{userMount, cloudConfigMount}),
 		},
 		{
 			name: "aks off resets it",
 			c:    Config{Profile: ProfileAKS},
-			want: provider("", ""),
+			want: provider("", "", []any{}, []any{}),
 		},
 		{
 			name: "aks off keeps the release's own",
 			c: Config{Profile: ProfileAKS, Releases: Releases{Atelet: ReleaseValues{Values: map[string]any{
-				"atelet": provider("/etc/provider.yaml", "/opt/provider"),
+				"atelet": provider("/etc/provider.yaml", "/opt/provider", []any{userVolume}, []any{userMount}),
 			}}}},
-			want: provider("/etc/provider.yaml", "/opt/provider"),
+			want: provider("/etc/provider.yaml", "/opt/provider", []any{userVolume}, []any{userMount}),
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

From dbee5d11627b2687abc9d6585d615babfd7b8d8f Mon Sep 17 00:00:00 2001
From: adminturneddevops <mlevan1992@gmail.com>
Date: Sun, 8 Nov 2026 20:45:34 -0500
Subject: [PATCH 10/10] charts: keep new atelet values in substrate-atelet only

The substrate chart's atelet template stays only for one release, for
compatibility. It goes back to upstream, and atelet.extraVolumes,
atelet.extraVolumeMounts and the atelet PodMonitor stay in the
substrate-atelet chart. The installer already installs atelet with that
chart.

Signed-off-by: adminturneddevops <mlevan1992@gmail.com>
---
 charts/substrate-atelet/README.md            | 11 ++++++-
 charts/substrate/README.md                   |  4 +--
 charts/substrate/templates/atelet.yaml       | 21 -------------
 charts/substrate/templates/podmonitors.yaml  |  1 -
 charts/substrate/tests/podmonitors_test.yaml | 23 ---------------
 charts/substrate/values.yaml                 | 11 +++----
 cmd/atelet/helm_test.go                      | 31 ++++++++++++++++----
 7 files changed, 41 insertions(+), 61 deletions(-)

diff --git a/charts/substrate-atelet/README.md b/charts/substrate-atelet/README.md
index 4080a84eb2..60f774f654 100644
--- a/charts/substrate-atelet/README.md
+++ b/charts/substrate-atelet/README.md
@@ -41,9 +41,18 @@ Use the credentials from the substrate release's `rustfs.accessKey` and
 
 This chart reads the same value paths as the substrate chart for `atelet`,
 `objectStorePlugin.node`, `objectStorePlugin.extraEnv`, `preview`, `otel`,
-`podMonitors`, `image`, `imagePullSecrets` and `global`, so the substrate release's values
+`image`, `imagePullSecrets` and `global`, so the substrate release's values
 file can configure both releases.
 
+These values are only in this chart:
+
+| Key | Default | Description |
+|---|---|---|
+| `atelet.extraVolumes` | `[]` | Volumes added to the atelet pod, such as a host file a credential provider reads |
+| `atelet.extraVolumeMounts` | `[]` | Volume mounts added to the atelet container |
+| `podMonitors.enabled` | `false` | Render a PodMonitor that scrapes atelet's `/metrics` |
+| `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus |
+
 Object storage is set under `objectStore` instead of `rustfs` and
 `atelet.storageBackend`:
 
diff --git a/charts/substrate/README.md b/charts/substrate/README.md
index 9975f61b2d..79abefe720 100644
--- a/charts/substrate/README.md
+++ b/charts/substrate/README.md
@@ -97,8 +97,6 @@ See `values.yaml` for the full set; the important keys:
 | `atelet.imageCredentialProviderConfigYAML` | `""` | Credential provider config as text, rendered into a ConfigMap, for nodes where that file cannot be authored or must differ from the kubelet's; mutually exclusive with the host path |
 | `atelet.nodeSelector` | `{}` | Node labels atelet pods must match; must cover the nodes of every WorkerPool |
 | `atelet.imageCredentialProviderBinDir` | `""` | Host path to the kubelet credential provider binaries; both paths are mounted read-only |
-| `atelet.extraVolumes` | `[]` | Volumes added to the atelet pod, such as a host file a credential provider reads |
-| `atelet.extraVolumeMounts` | `[]` | Volume mounts added to the atelet container |
 | `credentialProvider.enabled` | `true` | Deploy the Kubernetes Secret provider and route `k8s.io` credential URIs to it. With no provider, every credential injection fails closed |
 | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider |
 | `ateApi.actorJWTIssuer` | `https://<idp-service>.<namespace>.svc` | Issuer placed in actor JWTs; set a stable URL reachable by relying parties |
@@ -115,7 +113,7 @@ See `values.yaml` for the full set; the important keys:
 | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` |
 | `otel.logs.enabled` | `true` | Enable OTLP actor events from ateapi and the ateworkers, plus the router access log. Actor events go to stdout when OTLP logs are disabled |
 | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` |
-| `podMonitors.enabled` | `false` | Render PodMonitors that scrape `/metrics` on atelet, ate-api-server, ate-controller, atenet-router and atenet-egress; ate-controller's only while `otel.metrics.enabled`, since it serves no `/metrics` otherwise |
+| `podMonitors.enabled` | `false` | Render PodMonitors that scrape `/metrics` on ate-api-server, ate-controller, atenet-router and atenet-egress; ate-controller's only while `otel.metrics.enabled`, since it serves no `/metrics` otherwise |
 | `podMonitors.apiVersion` | `monitoring.coreos.com/v1` | PodMonitor API version; `azmonitoring.coreos.com/v1` for Azure Managed Prometheus |
 
 ## Router timeouts
diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml
index ca0bc9d140..e8c3ecad3b 100644
--- a/charts/substrate/templates/atelet.yaml
+++ b/charts/substrate/templates/atelet.yaml
@@ -309,9 +309,6 @@ spec:
           subPath: config.yaml
 {{- end }}
           readOnly: true
-{{- end }}
-{{- with .Values.atelet.extraVolumeMounts }}
-{{- toYaml . | nindent 8 }}
 {{- end }}
       volumes:
       - name: objectstore-plugin
@@ -372,22 +369,4 @@ spec:
                 matchLabels:
                   podcert.ate.dev/canarying: live
               path: trust-bundle.pem
-{{- with .Values.atelet.extraVolumes }}
-{{- toYaml . | nindent 6 }}
-{{- end }}
-{{- if .Values.podMonitors.enabled }}
----
-apiVersion: {{ .Values.podMonitors.apiVersion }}
-kind: PodMonitor
-metadata:
-  name: {{ include "substrate.fullname" (list "atelet" .) }}
-  namespace: {{ .Release.Namespace }}
-spec:
-  selector:
-    matchLabels:
-      app: atelet
-  podMetricsEndpoints:
-  - port: prometheus
-    path: /metrics
-{{- end }}
 {{- end }}
diff --git a/charts/substrate/templates/podmonitors.yaml b/charts/substrate/templates/podmonitors.yaml
index 302fbc7e84..ebfe5d4c52 100644
--- a/charts/substrate/templates/podmonitors.yaml
+++ b/charts/substrate/templates/podmonitors.yaml
@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */}}
 
-{{- /* atelet's PodMonitor is in atelet.yaml, so substrate-atelet renders it too. */}}
 {{- if .Values.podMonitors.enabled }}
 {{- $targets := list (list "ate-api-server" "prometheus") (list "atenet-router" "gw-metrics") (list "atenet-egress" "stats") }}
 {{- /* ate-controller closes its metrics port when OTEL_METRICS_EXPORTER is none. */}}
diff --git a/charts/substrate/tests/podmonitors_test.yaml b/charts/substrate/tests/podmonitors_test.yaml
index e6caf009f1..b008145705 100644
--- a/charts/substrate/tests/podmonitors_test.yaml
+++ b/charts/substrate/tests/podmonitors_test.yaml
@@ -15,7 +15,6 @@
 suite: PodMonitors
 templates:
 - podmonitors.yaml
-- atelet.yaml
 release:
   name: custom
   namespace: custom-system
@@ -82,25 +81,3 @@ tests:
   - notEqual:
       path: spec.selector.matchLabels.app
       value: ate-controller
-- it: scrapes atelet's metrics port
-  template: atelet.yaml
-  set:
-    podMonitors.enabled: true
-  documentSelector:
-    path: kind
-    value: PodMonitor
-  asserts:
-  - isAPIVersion:
-      of: monitoring.coreos.com/v1
-  - equal:
-      path: metadata.name
-      value: custom-atelet
-  - equal:
-      path: spec
-      value:
-        selector:
-          matchLabels:
-            app: atelet
-        podMetricsEndpoints:
-        - port: prometheus
-          path: /metrics
diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml
index f6f57c7448..e4bb553973 100644
--- a/charts/substrate/values.yaml
+++ b/charts/substrate/values.yaml
@@ -96,10 +96,6 @@ atelet:
   nodeSelector: {}
   extraArgs: []
   extraEnv: []
-  # Added to the atelet pod's volumes and its container's volumeMounts, such
-  # as a host file a credential provider reads.
-  extraVolumes: []
-  extraVolumeMounts: []
   resources:
     requests:
       cpu: 50m
@@ -235,9 +231,10 @@ otel:
     enabled: true
     endpoint: ""
 
-# PodMonitors that scrape the /metrics endpoints of atelet and the control
-# plane. They need the Prometheus Operator CRDs, or apiVersion
-# azmonitoring.coreos.com/v1 for Azure Managed Prometheus.
+# PodMonitors that scrape the control plane's /metrics endpoints. The
+# substrate-atelet chart has its own for atelet. They need the Prometheus
+# Operator CRDs, or apiVersion azmonitoring.coreos.com/v1 for Azure Managed
+# Prometheus.
 podMonitors:
   enabled: false
   apiVersion: monitoring.coreos.com/v1
diff --git a/cmd/atelet/helm_test.go b/cmd/atelet/helm_test.go
index 53d9fbc00d..47b3dd69de 100644
--- a/cmd/atelet/helm_test.go
+++ b/cmd/atelet/helm_test.go
@@ -151,7 +151,7 @@ func TestHelmAteletExtraVolumes(t *testing.T) {
 	if _, err := exec.LookPath("helm"); err != nil {
 		t.Skip("helm is not installed")
 	}
-	objs := helmTemplate(t, "substrate", "../../charts/substrate", "--show-only", "templates/atelet.yaml",
+	objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet",
 		"--set-json", `atelet.extraVolumes=[{"name": "extra", "hostPath": {"path": "/etc/extra.json", "type": "File"}}]`,
 		"--set-json", `atelet.extraVolumeMounts=[{"name": "extra", "mountPath": "/etc/extra.json", "readOnly": true}]`)
 	for _, obj := range objs {
@@ -176,6 +176,31 @@ func TestHelmAteletExtraVolumes(t *testing.T) {
 	t.Fatal("no atelet DaemonSet rendered")
 }
 
+func TestHelmAteletPodMonitor(t *testing.T) {
+	if _, err := exec.LookPath("helm"); err != nil {
+		t.Skip("helm is not installed")
+	}
+	objs := helmTemplate(t, "atelet", "../../charts/substrate-atelet",
+		"--set", "podMonitors.enabled=true", "--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1")
+	for _, obj := range objs {
+		if obj.GetKind() != "PodMonitor" {
+			continue
+		}
+		if got := obj.GetAPIVersion(); got != "azmonitoring.coreos.com/v1" {
+			t.Errorf("PodMonitor apiVersion = %q, want azmonitoring.coreos.com/v1", got)
+		}
+		want := map[string]any{
+			"selector":            map[string]any{"matchLabels": map[string]any{"app": "atelet"}},
+			"podMetricsEndpoints": []any{map[string]any{"port": "prometheus", "path": "/metrics"}},
+		}
+		if diff := cmp.Diff(want, obj.Object["spec"]); diff != "" {
+			t.Errorf("PodMonitor spec mismatch (-want +got):\n%s", diff)
+		}
+		return
+	}
+	t.Fatal("no PodMonitor rendered")
+}
+
 func TestHelmSandboxAssetStorage(t *testing.T) {
 	for _, tc := range []struct {
 		name, backend string
@@ -318,10 +343,6 @@ func TestHelmAteletChartMatchesSubstrateChart(t *testing.T) {
 				"--set", "atelet.nodeSelector.pool=agents",
 				"--set", "atelet.imageCredentialProviderConfig=/var/lib/kubelet/credential-provider-config.yaml",
 				"--set", "atelet.imageCredentialProviderBinDir=/var/lib/kubelet/credential-provider",
-				"--set-json", `atelet.extraVolumes=[{"name": "azure-json", "hostPath": {"path": "/etc/kubernetes/azure.json", "type": "File"}}]`,
-				"--set-json", `atelet.extraVolumeMounts=[{"name": "azure-json", "mountPath": "/etc/kubernetes/azure.json", "readOnly": true}]`,
-				"--set", "podMonitors.enabled=true",
-				"--set", "podMonitors.apiVersion=azmonitoring.coreos.com/v1",
 				"--set", "image.tag=test",
 			},
 			substrate: append([]string{"--set-json", `objectStorePlugin.extraEnv=[