diff --git a/cmd/ate-setup/internal/kube/apply_test.go b/cmd/ate-setup/internal/kube/apply_test.go index ee73adf75f..2e261aa429 100644 --- a/cmd/ate-setup/internal/kube/apply_test.go +++ b/cmd/ate-setup/internal/kube/apply_test.go @@ -27,6 +27,10 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery/cached/memory" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/restmapper" + clienttesting "k8s.io/client-go/testing" ) // Teardown walks a fixed list of manifests covering every install shape, so a @@ -158,3 +162,33 @@ func TestApplyWithRetryStopsOnCancellation(t *testing.T) { t.Errorf("apply attempted %d times, want 1", ri.attempts) } } + +// ApplyTolerant skips a kind whose CRD the cluster does not serve and never +// reaches the dynamic client for it: the zero-value Dynamic panics if it does. +func TestApplyTolerantSkipsUnservedKind(t *testing.T) { + // A cluster serves at least the core group; only the Managed Prometheus + // group is missing. + served := &fakediscovery.FakeDiscovery{Fake: &clienttesting.Fake{Resources: []*metav1.APIResourceList{{ + GroupVersion: "v1", + APIResources: []metav1.APIResource{{Name: "configmaps", Kind: "ConfigMap", Namespaced: true}}, + }}}} + disco := memory.NewMemCacheClient(served) + c := &Client{discovery: disco, mapper: restmapper.NewDeferredDiscoveryRESTMapper(disco)} + + pm := &unstructured.Unstructured{} + pm.SetAPIVersion("monitoring.googleapis.com/v1") + pm.SetKind("PodMonitoring") + pm.SetNamespace("ate-system") + pm.SetName("atenet-router-envoy") + + var skipped []string + err := c.ApplyTolerant(t.Context(), []*unstructured.Unstructured{pm}, func(obj *unstructured.Unstructured, _ error) { + skipped = append(skipped, Describe(obj)) + }) + if err != nil { + t.Fatalf("ApplyTolerant() = %v, want the unserved kind skipped", err) + } + if len(skipped) != 1 { + t.Errorf("skipped %v, want the PodMonitoring reported once", skipped) + } +} diff --git a/cmd/ate-setup/internal/steps/deploy.go b/cmd/ate-setup/internal/steps/deploy.go index 992b6429f8..be1b0c2a8b 100644 --- a/cmd/ate-setup/internal/steps/deploy.go +++ b/cmd/ate-setup/internal/steps/deploy.go @@ -164,6 +164,9 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { if err := e.applyWithAPIServerEnvHash(ctx, manifests); err != nil { return err } + if err := e.applyRouterMonitoring(ctx); err != nil { + return err + } // After the bundle, which resets the pod template to the sidecar-free base. if err := e.reconcileCloudSQLProxySidecar(ctx); err != nil { @@ -218,6 +221,27 @@ func (e *Env) DeployAteSystem(ctx context.Context, opts DeployOptions) error { return e.applyOtelEndpointOverride(ctx) } +// applyRouterMonitoring applies the Managed Prometheus scrape of the router's +// Envoy. Its PodMonitoring kind exists only where Google Managed Prometheus +// does, so the object is skipped on a cluster that does not serve it, not +// allowed to fail the install. Kind scrapes through its own collector. +// +// This is a temporary, GKE-specific skip, not an extension point. Do not extend +// it to other in-tree provider CRDs: provider-specific CRDs belong out of tree, +// and this one should move there longer term. +func (e *Env) applyRouterMonitoring(ctx context.Context) error { + if e.Cfg.Kind { + return nil + } + objs, err := kube.LoadPath(e.Cfg.Manifest("atenet-router-monitoring.yaml")) + if err != nil { + return err + } + return e.Kube.ApplyTolerant(ctx, objs, func(obj *unstructured.Unstructured, _ error) { + log.Warnf("skipping %s: its CRD is not installed on this cluster", kube.Describe(obj)) + }) +} + // DeployPodCertificateController creates any missing signer CA pools, applies // the podcertificate controller, and waits until it is rolling and has // published its trust bundles. diff --git a/cmd/ate-setup/internal/steps/deploy_test.go b/cmd/ate-setup/internal/steps/deploy_test.go index 12c504fdb8..7c1a1a95b1 100644 --- a/cmd/ate-setup/internal/steps/deploy_test.go +++ b/cmd/ate-setup/internal/steps/deploy_test.go @@ -117,6 +117,61 @@ func TestSetPodcertWorkersPerSignerRejectsMissingDeployment(t *testing.T) { } } +// A stock cluster has no Managed Prometheus CRD, so no system bundle may carry +// one: applying it would fail the whole install. +func TestSystemBundlesNeedNoManagedPrometheusCRD(t *testing.T) { + for _, tc := range []struct { + name string + cfg config.Config + }{ + {"base", config.Config{Router: config.RouterEnvoy}}, + {"agentgateway", config.Config{Router: config.RouterAgentgateway}}, + {"kind", config.Config{Router: config.RouterEnvoy, Kind: true}}, + {"kind agentgateway", config.Config{Router: config.RouterAgentgateway, Kind: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := tc.cfg + cfg.Root = repoRoot(t) + e := &Env{Cfg: &cfg} + manifest, err := e.render(cfg.Path(SystemOverlay(&cfg))) + if err != nil { + t.Fatalf("render: %v", err) + } + objs, err := kube.DecodeManifestBytes(manifest) + if err != nil { + t.Fatalf("decoding the bundle: %v", err) + } + for _, obj := range objs { + if obj.GroupVersionKind().Group == "monitoring.googleapis.com" { + t.Errorf("bundle carries %s, which needs a Managed Prometheus CRD", kube.Describe(obj)) + } + } + }) + } +} + +// Kind has no Managed Prometheus, so it never reaches the cluster for the +// router monitoring. The nil Kube client panics if it does. +func TestApplyRouterMonitoringSkipsKind(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: repoRoot(t), Kind: true}} + if err := e.applyRouterMonitoring(t.Context()); err != nil { + t.Errorf("applyRouterMonitoring() = %v, want nil", err) + } +} + +// The monitoring leaves the bundles, so the file the installer applies on its +// own has to be the router's PodMonitoring. +func TestRouterMonitoringManifest(t *testing.T) { + e := &Env{Cfg: &config.Config{Root: repoRoot(t)}} + objs, err := kube.LoadPath(e.Cfg.Manifest("atenet-router-monitoring.yaml")) + if err != nil { + t.Fatal(err) + } + if len(objs) != 1 || objs[0].GetKind() != "PodMonitoring" || objs[0].GetNamespace() != "ate-system" { + t.Errorf("manifest holds %v, want one PodMonitoring in ate-system", objs) + } +} + // Upgrading from a release that predates a signer pool: deploying the // controller alone has to create the missing pool before the new pod needs it. // The manifest root is empty, so the deploy stops at rendering, after the pools. diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index 765aaf937b..5d6607019e 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -26,12 +26,15 @@ kind: Kustomization # both the overlay's flags and the env override. The same goes for the # sandboxconfig and atenet-egress files: cmd/ate-setup applies the variant it # selects, so they are not listed here either. +# +# atenet-router-monitoring.yaml is absent too: its PodMonitoring kind exists +# only where Google Managed Prometheus does, so cmd/ate-setup applies it +# separately and skips it on a cluster without that CRD. resources: - ../ate-api-server.yaml - ../ate-idp-server.yaml - ../ate-controller.yaml - ../atelet.yaml - ../atenet-router.yaml - - ../atenet-router-monitoring.yaml - ../ate-otel-config.yaml - ../ate-system-networkpolicies.yaml