From 2e9ad0b53912ee95ad852bd4b01db9211d968ddd Mon Sep 17 00:00:00 2001 From: Wan Nur Shafiqah <176239359+shanurwan@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:25:13 +0800 Subject: [PATCH] fix(operator): requeue with a delay when the ObjectStore is missing When a Cluster refers to an ObjectStore that does not exist yet, the pre-reconcile hook asked for a requeue without a delay. The operator maps that to a stopped reconciliation with RequeueAfter 0, which controller-runtime does not requeue, so the Cluster never got any instance until something changed it. This happens when both objects are applied together and the Cluster lands first, as with Helm or a GitOps tool. Set RequeueAfter to 10 seconds in that case, and add a unit test for the Pre hook and an e2e test that creates the Cluster before its ObjectStore. Closes #1236 Signed-off-by: Wan Nur Shafiqah <176239359+shanurwan@users.noreply.github.com> Assisted-by: Claude Opus 5.5 --- internal/cnpgi/operator/reconciler.go | 10 +- internal/cnpgi/operator/reconciler_test.go | 143 ++++++++++++++++++ test/e2e/e2e_suite_test.go | 1 + .../internal/tests/objectstoreorder/doc.go | 23 +++ .../objectstoreorder/objectstore_order.go | 128 ++++++++++++++++ 5 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 internal/cnpgi/operator/reconciler_test.go create mode 100644 test/e2e/internal/tests/objectstoreorder/doc.go create mode 100644 test/e2e/internal/tests/objectstoreorder/objectstore_order.go diff --git a/internal/cnpgi/operator/reconciler.go b/internal/cnpgi/operator/reconciler.go index dd92dd877..6b6d39118 100644 --- a/internal/cnpgi/operator/reconciler.go +++ b/internal/cnpgi/operator/reconciler.go @@ -35,6 +35,13 @@ import ( "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/rbac" ) +// objectStoreNotFoundRequeueAfter is how many seconds the operator waits +// before running the reconciliation loop again when a Cluster refers to an +// ObjectStore that does not exist yet. A requeue request without a delay is +// dropped by the operator, so a Cluster created before its ObjectStore would +// otherwise never be reconciled again until it is changed. +const objectStoreNotFoundRequeueAfter int64 = 10 + // ReconcilerImplementation implements the Reconciler capability type ReconcilerImplementation struct { Client client.Client @@ -101,7 +108,8 @@ func (r ReconcilerImplementation) Pre( "name", barmanObjectKey.Name, "namespace", barmanObjectKey.Namespace) return &reconciler.ReconcilerHooksResult{ - Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_REQUEUE, + Behavior: reconciler.ReconcilerHooksResult_BEHAVIOR_REQUEUE, + RequeueAfter: objectStoreNotFoundRequeueAfter, }, nil } diff --git a/internal/cnpgi/operator/reconciler_test.go b/internal/cnpgi/operator/reconciler_test.go new file mode 100644 index 000000000..d7d9cdf63 --- /dev/null +++ b/internal/cnpgi/operator/reconciler_test.go @@ -0,0 +1,143 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package operator + +import ( + "encoding/json" + + cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + "github.com/cloudnative-pg/cnpg-i/pkg/reconciler" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ReconcilerImplementation", func() { + const ( + namespace = "test-namespace" + clusterName = "cluster" + objectStoreName = "object-store" + ) + + var cluster *cnpgv1.Cluster + + newScheme := func() *runtime.Scheme { + s := runtime.NewScheme() + barmancloudv1.AddKnownTypes(s) + Expect(cnpgv1.AddToScheme(s)).To(Succeed()) + Expect(rbacv1.AddToScheme(s)).To(Succeed()) + return s + } + + newRequest := func(obj any) *reconciler.ReconcilerHooksRequest { + definition, err := json.Marshal(obj) + Expect(err).NotTo(HaveOccurred()) + return &reconciler.ReconcilerHooksRequest{ResourceDefinition: definition} + } + + BeforeEach(func() { + cluster = &cnpgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: clusterName, + Namespace: namespace, + }, + Spec: cnpgv1.ClusterSpec{ + Plugins: []cnpgv1.PluginConfiguration{ + { + Name: metadata.PluginName, + IsWALArchiver: ptr.To(true), + Parameters: map[string]string{ + "barmanObjectName": objectStoreName, + }, + }, + }, + }, + } + }) + + Describe("Pre", func() { + It("asks to be called again after a delay when the ObjectStore does not exist yet", func(ctx SpecContext) { + r := ReconcilerImplementation{ + Client: fake.NewClientBuilder().WithScheme(newScheme()).Build(), + } + + result, err := r.Pre(ctx, newRequest(cluster)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.GetBehavior()).To(Equal(reconciler.ReconcilerHooksResult_BEHAVIOR_REQUEUE)) + // Without a delay the operator does not requeue at all, and the + // Cluster is never reconciled again (issue #1236). + Expect(result.GetRequeueAfter()).To(BeNumerically(">", 0)) + Expect(result.GetRequeueAfter()).To(Equal(objectStoreNotFoundRequeueAfter)) + }) + + It("continues once the ObjectStore exists", func(ctx SpecContext) { + store := &barmancloudv1.ObjectStore{ + TypeMeta: metav1.TypeMeta{ + Kind: "ObjectStore", + APIVersion: barmancloudv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: objectStoreName, + Namespace: namespace, + }, + } + r := ReconcilerImplementation{ + Client: fake.NewClientBuilder().WithScheme(newScheme()).WithObjects(store).Build(), + } + + result, err := r.Pre(ctx, newRequest(cluster)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.GetBehavior()).To(Equal(reconciler.ReconcilerHooksResult_BEHAVIOR_CONTINUE)) + Expect(result.GetRequeueAfter()).To(BeZero()) + }) + + It("continues without looking up ObjectStores for kinds other than Cluster", func(ctx SpecContext) { + r := ReconcilerImplementation{ + Client: fake.NewClientBuilder().WithScheme(newScheme()).Build(), + } + backup := &cnpgv1.Backup{ + TypeMeta: metav1.TypeMeta{ + Kind: "Backup", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + Namespace: namespace, + }, + } + + result, err := r.Pre(ctx, newRequest(backup)) + Expect(err).NotTo(HaveOccurred()) + Expect(result.GetBehavior()).To(Equal(reconciler.ReconcilerHooksResult_BEHAVIOR_CONTINUE)) + }) + }) +}) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 635ed3077..494033743 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,7 @@ import ( _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/backup" _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/credentialrotation" _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/highavailability" + _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/objectstoreorder" _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/replicacluster" _ "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/tests/walrestore" diff --git a/test/e2e/internal/tests/objectstoreorder/doc.go b/test/e2e/internal/tests/objectstoreorder/doc.go new file mode 100644 index 000000000..1861eb8b1 --- /dev/null +++ b/test/e2e/internal/tests/objectstoreorder/doc.go @@ -0,0 +1,23 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +// Package objectstoreorder contains tests validating that a Cluster created +// before the ObjectStore it refers to is reconciled once the ObjectStore +// exists, as happens when both are applied together by Helm or a GitOps tool. +package objectstoreorder diff --git a/test/e2e/internal/tests/objectstoreorder/objectstore_order.go b/test/e2e/internal/tests/objectstoreorder/objectstore_order.go new file mode 100644 index 000000000..61d97ae2e --- /dev/null +++ b/test/e2e/internal/tests/objectstoreorder/objectstore_order.go @@ -0,0 +1,128 @@ +/* +Copyright © contributors to CloudNativePG, established as +CloudNativePG a Series of LF Projects, LLC. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package objectstoreorder + +import ( + "time" + + cloudnativepgv1 "github.com/cloudnative-pg/api/pkg/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + internalClient "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/client" + internalCluster "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/cluster" + nmsp "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/namespace" + "github.com/cloudnative-pg/plugin-barman-cloud/test/e2e/internal/objectstore" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + objectStoreName = "store" + s3SecretName = "s3" + clusterName = "cluster" +) + +var _ = Describe("Cluster created before its ObjectStore", func() { + var namespace *corev1.Namespace + var cl client.Client + + BeforeEach(func(ctx SpecContext) { + var err error + cl, _, err = internalClient.NewClient() + Expect(err).NotTo(HaveOccurred()) + namespace, err = nmsp.CreateUniqueNamespace(ctx, cl, "objectstore-order") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func(ctx SpecContext) { + Expect(cl.Delete(ctx, namespace)).To(Succeed()) + }) + + It("becomes ready once the ObjectStore is created, without any change to the Cluster", func(ctx SpecContext) { + By("starting the object storage server") + resources := objectstore.NewS3ObjectStoreResources(namespace.Name, s3SecretName) + Expect(resources.Create(ctx, cl)).To(Succeed()) + + By("creating the Cluster before the ObjectStore it refers to") + cluster := newCluster(namespace.Name, clusterName, objectStoreName) + Expect(cl.Create(ctx, cluster)).To(Succeed()) + createdGeneration := cluster.Generation + + By("checking that no instance is created while the ObjectStore is missing") + Consistently(func(g Gomega) { + var pods corev1.PodList + g.Expect(cl.List(ctx, &pods, + client.InNamespace(namespace.Name), + client.MatchingLabels{"cnpg.io/cluster": clusterName}, + )).To(Succeed()) + g.Expect(pods.Items).To(BeEmpty()) + }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(Succeed()) + + By("creating the ObjectStore") + store := objectstore.NewS3ObjectStore(namespace.Name, objectStoreName, s3SecretName) + Expect(cl.Create(ctx, store)).To(Succeed()) + + By("waiting for the Cluster to become ready on its own") + Eventually(func(g Gomega) { + g.Expect(cl.Get(ctx, types.NamespacedName{ + Name: cluster.Name, + Namespace: cluster.Namespace, + }, cluster)).To(Succeed()) + g.Expect(internalCluster.IsReady(*cluster)).To(BeTrue()) + }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + By("checking that the Cluster spec was not changed to get it reconciled") + Expect(cluster.Generation).To(Equal(createdGeneration)) + }) +}) + +func newCluster(namespace, name, objectStore string) *cloudnativepgv1.Cluster { + return &cloudnativepgv1.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "postgresql.cnpg.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: cloudnativepgv1.ClusterSpec{ + Instances: 1, + ImagePullPolicy: corev1.PullAlways, + Plugins: []cloudnativepgv1.PluginConfiguration{ + { + Name: "barman-cloud.cloudnative-pg.io", + Parameters: map[string]string{ + "barmanObjectName": objectStore, + }, + IsWALArchiver: ptr.To(true), + }, + }, + StorageConfiguration: cloudnativepgv1.StorageConfiguration{ + Size: "1Gi", + }, + }, + } +}