From 763bfda817cb145ab10782a669d012323630f8d7 Mon Sep 17 00:00:00 2001 From: I515719 Date: Tue, 3 Nov 2026 15:48:45 +0800 Subject: [PATCH 1/5] fix(DM01-6271): add zone fallback for GKE cluster creation on GCE_STOCKOUT When a GKE cluster creation fails with ERROR status (e.g. GCE_STOCKOUT), the operator now automatically deletes the failed cluster and retries in the next available zone instead of permanently failing. - Add Zones []string to GKEClusterSpec for user-configurable fallback order - Add ZoneIndex int to GKEClusterStatus to track current fallback position - Default fallback covers europe-west1/west3 zones only (all current users require n1 machine types unavailable in europe-west10/west12) - Primary zone is always tried first; duplicates removed from default list - Update DeepCopyInto to correctly handle the new Zones slice field - Permanently fail only after all configured zones are exhausted --- .../gcp/pkg/apis/gcp/v1alpha1/types.go | 34 +++++++------- .../gcp/v1alpha1/zz_generated.deepcopy.go | 7 ++- src/services/gcp/pkg/stub/handler.go | 44 ++++++++++++++++++- 3 files changed, 66 insertions(+), 19 deletions(-) diff --git a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go index e514d1f84..77c884367 100644 --- a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go +++ b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go @@ -16,22 +16,23 @@ type GKECluster struct { } type GKEClusterSpec struct { - DiskSize int32 `json:"diskSize,omitempty"` - MachineType string `json:"machineType,omitempty"` - EnableNetworkPolicy bool `json:"enableNetworkPolicy,omitempty"` - StackType string `json:"stackType,omitempty"` - DisableLegacyAuthorization bool `json:"disableLegacyAuthorization,omitempty"` - EnablePodSecurityPolicy bool `json:"enablePodSecurityPolicy,omitempty"` - NumNodes int32 `json:"numNodes,omitempty"` - Preemptible bool `json:"preemptible,omitempty"` - EnableAutoscaling bool `json:"enableAutoscaling,omitempty"` - MaxNodes int32 `json:"maxNodes,omitempty"` - MinNodes int32 `json:"minNodes,omitempty"` - ClusterVersion string `json:"clusterVersion,omitempty"` - Zone string `json:"zone"` - ServiceCidr string `json:"serviceCidr,omitempty"` - ClusterCidr string `json:"clusterCidr,omitempty"` - EnableManagedPrometheus bool `json:"enableManagedPrometheus,omitempty"` + DiskSize int32 `json:"diskSize,omitempty"` + MachineType string `json:"machineType,omitempty"` + EnableNetworkPolicy bool `json:"enableNetworkPolicy,omitempty"` + StackType string `json:"stackType,omitempty"` + DisableLegacyAuthorization bool `json:"disableLegacyAuthorization,omitempty"` + EnablePodSecurityPolicy bool `json:"enablePodSecurityPolicy,omitempty"` + NumNodes int32 `json:"numNodes,omitempty"` + Preemptible bool `json:"preemptible,omitempty"` + EnableAutoscaling bool `json:"enableAutoscaling,omitempty"` + MaxNodes int32 `json:"maxNodes,omitempty"` + MinNodes int32 `json:"minNodes,omitempty"` + ClusterVersion string `json:"clusterVersion,omitempty"` + Zone string `json:"zone"` + Zones []string `json:"zones,omitempty"` + ServiceCidr string `json:"serviceCidr,omitempty"` + ClusterCidr string `json:"clusterCidr,omitempty"` + EnableManagedPrometheus bool `json:"enableManagedPrometheus,omitempty"` } type GKEClusterStatus struct { @@ -39,6 +40,7 @@ type GKEClusterStatus struct { Message string `json:"message,omitempty"` ClusterName string `json:"clusterName,omitempty"` FirstCleanedAt string `json:"firstCleanedAt,omitempty"` + ZoneIndex int `json:"zoneIndex,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go b/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go index 51ce83ebb..9a07722ec 100644 --- a/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go +++ b/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go @@ -13,7 +13,7 @@ func (in *GKECluster) DeepCopyInto(out *GKECluster) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status return } @@ -74,6 +74,11 @@ func (in *GKEClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GKEClusterSpec) DeepCopyInto(out *GKEClusterSpec) { *out = *in + if in.Zones != nil { + in, out := &in.Zones, &out.Zones + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go index fc110311b..a137889b0 100644 --- a/src/services/gcp/pkg/stub/handler.go +++ b/src/services/gcp/pkg/stub/handler.go @@ -311,8 +311,48 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl return &status, nil } if gkecluster.Status == "ERROR" { - log.Errorf("Error creating cluster %s", cr.Status.ClusterName) - return nil, goerrors.New(fmt.Sprintf("error creating GKE cluster %s, which means the cluster get an ERROR status on GCP, please restart current job or check it on GCP directly", cr.Status.ClusterName)) + log.Errorf("GKE cluster %s is in ERROR state in zone %s (possibly GCE_STOCKOUT), attempting zone fallback", cr.Status.ClusterName, cr.Spec.Zone) + + // Delete the failed cluster on GCP; GC will catch it if this fails + if err := deleteRemoteCluster(cr, log); err != nil { + log.Warningf("Failed to delete errored cluster %s, GC will clean it up: %v", cr.Status.ClusterName, err) + } + + // Build the ordered zone list: Zones takes precedence, fall back to single Zone. + // Default covers europe-west1/west3 only — all current users require n1 machine + // types which are not available in europe-west10/west12. + zones := cr.Spec.Zones + if len(zones) == 0 { + allZones := []string{ + "europe-west3-c", "europe-west3-a", "europe-west3-b", + "europe-west1-b", "europe-west1-c", "europe-west1-d", + } + zones = []string{cr.Spec.Zone} + for _, z := range allZones { + if z != cr.Spec.Zone { + zones = append(zones, z) + } + } + } + + nextIndex := cr.Status.ZoneIndex + 1 + if nextIndex >= len(zones) { + return nil, fmt.Errorf("GKE cluster creation failed in all configured zones %v", zones) + } + + nextZone := zones[nextIndex] + log.Infof("Retrying GKE cluster creation in zone %s (attempt %d/%d)", nextZone, nextIndex+1, len(zones)) + + cr.Spec.Zone = nextZone + cr.Status.ZoneIndex = nextIndex + cr.SetFinalizers([]string{}) + cr.Status.ClusterName = "" + cr.Status.Status = "pending" + cr.Status.Message = fmt.Sprintf("Retrying in zone %s (%d/%d)", nextZone, nextIndex+1, len(zones)) + if err := action.Update(cr); err != nil { + return nil, err + } + return &cr.Status, nil } } From d6667d441c136b792ae44b38467893f53b85b639 Mon Sep 17 00:00:00 2001 From: I515719 Date: Tue, 3 Nov 2026 15:54:54 +0800 Subject: [PATCH 2/5] fix(DM01-6271): add machine type fallback on top of zone fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the GCE_STOCKOUT fallback to also try equivalent machine types (n1 → n2 → e2 for standard/highmem; n1 → n2 for highcpu) after all zones are exhausted for the current machine type. - Add MachineTypes []string to GKEClusterSpec for user-configurable list - Replace ZoneIndex with FallbackIndex in GKEClusterStatus (flat index over zones × machineTypes product) - Add buildFallbackList(): auto-generates equivalent machine types by swapping series (n1/n2/e2) while keeping same tier and vCPU count - Add parseMachineType() and buildEquivalentMachineTypes() helpers - Update DeepCopyInto for new MachineTypes slice field --- .../gcp/pkg/apis/gcp/v1alpha1/types.go | 9 +- .../gcp/v1alpha1/zz_generated.deepcopy.go | 5 + src/services/gcp/pkg/stub/handler.go | 104 +++++++++++++----- 3 files changed, 88 insertions(+), 30 deletions(-) diff --git a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go index 77c884367..99a231520 100644 --- a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go +++ b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go @@ -18,6 +18,7 @@ type GKECluster struct { type GKEClusterSpec struct { DiskSize int32 `json:"diskSize,omitempty"` MachineType string `json:"machineType,omitempty"` + MachineTypes []string `json:"machineTypes,omitempty"` EnableNetworkPolicy bool `json:"enableNetworkPolicy,omitempty"` StackType string `json:"stackType,omitempty"` DisableLegacyAuthorization bool `json:"disableLegacyAuthorization,omitempty"` @@ -36,11 +37,11 @@ type GKEClusterSpec struct { } type GKEClusterStatus struct { - Status string `json:"status,omitempty"` - Message string `json:"message,omitempty"` - ClusterName string `json:"clusterName,omitempty"` + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + ClusterName string `json:"clusterName,omitempty"` FirstCleanedAt string `json:"firstCleanedAt,omitempty"` - ZoneIndex int `json:"zoneIndex,omitempty"` + FallbackIndex int `json:"fallbackIndex,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go b/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go index 9a07722ec..752f291ae 100644 --- a/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go +++ b/src/services/gcp/pkg/apis/gcp/v1alpha1/zz_generated.deepcopy.go @@ -79,6 +79,11 @@ func (in *GKEClusterSpec) DeepCopyInto(out *GKEClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.MachineTypes != nil { + in, out := &in.MachineTypes, &out.MachineTypes + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go index a137889b0..6a9a3e505 100644 --- a/src/services/gcp/pkg/stub/handler.go +++ b/src/services/gcp/pkg/stub/handler.go @@ -84,6 +84,73 @@ func NewHandler() handler.Handler { type Handler struct{} +type fallbackOption struct { + Zone string + MachineType string +} + +func parseMachineType(mt string) (series, tier, size string, ok bool) { + parts := strings.SplitN(mt, "-", 3) + if len(parts) != 3 { + return "", "", "", false + } + return parts[0], parts[1], parts[2], true +} + +func buildEquivalentMachineTypes(mt string) []string { + series, tier, size, ok := parseMachineType(mt) + if !ok { + return []string{mt} + } + var candidates []string + switch series { + case "n1", "n2", "e2": + if tier == "highcpu" { + candidates = []string{"n1", "n2"} // e2 has no highcpu series + } else { + candidates = []string{"n1", "n2", "e2"} + } + default: + return []string{mt} + } + result := []string{mt} + for _, s := range candidates { + if s != series { + result = append(result, s+"-"+tier+"-"+size) + } + } + return result +} + +func buildFallbackList(cr *v1alpha1.GKECluster) []fallbackOption { + zones := cr.Spec.Zones + if len(zones) == 0 { + allZones := []string{ + "europe-west3-c", "europe-west3-a", "europe-west3-b", + "europe-west1-b", "europe-west1-c", "europe-west1-d", + } + zones = []string{cr.Spec.Zone} + for _, z := range allZones { + if z != cr.Spec.Zone { + zones = append(zones, z) + } + } + } + + machineTypes := cr.Spec.MachineTypes + if len(machineTypes) == 0 { + machineTypes = buildEquivalentMachineTypes(cr.Spec.MachineType) + } + + var list []fallbackOption + for _, mt := range machineTypes { + for _, z := range zones { + list = append(list, fallbackOption{Zone: z, MachineType: mt}) + } + } + return list +} + func setClusterName(cr *v1alpha1.GKECluster, log *logrus.Entry) error { finalizers := cr.GetFinalizers() if len(finalizers) == 0 { @@ -311,44 +378,29 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl return &status, nil } if gkecluster.Status == "ERROR" { - log.Errorf("GKE cluster %s is in ERROR state in zone %s (possibly GCE_STOCKOUT), attempting zone fallback", cr.Status.ClusterName, cr.Spec.Zone) + log.Errorf("GKE cluster %s is in ERROR state in zone %s with machineType %s (possibly GCE_STOCKOUT), attempting fallback", cr.Status.ClusterName, cr.Spec.Zone, cr.Spec.MachineType) // Delete the failed cluster on GCP; GC will catch it if this fails if err := deleteRemoteCluster(cr, log); err != nil { log.Warningf("Failed to delete errored cluster %s, GC will clean it up: %v", cr.Status.ClusterName, err) } - // Build the ordered zone list: Zones takes precedence, fall back to single Zone. - // Default covers europe-west1/west3 only — all current users require n1 machine - // types which are not available in europe-west10/west12. - zones := cr.Spec.Zones - if len(zones) == 0 { - allZones := []string{ - "europe-west3-c", "europe-west3-a", "europe-west3-b", - "europe-west1-b", "europe-west1-c", "europe-west1-d", - } - zones = []string{cr.Spec.Zone} - for _, z := range allZones { - if z != cr.Spec.Zone { - zones = append(zones, z) - } - } - } - - nextIndex := cr.Status.ZoneIndex + 1 - if nextIndex >= len(zones) { - return nil, fmt.Errorf("GKE cluster creation failed in all configured zones %v", zones) + fallbackList := buildFallbackList(cr) + nextIndex := cr.Status.FallbackIndex + 1 + if nextIndex >= len(fallbackList) { + return nil, fmt.Errorf("GKE cluster creation failed for all fallback options (zones x machineTypes exhausted)") } - nextZone := zones[nextIndex] - log.Infof("Retrying GKE cluster creation in zone %s (attempt %d/%d)", nextZone, nextIndex+1, len(zones)) + next := fallbackList[nextIndex] + log.Infof("Retrying with zone=%s machineType=%s (attempt %d/%d)", next.Zone, next.MachineType, nextIndex+1, len(fallbackList)) - cr.Spec.Zone = nextZone - cr.Status.ZoneIndex = nextIndex + cr.Spec.Zone = next.Zone + cr.Spec.MachineType = next.MachineType + cr.Status.FallbackIndex = nextIndex cr.SetFinalizers([]string{}) cr.Status.ClusterName = "" cr.Status.Status = "pending" - cr.Status.Message = fmt.Sprintf("Retrying in zone %s (%d/%d)", nextZone, nextIndex+1, len(zones)) + cr.Status.Message = fmt.Sprintf("Retrying in zone %s with %s (%d/%d)", next.Zone, next.MachineType, nextIndex+1, len(fallbackList)) if err := action.Update(cr); err != nil { return nil, err } From a0c57ade7a51dac53c666120dc5189ca9ec46055 Mon Sep 17 00:00:00 2001 From: I515719 Date: Tue, 3 Nov 2026 16:00:43 +0800 Subject: [PATCH 3/5] fix(DM01-6271): improve error message when all fallback options exhausted Include tried zones, machine types, root cause (GCE_STOCKOUT), and point users to GCP status page or administrator instead of vague retry advice. --- src/services/gcp/pkg/stub/handler.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go index 6a9a3e505..303e2eb73 100644 --- a/src/services/gcp/pkg/stub/handler.go +++ b/src/services/gcp/pkg/stub/handler.go @@ -388,7 +388,30 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl fallbackList := buildFallbackList(cr) nextIndex := cr.Status.FallbackIndex + 1 if nextIndex >= len(fallbackList) { - return nil, fmt.Errorf("GKE cluster creation failed for all fallback options (zones x machineTypes exhausted)") + // Collect unique zones and machine types for diagnosis + zoneSet := make(map[string]struct{}) + mtSet := make(map[string]struct{}) + for _, o := range fallbackList { + zoneSet[o.Zone] = struct{}{} + mtSet[o.MachineType] = struct{}{} + } + var zones, mts []string + for z := range zoneSet { + zones = append(zones, z) + } + for m := range mtSet { + mts = append(mts, m) + } + return nil, fmt.Errorf( + "GKE cluster creation failed after %d attempts. "+ + "Zones tried: [%s]. Machine types tried: [%s]. "+ + "Root cause: GCE_STOCKOUT — Google Compute Engine does not have enough resources "+ + "available in any of the configured zones. "+ + "Check https://status.cloud.google.com or contact your GCP administrator.", + len(fallbackList), + strings.Join(zones, ", "), + strings.Join(mts, ", "), + ) } next := fallbackList[nextIndex] From cedb9b34edb46f07747538d01738d7d056a05680 Mon Sep 17 00:00:00 2001 From: I515719 Date: Tue, 3 Nov 2026 17:03:50 +0800 Subject: [PATCH 4/5] fix(DM01-6271): store fallback state in Status instead of mutating Spec Spec.Zone and Spec.MachineType now always reflect the user's original intent. Fallback runtime state is tracked in Status.FallbackZone and Status.FallbackMachineType. createCluster prefers Status fallback values when set, falling back to Spec otherwise. This also makes buildFallbackList stable: it always builds from Spec (original values), so FallbackIndex reliably maps to the same list. --- .../gcp/pkg/apis/gcp/v1alpha1/types.go | 12 +++++++----- src/services/gcp/pkg/stub/handler.go | 18 +++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go index 99a231520..9a4213b0b 100644 --- a/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go +++ b/src/services/gcp/pkg/apis/gcp/v1alpha1/types.go @@ -37,11 +37,13 @@ type GKEClusterSpec struct { } type GKEClusterStatus struct { - Status string `json:"status,omitempty"` - Message string `json:"message,omitempty"` - ClusterName string `json:"clusterName,omitempty"` - FirstCleanedAt string `json:"firstCleanedAt,omitempty"` - FallbackIndex int `json:"fallbackIndex,omitempty"` + Status string `json:"status,omitempty"` + Message string `json:"message,omitempty"` + ClusterName string `json:"clusterName,omitempty"` + FirstCleanedAt string `json:"firstCleanedAt,omitempty"` + FallbackIndex int `json:"fallbackIndex,omitempty"` + FallbackZone string `json:"fallbackZone,omitempty"` + FallbackMachineType string `json:"fallbackMachineType,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go index 303e2eb73..55cebc8b8 100644 --- a/src/services/gcp/pkg/stub/handler.go +++ b/src/services/gcp/pkg/stub/handler.go @@ -208,12 +208,16 @@ func createCluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKEClu } log.Infof("Create GKE cluster %s", cr.Status.ClusterName) + zone := cr.Spec.Zone + if cr.Status.FallbackZone != "" { + zone = cr.Status.FallbackZone + } args := []string{"container", "clusters", "create", cr.Status.ClusterName, "--async", "--enable-autorepair", "--scopes=gke-default,storage-rw", - "--zone", cr.Spec.Zone, + "--zone", zone, } args = append(args, "--metadata") @@ -224,9 +228,13 @@ func createCluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKEClu args = append(args, strconv.Itoa(int(cr.Spec.DiskSize))) } - if cr.Spec.MachineType != "" { + machineType := cr.Spec.MachineType + if cr.Status.FallbackMachineType != "" { + machineType = cr.Status.FallbackMachineType + } + if machineType != "" { args = append(args, "--machine-type") - args = append(args, cr.Spec.MachineType) + args = append(args, machineType) } if cr.Spec.StackType != "" { @@ -417,8 +425,8 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl next := fallbackList[nextIndex] log.Infof("Retrying with zone=%s machineType=%s (attempt %d/%d)", next.Zone, next.MachineType, nextIndex+1, len(fallbackList)) - cr.Spec.Zone = next.Zone - cr.Spec.MachineType = next.MachineType + cr.Status.FallbackZone = next.Zone + cr.Status.FallbackMachineType = next.MachineType cr.Status.FallbackIndex = nextIndex cr.SetFinalizers([]string{}) cr.Status.ClusterName = "" From f0ad24dc70064fed8c2f60b6af6228eaafb8ee75 Mon Sep 17 00:00:00 2001 From: I515719 Date: Tue, 3 Nov 2026 17:22:23 +0800 Subject: [PATCH 5/5] fix(DM01-6271): remove comments --- src/services/gcp/pkg/stub/handler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/services/gcp/pkg/stub/handler.go b/src/services/gcp/pkg/stub/handler.go index 55cebc8b8..ec81a287e 100644 --- a/src/services/gcp/pkg/stub/handler.go +++ b/src/services/gcp/pkg/stub/handler.go @@ -106,7 +106,7 @@ func buildEquivalentMachineTypes(mt string) []string { switch series { case "n1", "n2", "e2": if tier == "highcpu" { - candidates = []string{"n1", "n2"} // e2 has no highcpu series + candidates = []string{"n1", "n2"} } else { candidates = []string{"n1", "n2", "e2"} } @@ -388,7 +388,6 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl if gkecluster.Status == "ERROR" { log.Errorf("GKE cluster %s is in ERROR state in zone %s with machineType %s (possibly GCE_STOCKOUT), attempting fallback", cr.Status.ClusterName, cr.Spec.Zone, cr.Spec.MachineType) - // Delete the failed cluster on GCP; GC will catch it if this fails if err := deleteRemoteCluster(cr, log); err != nil { log.Warningf("Failed to delete errored cluster %s, GC will clean it up: %v", cr.Status.ClusterName, err) } @@ -396,7 +395,6 @@ func syncGKECluster(cr *v1alpha1.GKECluster, log *logrus.Entry) (*v1alpha1.GKECl fallbackList := buildFallbackList(cr) nextIndex := cr.Status.FallbackIndex + 1 if nextIndex >= len(fallbackList) { - // Collect unique zones and machine types for diagnosis zoneSet := make(map[string]struct{}) mtSet := make(map[string]struct{}) for _, o := range fallbackList {