From c5817d7855287d771b8434bef625c88438c5c82a Mon Sep 17 00:00:00 2001 From: Pavel Tishkov Date: Thu, 12 Jun 2025 12:48:16 +0300 Subject: [PATCH 1/5] test(module): add cleanup node labels hook Signed-off-by: Pavel Tishkov --- images/hooks/cmd/cleanup-labels/main.go | 116 ++++++++++++++++++++++++ images/hooks/werf.inc.yaml | 1 + 2 files changed, 117 insertions(+) create mode 100644 images/hooks/cmd/cleanup-labels/main.go diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go new file mode 100644 index 0000000000..6547b3951a --- /dev/null +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -0,0 +1,116 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The purpose of this hook is to prevent already launched virt-handler pods from flapping, since the node group configuration virtualization-detect-kvm.sh will be responsible for installing the label virtualization.deckhouse.io/kvm-enabled. + +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/pkg/app" + "github.com/deckhouse/module-sdk/pkg/registry" + "k8s.io/utils/ptr" + + "hooks/pkg/common" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + nodesMetadataSnapshot = "virthandler-nodes" + virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" + nodeJQFilter = ".metadata" +) + +var _ = registry.RegisterFunc(configDiscoveryService, handleVirtHandlerNodes) + +var configDiscoveryService = &pkg.HookConfig{ + OnAfterDeleteHelm: &pkg.OrderedConfig{Order: 5}, + Kubernetes: []pkg.KubernetesConfig{ + { + Name: nodesMetadataSnapshot, + APIVersion: "v1", + Kind: "Node", + JqFilter: nodeJQFilter, + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: virtHandlerLabel, + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + ExecuteHookOnSynchronization: ptr.To(false), + ExecuteHookOnEvents: ptr.To(false), + }, + }, + + Queue: fmt.Sprintf("modules/%s", common.MODULE_NAME), +} + +func handleVirtHandlerNodes(_ context.Context, input *pkg.HookInput) error { + input.Logger.Info("Start.") + + nodeMetadatas := input.Snapshots.Get(nodesMetadataSnapshot) + input.Logger.Info(fmt.Sprintf("Found %d nodes", len(nodeMetadatas))) + if len(nodeMetadatas) == 0 { + return nil + } + + for _, nodeMetadata := range nodeMetadatas { + metadata := &metav1.ObjectMeta{} + if err := nodeMetadata.UnmarshalTo(metadata); err != nil { + input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err)) + continue + } + + patches := []map[string]interface{}{} + + for key, _ := range metadata.Labels { + if strings.Contains(key, "virtualization.deckhouse.io") { + patches = append(patches, map[string]interface{}{ + "op": "remove", + "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), + }) + } + } + + if len(patches) == 0 { + input.Logger.Info("No labels found, nothing to do.") + continue + } else { + input.Logger.Info(fmt.Sprintf("Removing %d labels from node %s", len(patches), metadata.Name)) + } + + input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", metadata.Name) + } + input.Logger.Info("Done.") + return nil +} + +func jsonPatchEscape(s string) string { + s = strings.ReplaceAll(s, "~", "~0") + s = strings.ReplaceAll(s, "/", "~1") + return s +} + +func main() { + app.Run() +} diff --git a/images/hooks/werf.inc.yaml b/images/hooks/werf.inc.yaml index 5d13dd4b33..ccb6d9410f 100644 --- a/images/hooks/werf.inc.yaml +++ b/images/hooks/werf.inc.yaml @@ -35,3 +35,4 @@ shell: - go build -ldflags="-s -w" -o /hooks/generate-secret-for-dvcr ./cmd/generate-secret-for-dvcr - go build -ldflags="-s -w" -o /hooks/discovery-clusterip-service-for-dvcr ./cmd/discovery-clusterip-service-for-dvcr - go build -ldflags="-s -w" -o /hooks/discovery-workload-nodes ./cmd/discovery-workload-nodes + - go build -ldflags="-s -w" -o /hooks/cleanup-labels ./cmd/cleanup-labels From 1714537afdf884e211b845297b69345d8e4ce662 Mon Sep 17 00:00:00 2001 From: Pavel Tishkov Date: Thu, 12 Jun 2025 15:27:52 +0300 Subject: [PATCH 2/5] refactor Signed-off-by: Pavel Tishkov --- images/hooks/cmd/cleanup-labels/main.go | 42 ++++++++++++++----------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go index 6547b3951a..e6e6e1da59 100644 --- a/images/hooks/cmd/cleanup-labels/main.go +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -34,18 +34,27 @@ import ( ) const ( - nodesMetadataSnapshot = "virthandler-nodes" - virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" - nodeJQFilter = ".metadata" + nodesSnapshot = "virthandler-nodes" + virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" + labelPattern = "virtualization.deckhouse.io" + nodeJQFilter = `{ + "name": .metadata.name, + "labels": .metadata.labels, + }` ) -var _ = registry.RegisterFunc(configDiscoveryService, handleVirtHandlerNodes) +type NodeInfo struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` +} + +var _ = registry.RegisterFunc(configDiscoveryService, cleanUpNodeLabels) var configDiscoveryService = &pkg.HookConfig{ OnAfterDeleteHelm: &pkg.OrderedConfig{Order: 5}, Kubernetes: []pkg.KubernetesConfig{ { - Name: nodesMetadataSnapshot, + Name: nodesSnapshot, APIVersion: "v1", Kind: "Node", JqFilter: nodeJQFilter, @@ -65,26 +74,25 @@ var configDiscoveryService = &pkg.HookConfig{ Queue: fmt.Sprintf("modules/%s", common.MODULE_NAME), } -func handleVirtHandlerNodes(_ context.Context, input *pkg.HookInput) error { +func cleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { input.Logger.Info("Start.") - nodeMetadatas := input.Snapshots.Get(nodesMetadataSnapshot) - input.Logger.Info(fmt.Sprintf("Found %d nodes", len(nodeMetadatas))) - if len(nodeMetadatas) == 0 { + nodes := input.Snapshots.Get(nodesSnapshot) + if len(nodes) == 0 { return nil } - for _, nodeMetadata := range nodeMetadatas { - metadata := &metav1.ObjectMeta{} - if err := nodeMetadata.UnmarshalTo(metadata); err != nil { + for _, node := range nodes { + nodeInfo := &NodeInfo{} + if err := node.UnmarshalTo(nodeInfo); err != nil { input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err)) continue } patches := []map[string]interface{}{} - for key, _ := range metadata.Labels { - if strings.Contains(key, "virtualization.deckhouse.io") { + for key, _ := range nodeInfo.Labels { + if strings.Contains(key, labelPattern) { patches = append(patches, map[string]interface{}{ "op": "remove", "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), @@ -93,15 +101,13 @@ func handleVirtHandlerNodes(_ context.Context, input *pkg.HookInput) error { } if len(patches) == 0 { - input.Logger.Info("No labels found, nothing to do.") continue } else { - input.Logger.Info(fmt.Sprintf("Removing %d labels from node %s", len(patches), metadata.Name)) + input.Logger.Info(fmt.Sprintf("Removing %d labels contains %s from node %s", len(patches), labelPattern, nodeInfo.Name)) } - input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", metadata.Name) + input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", nodeInfo.Name) } - input.Logger.Info("Done.") return nil } From 919a8cc11b8579d7255453e78e957153dc7e2288 Mon Sep 17 00:00:00 2001 From: Pavel Tishkov Date: Thu, 12 Jun 2025 15:30:17 +0300 Subject: [PATCH 3/5] refactor Signed-off-by: Pavel Tishkov --- images/hooks/cmd/cleanup-labels/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go index e6e6e1da59..130aa4402e 100644 --- a/images/hooks/cmd/cleanup-labels/main.go +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -89,11 +89,11 @@ func cleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { continue } - patches := []map[string]interface{}{} + patches := make([]map[string]string, 0) for key, _ := range nodeInfo.Labels { if strings.Contains(key, labelPattern) { - patches = append(patches, map[string]interface{}{ + patches = append(patches, map[string]string{ "op": "remove", "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), }) From d2f48342abb1fec7f48bdfd9853cc92611b4fb4f Mon Sep 17 00:00:00 2001 From: Pavel Tishkov Date: Sat, 14 Jun 2025 18:22:16 +0300 Subject: [PATCH 4/5] add tests Signed-off-by: Pavel Tishkov --- images/hooks/cmd/cleanup-labels/main.go | 19 +-- images/hooks/cmd/cleanup-labels/main_test.go | 149 +++++++++++++++++++ 2 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 images/hooks/cmd/cleanup-labels/main_test.go diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go index 130aa4402e..0901a04ce8 100644 --- a/images/hooks/cmd/cleanup-labels/main.go +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -34,10 +34,11 @@ import ( ) const ( - nodesSnapshot = "virthandler-nodes" - virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" - labelPattern = "virtualization.deckhouse.io" - nodeJQFilter = `{ + nodesSnapshot = "virthandler-nodes" + virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" + labelPattern = "virtualization.deckhouse.io/" + logMessageTemplate = "Removing %d label(s) contains %s from node %s" + nodeJQFilter = `{ "name": .metadata.name, "labels": .metadata.labels, }` @@ -48,7 +49,7 @@ type NodeInfo struct { Labels map[string]string `json:"labels"` } -var _ = registry.RegisterFunc(configDiscoveryService, cleanUpNodeLabels) +var _ = registry.RegisterFunc(configDiscoveryService, handleCleanUpNodeLabels) var configDiscoveryService = &pkg.HookConfig{ OnAfterDeleteHelm: &pkg.OrderedConfig{Order: 5}, @@ -74,10 +75,10 @@ var configDiscoveryService = &pkg.HookConfig{ Queue: fmt.Sprintf("modules/%s", common.MODULE_NAME), } -func cleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { - input.Logger.Info("Start.") - +func handleCleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { nodes := input.Snapshots.Get(nodesSnapshot) + + input.Logger.Info(fmt.Sprintf("Number of nodes with label \"%s\": %d", virtHandlerLabel, len(nodes))) if len(nodes) == 0 { return nil } @@ -103,7 +104,7 @@ func cleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { if len(patches) == 0 { continue } else { - input.Logger.Info(fmt.Sprintf("Removing %d labels contains %s from node %s", len(patches), labelPattern, nodeInfo.Name)) + input.Logger.Info(fmt.Sprintf(logMessageTemplate, len(patches), labelPattern, nodeInfo.Name)) } input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", nodeInfo.Name) diff --git a/images/hooks/cmd/cleanup-labels/main_test.go b/images/hooks/cmd/cleanup-labels/main_test.go new file mode 100644 index 0000000000..75961f1704 --- /dev/null +++ b/images/hooks/cmd/cleanup-labels/main_test.go @@ -0,0 +1,149 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/testing/mock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func createSnapshotMock(nodeInfo NodeInfo) pkg.Snapshot { + m := mock.NewSnapshotMock(GinkgoT()) + m.UnmarshalToMock.Set(func(v any) error { + target, ok := v.(*NodeInfo) + if !ok { + return fmt.Errorf("expected *NodeInfo, got %T", v) + } + *target = nodeInfo + return nil + }) + return m +} + +func TestCleanUpVirtHandlerNodeLabels(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cleanup virtHandler node labels Suite") +} + +var _ = Describe("Cleanup virtHandler node labels", func() { + var ( + snapshots *mock.SnapshotsMock + values *mock.PatchableValuesCollectorMock + patchCollector *mock.PatchCollectorMock + input *pkg.HookInput + buf *bytes.Buffer + ) + + BeforeEach(func() { + snapshots = mock.NewSnapshotsMock(GinkgoT()) + values = mock.NewPatchableValuesCollectorMock(GinkgoT()) + patchCollector = mock.NewPatchCollectorMock(GinkgoT()) + + buf = bytes.NewBuffer([]byte{}) + + input = &pkg.HookInput{ + Values: values, + Snapshots: snapshots, + Logger: log.NewLogger(log.Options{ + Level: log.LevelDebug.Level(), + Output: buf, + TimeFunc: func(_ time.Time) time.Time { + parsedTime, err := time.Parse(time.DateTime, "2006-01-02 15:04:05") + Expect(err).ShouldNot(HaveOccurred()) + return parsedTime + }, + }), + PatchCollector: patchCollector, + } + }) + + Context("Empty cluster", func() { + It("Hook must execute successfully", func() { + snapshots.GetMock.When(nodesSnapshot).Then( + []pkg.Snapshot{}, + ) + err := handleCleanUpNodeLabels(context.Background(), input) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + Context("Nodes should be patched.", func() { + It("Hook must execute successfully", func() { + expectedNodes := map[string]interface{}{ + "node1": []map[string]string{ + map[string]string{ + "op": "remove", + "path": "/metadata/labels/kubevirt.internal.virtualization.deckhouse.io~1schedulable", + }, + }, + "node2": []map[string]string{ + { + "op": "remove", + "path": "/metadata/labels/kubevirt.internal.virtualization.deckhouse.io~1schedulable", + }, + map[string]string{ + "op": "remove", + "path": "/metadata/labels/virtualization.deckhouse.io~1kvm-enabled", + }, + }, + } + + snapshots.GetMock.When(nodesSnapshot).Then([]pkg.Snapshot{ + // should be patched + createSnapshotMock(NodeInfo{ + Name: "node1", + Labels: map[string]string{ + "kubevirt.internal.virtualization.deckhouse.io/schedulable": "true", + }, + }), + // should be patched + createSnapshotMock(NodeInfo{ + Name: "node2", + Labels: map[string]string{ + "kubevirt.internal.virtualization.deckhouse.io/schedulable": "true", + "virtualization.deckhouse.io/kvm-enabled": "true", + }, + }), + }) + + patchCollector.PatchWithJSONMock.Set(func(patch any, apiVersion, kind, namespace, name string, opts ...pkg.PatchCollectorOption) { + p, ok := patch.([]map[string]string) + Expect(ok).To(BeTrue()) + Expect(expectedNodes).To(HaveKey(name)) + Expect(p).To(Equal(expectedNodes[name])) + delete(expectedNodes, name) + }) + + err := handleCleanUpNodeLabels(context.Background(), input) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(buf.String()).To(ContainSubstring(fmt.Sprintf(logMessageTemplate, 1, labelPattern, "node1"))) + Expect(buf.String()).To(ContainSubstring(fmt.Sprintf(logMessageTemplate, 2, labelPattern, "node2"))) + + Expect(expectedNodes).To(HaveLen(0)) + }) + }) +}) From ba3d0fb91247f1d35fd66cbf0ab7007746e5e123 Mon Sep 17 00:00:00 2001 From: Pavel Tishkov Date: Sun, 15 Jun 2025 20:12:56 +0300 Subject: [PATCH 5/5] mvp Signed-off-by: Pavel Tishkov --- images/hooks/cmd/cleanup-labels/main.go | 60 ++++++++++++------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go index 0901a04ce8..e480d16bab 100644 --- a/images/hooks/cmd/cleanup-labels/main.go +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -26,11 +26,11 @@ import ( "github.com/deckhouse/module-sdk/pkg" "github.com/deckhouse/module-sdk/pkg/app" "github.com/deckhouse/module-sdk/pkg/registry" - "k8s.io/utils/ptr" "hooks/pkg/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" ) const ( @@ -49,9 +49,9 @@ type NodeInfo struct { Labels map[string]string `json:"labels"` } -var _ = registry.RegisterFunc(configDiscoveryService, handleCleanUpNodeLabels) +var _ = registry.RegisterFunc(configVirtHandlerNodeCleanUp, handleCleanUpNodeLabels) -var configDiscoveryService = &pkg.HookConfig{ +var configVirtHandlerNodeCleanUp = &pkg.HookConfig{ OnAfterDeleteHelm: &pkg.OrderedConfig{Order: 5}, Kubernetes: []pkg.KubernetesConfig{ { @@ -68,7 +68,7 @@ var configDiscoveryService = &pkg.HookConfig{ }, }, ExecuteHookOnSynchronization: ptr.To(false), - ExecuteHookOnEvents: ptr.To(false), + // ExecuteHookOnEvents: ptr.To(false), }, }, @@ -83,32 +83,32 @@ func handleCleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { return nil } - for _, node := range nodes { - nodeInfo := &NodeInfo{} - if err := node.UnmarshalTo(nodeInfo); err != nil { - input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err)) - continue - } - - patches := make([]map[string]string, 0) - - for key, _ := range nodeInfo.Labels { - if strings.Contains(key, labelPattern) { - patches = append(patches, map[string]string{ - "op": "remove", - "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), - }) - } - } - - if len(patches) == 0 { - continue - } else { - input.Logger.Info(fmt.Sprintf(logMessageTemplate, len(patches), labelPattern, nodeInfo.Name)) - } - - input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", nodeInfo.Name) - } + // for _, node := range nodes { + // nodeInfo := &NodeInfo{} + // if err := node.UnmarshalTo(nodeInfo); err != nil { + // input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err)) + // continue + // } + + // patches := make([]map[string]string, 0) + + // for key, _ := range nodeInfo.Labels { + // if strings.Contains(key, labelPattern) { + // patches = append(patches, map[string]string{ + // "op": "remove", + // "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), + // }) + // } + // } + + // if len(patches) == 0 { + // continue + // } else { + // input.Logger.Info(fmt.Sprintf(logMessageTemplate, len(patches), labelPattern, nodeInfo.Name)) + // } + + // input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", nodeInfo.Name) + // } return nil }