diff --git a/images/hooks/cmd/cleanup-labels/main.go b/images/hooks/cmd/cleanup-labels/main.go new file mode 100644 index 0000000000..e480d16bab --- /dev/null +++ b/images/hooks/cmd/cleanup-labels/main.go @@ -0,0 +1,123 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// The purpose of this hook is to prevent already launched virt-handler pods from flapping, since the node group configuration virtualization-detect-kvm.sh will be responsible for installing the label virtualization.deckhouse.io/kvm-enabled. + +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/pkg/app" + "github.com/deckhouse/module-sdk/pkg/registry" + + "hooks/pkg/common" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +const ( + nodesSnapshot = "virthandler-nodes" + virtHandlerLabel = "kubevirt.internal.virtualization.deckhouse.io/schedulable" + labelPattern = "virtualization.deckhouse.io/" + logMessageTemplate = "Removing %d label(s) contains %s from node %s" + nodeJQFilter = `{ + "name": .metadata.name, + "labels": .metadata.labels, + }` +) + +type NodeInfo struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` +} + +var _ = registry.RegisterFunc(configVirtHandlerNodeCleanUp, handleCleanUpNodeLabels) + +var configVirtHandlerNodeCleanUp = &pkg.HookConfig{ + OnAfterDeleteHelm: &pkg.OrderedConfig{Order: 5}, + Kubernetes: []pkg.KubernetesConfig{ + { + Name: nodesSnapshot, + APIVersion: "v1", + Kind: "Node", + JqFilter: nodeJQFilter, + LabelSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: virtHandlerLabel, + Operator: metav1.LabelSelectorOpExists, + }, + }, + }, + ExecuteHookOnSynchronization: ptr.To(false), + // ExecuteHookOnEvents: ptr.To(false), + }, + }, + + Queue: fmt.Sprintf("modules/%s", common.MODULE_NAME), +} + +func handleCleanUpNodeLabels(_ context.Context, input *pkg.HookInput) error { + nodes := input.Snapshots.Get(nodesSnapshot) + + input.Logger.Info(fmt.Sprintf("Number of nodes with label \"%s\": %d", virtHandlerLabel, len(nodes))) + if len(nodes) == 0 { + return nil + } + + // for _, node := range nodes { + // nodeInfo := &NodeInfo{} + // if err := node.UnmarshalTo(nodeInfo); err != nil { + // input.Logger.Error(fmt.Sprintf("Failed to unmarshal node metadata %v", err)) + // continue + // } + + // patches := make([]map[string]string, 0) + + // for key, _ := range nodeInfo.Labels { + // if strings.Contains(key, labelPattern) { + // patches = append(patches, map[string]string{ + // "op": "remove", + // "path": fmt.Sprintf("/metadata/labels/%s", jsonPatchEscape(key)), + // }) + // } + // } + + // if len(patches) == 0 { + // continue + // } else { + // input.Logger.Info(fmt.Sprintf(logMessageTemplate, len(patches), labelPattern, nodeInfo.Name)) + // } + + // input.PatchCollector.PatchWithJSON(patches, "v1", "Node", "", nodeInfo.Name) + // } + return nil +} + +func jsonPatchEscape(s string) string { + s = strings.ReplaceAll(s, "~", "~0") + s = strings.ReplaceAll(s, "/", "~1") + return s +} + +func main() { + app.Run() +} diff --git a/images/hooks/cmd/cleanup-labels/main_test.go b/images/hooks/cmd/cleanup-labels/main_test.go new file mode 100644 index 0000000000..75961f1704 --- /dev/null +++ b/images/hooks/cmd/cleanup-labels/main_test.go @@ -0,0 +1,149 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/testing/mock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func createSnapshotMock(nodeInfo NodeInfo) pkg.Snapshot { + m := mock.NewSnapshotMock(GinkgoT()) + m.UnmarshalToMock.Set(func(v any) error { + target, ok := v.(*NodeInfo) + if !ok { + return fmt.Errorf("expected *NodeInfo, got %T", v) + } + *target = nodeInfo + return nil + }) + return m +} + +func TestCleanUpVirtHandlerNodeLabels(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Cleanup virtHandler node labels Suite") +} + +var _ = Describe("Cleanup virtHandler node labels", func() { + var ( + snapshots *mock.SnapshotsMock + values *mock.PatchableValuesCollectorMock + patchCollector *mock.PatchCollectorMock + input *pkg.HookInput + buf *bytes.Buffer + ) + + BeforeEach(func() { + snapshots = mock.NewSnapshotsMock(GinkgoT()) + values = mock.NewPatchableValuesCollectorMock(GinkgoT()) + patchCollector = mock.NewPatchCollectorMock(GinkgoT()) + + buf = bytes.NewBuffer([]byte{}) + + input = &pkg.HookInput{ + Values: values, + Snapshots: snapshots, + Logger: log.NewLogger(log.Options{ + Level: log.LevelDebug.Level(), + Output: buf, + TimeFunc: func(_ time.Time) time.Time { + parsedTime, err := time.Parse(time.DateTime, "2006-01-02 15:04:05") + Expect(err).ShouldNot(HaveOccurred()) + return parsedTime + }, + }), + PatchCollector: patchCollector, + } + }) + + Context("Empty cluster", func() { + It("Hook must execute successfully", func() { + snapshots.GetMock.When(nodesSnapshot).Then( + []pkg.Snapshot{}, + ) + err := handleCleanUpNodeLabels(context.Background(), input) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + + Context("Nodes should be patched.", func() { + It("Hook must execute successfully", func() { + expectedNodes := map[string]interface{}{ + "node1": []map[string]string{ + map[string]string{ + "op": "remove", + "path": "/metadata/labels/kubevirt.internal.virtualization.deckhouse.io~1schedulable", + }, + }, + "node2": []map[string]string{ + { + "op": "remove", + "path": "/metadata/labels/kubevirt.internal.virtualization.deckhouse.io~1schedulable", + }, + map[string]string{ + "op": "remove", + "path": "/metadata/labels/virtualization.deckhouse.io~1kvm-enabled", + }, + }, + } + + snapshots.GetMock.When(nodesSnapshot).Then([]pkg.Snapshot{ + // should be patched + createSnapshotMock(NodeInfo{ + Name: "node1", + Labels: map[string]string{ + "kubevirt.internal.virtualization.deckhouse.io/schedulable": "true", + }, + }), + // should be patched + createSnapshotMock(NodeInfo{ + Name: "node2", + Labels: map[string]string{ + "kubevirt.internal.virtualization.deckhouse.io/schedulable": "true", + "virtualization.deckhouse.io/kvm-enabled": "true", + }, + }), + }) + + patchCollector.PatchWithJSONMock.Set(func(patch any, apiVersion, kind, namespace, name string, opts ...pkg.PatchCollectorOption) { + p, ok := patch.([]map[string]string) + Expect(ok).To(BeTrue()) + Expect(expectedNodes).To(HaveKey(name)) + Expect(p).To(Equal(expectedNodes[name])) + delete(expectedNodes, name) + }) + + err := handleCleanUpNodeLabels(context.Background(), input) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(buf.String()).To(ContainSubstring(fmt.Sprintf(logMessageTemplate, 1, labelPattern, "node1"))) + Expect(buf.String()).To(ContainSubstring(fmt.Sprintf(logMessageTemplate, 2, labelPattern, "node2"))) + + Expect(expectedNodes).To(HaveLen(0)) + }) + }) +}) diff --git a/images/hooks/werf.inc.yaml b/images/hooks/werf.inc.yaml index 5d13dd4b33..ccb6d9410f 100644 --- a/images/hooks/werf.inc.yaml +++ b/images/hooks/werf.inc.yaml @@ -35,3 +35,4 @@ shell: - go build -ldflags="-s -w" -o /hooks/generate-secret-for-dvcr ./cmd/generate-secret-for-dvcr - go build -ldflags="-s -w" -o /hooks/discovery-clusterip-service-for-dvcr ./cmd/discovery-clusterip-service-for-dvcr - go build -ldflags="-s -w" -o /hooks/discovery-workload-nodes ./cmd/discovery-workload-nodes + - go build -ldflags="-s -w" -o /hooks/cleanup-labels ./cmd/cleanup-labels