diff --git a/test/e2e/blockdevice/vd_snapshots.go b/test/e2e/blockdevice/vd_snapshots.go new file mode 100644 index 0000000000..ad54fa99ac --- /dev/null +++ b/test/e2e/blockdevice/vd_snapshots.go @@ -0,0 +1,309 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package blockdevice + +import ( + "context" + "fmt" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + vdbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vd" + vdsnapshotbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vdsnapshot" + vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" + "github.com/deckhouse/virtualization/test/e2e/internal/config" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/object" + "github.com/deckhouse/virtualization/test/e2e/internal/precheck" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +var _ = Describe("VirtualDiskSnapshots", Label(precheck.PrecheckImmediateStorageClass, precheck.PrecheckSnapshot), func() { + var ( + ctx context.Context + cfg *config.Config + ) + + BeforeEach(func() { + ctx = context.Background() + + cfg = framework.GetConfig() + if cfg.StorageClass.TemplateStorageClass != nil && cfg.StorageClass.TemplateStorageClass.Provisioner == config.NFS { + Skip("Concurrent snapshotting is not supported on NFS on the VolumeSnapshot side, skipping") + } + }) + + It("validates snapshot lifecycle for running VM's disk", func() { + f := framework.NewFramework("virtual-disk-snapshots-running-vm-disk") + f.Before() + DeferCleanup(f.After) + + By("Environment preparation") + vd := object.NewVDFromCVI("vd", f.Namespace().Name, object.PrecreatedCVIUbuntu) + vm := object.NewMinimalVM("vm-", f.Namespace().Name, + vmbuilder.WithName("vm"), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.VirtualDiskKind, + Name: vd.Name, + }), + ) + + err := f.CreateWithDeferredDeletion(ctx, vd, vm) + Expect(err).NotTo(HaveOccurred()) + + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(vm), framework.LongTimeout) + + By("Creating snapshot") + vdSnapshot := generateVDSnapshot("vdsnapshot", vd) + + err = f.CreateWithDeferredDeletion(ctx, vdSnapshot) + Expect(err).NotTo(HaveOccurred()) + ensureVMWasFrozen(ctx, f, vm, framework.MiddleTimeout) + + By("Waiting for ready snapshot phase") + util.UntilObjectPhase(ctx, string(v1alpha2.VirtualDiskSnapshotPhaseReady), framework.MiddleTimeout, vdSnapshot) + + By("Checking VirtualDiskSnapshot consistency") + checkVdSnapshotConsistentlyAndReadyToUse(ctx, f, vdSnapshot) + + By("Ensuring the virtual machine filesystem is unfrozen") + checkVMUnfrozen(ctx, f, vm) + + By("Ensuring the disk is attached to the VM") + util.UntilDisksAreAttachedInVMStatus(ctx, f, framework.ShortTimeout, vm, vd) + }) + + It("validates snapshots for a disk with no consumer", func() { + f := framework.NewFramework("virtual-disk-snapshots-no-consumer") + f.Before() + DeferCleanup(f.After) + + By("Environment preparation") + vd := object.NewVDFromCVI( + "vd-no-consumer", + f.Namespace().Name, + object.PrecreatedCVIAlpineBIOS, + vdbuilder.WithStorageClass(ptr.To(cfg.StorageClass.ImmediateStorageClass.Name)), + ) + + err := f.CreateWithDeferredDeletion(ctx, vd) + Expect(err).NotTo(HaveOccurred()) + + util.UntilObjectPhase(ctx, string(v1alpha2.DiskReady), framework.LongTimeout, vd) + + By("Creating snapshot") + vdSnapshot := generateVDSnapshot("vdsnapshot", vd) + + err = f.CreateWithDeferredDeletion(ctx, vdSnapshot) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for ready snapshot phase") + util.UntilObjectPhase(ctx, string(v1alpha2.VirtualDiskSnapshotPhaseReady), framework.MiddleTimeout, vdSnapshot) + + By("Checking VirtualDiskSnapshot consistency") + checkVdSnapshotConsistentlyAndReadyToUse(ctx, f, vdSnapshot) + }) + + It("validates snapshots for a hotplug scenario", func() { + f := framework.NewFramework("virtual-disk-snapshots-hotplug") + f.Before() + DeferCleanup(f.After) + + By("Environment preparation") + vdRoot := object.NewVDFromCVI("vd-root", f.Namespace().Name, object.PrecreatedCVIUbuntu) + vdAttach := object.NewBlankVD("vd-attach", f.Namespace().Name, nil, ptr.To(resource.MustParse("100Mi"))) + + vm := object.NewMinimalVM("vm-", f.Namespace().Name, + vmbuilder.WithName("vm-hotplug"), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.VirtualDiskKind, + Name: vdRoot.Name, + }), + ) + vmbda := object.NewVMBDAFromDisk("vd-attach-attachment", vm.Name, vdAttach) + + err := f.CreateWithDeferredDeletion(ctx, vdRoot, vdAttach, vm, vmbda) + Expect(err).NotTo(HaveOccurred()) + + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(vm), framework.LongTimeout) + util.UntilObjectPhase(ctx, string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.MiddleTimeout, vmbda) + + By("Creating snapshots") + vdSnapshotRoot := generateVDSnapshot("vdsnapshot-root", vdRoot) + vdSnapshotAttach := generateVDSnapshot("vdsnapshot-attach", vdAttach) + + err = f.CreateWithDeferredDeletion(ctx, vdSnapshotRoot, vdSnapshotAttach) + Expect(err).NotTo(HaveOccurred()) + ensureVMWasFrozen(ctx, f, vm, framework.MiddleTimeout) + + By("Waiting for ready snapshots phase") + util.UntilObjectPhase(ctx, string(v1alpha2.VirtualDiskSnapshotPhaseReady), framework.MiddleTimeout, vdSnapshotRoot, vdSnapshotAttach) + + By("Checking VirtualDiskSnapshots consistency") + checkVdSnapshotConsistentlyAndReadyToUse(ctx, f, vdSnapshotRoot) + checkVdSnapshotConsistentlyAndReadyToUse(ctx, f, vdSnapshotAttach) + + By("Ensuring the virtual machine filesystem is unfrozen") + checkVMUnfrozen(ctx, f, vm) + + By("Ensuring disks are attached to the VM") + util.UntilDisksAreAttachedInVMStatus(ctx, f, framework.ShortTimeout, vm, vdRoot, vdAttach) + }) + + It("validates concurrent snapshots", func() { + f := framework.NewFramework("virtual-disk-snapshots-concurrent") + f.Before() + DeferCleanup(f.After) + + By("Environment preparation") + vdRoot := object.NewVDFromCVI("vd-root", f.Namespace().Name, object.PrecreatedCVIUbuntu) + vdAttach := object.NewBlankVD("vd-attach", f.Namespace().Name, nil, ptr.To(resource.MustParse("100Mi"))) + + vm := object.NewMinimalVM("vm-", f.Namespace().Name, + vmbuilder.WithName("vm-concurrent"), + vmbuilder.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.VirtualDiskKind, + Name: vdRoot.Name, + }), + ) + vmbda := object.NewVMBDAFromDisk("vd-attach-attachment", vm.Name, vdAttach) + + err := f.CreateWithDeferredDeletion(ctx, vdRoot, vdAttach, vm, vmbda) + Expect(err).NotTo(HaveOccurred()) + + util.UntilVMAgentReady(ctx, crclient.ObjectKeyFromObject(vm), framework.LongTimeout) + util.UntilObjectPhase(ctx, string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.MiddleTimeout, vmbda) + + By("Creating snapshots") + vdSnapshots := concurentlyVDSnapshotsCreation(ctx, f, []*v1alpha2.VirtualDisk{vdRoot, vdAttach}, 5) + ensureVMWasFrozen(ctx, f, vm, framework.MiddleTimeout) + + By("Waiting for ready snapshots phase") + util.UntilObjectPhase(ctx, string(v1alpha2.VirtualDiskSnapshotPhaseReady), framework.MiddleTimeout, util.ToObjects(vdSnapshots)...) + + By("Checking VirtualDiskSnapshots consistency") + for _, vdSnapshot := range vdSnapshots { + checkVdSnapshotConsistentlyAndReadyToUse(ctx, f, vdSnapshot) + } + + By("Ensuring the virtual machine filesystem is unfrozen") + checkVMUnfrozen(ctx, f, vm) + + By("Ensuring disks are attached to the VM") + util.UntilDisksAreAttachedInVMStatus(ctx, f, framework.ShortTimeout, vm, vdRoot, vdAttach) + }) +}) + +func checkVdSnapshotConsistentlyAndReadyToUse(ctx context.Context, f *framework.Framework, vdSnapshot *v1alpha2.VirtualDiskSnapshot) { + GinkgoHelper() + + key := crclient.ObjectKeyFromObject(vdSnapshot) + actualVDSnapshot := &v1alpha2.VirtualDiskSnapshot{} + err := f.GenericClient().Get(ctx, key, actualVDSnapshot) + Expect(err).NotTo(HaveOccurred()) + + Expect(actualVDSnapshot.Status.Consistent).NotTo(BeNil(), "VirtualDiskSnapshot status.consistent must be set") + Expect(*actualVDSnapshot.Status.Consistent).To(BeTrue(), "VirtualDiskSnapshot status.consistent must be true") + Expect(actualVDSnapshot.Status.VolumeSnapshotName).NotTo(BeEmpty(), "VirtualDiskSnapshot status.volumeSnapshotName must be set") +} + +func ensureVMWasFrozen(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, timeout time.Duration) { + GinkgoHelper() + + Eventually(func() error { + var currentVM v1alpha2.VirtualMachine + err := f.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(vm), ¤tVM) + if err != nil { + return err + } + + frozenCondition, ok := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, currentVM.Status.Conditions) + if !ok { + return fmt.Errorf("filesystem frozen condition not found") + } + if frozenCondition.Status != metav1.ConditionTrue { + return fmt.Errorf("filesystem frozen condition is not true") + } + + return nil + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +} + +func generateVDSnapshot(name string, vd *v1alpha2.VirtualDisk) *v1alpha2.VirtualDiskSnapshot { + return vdsnapshotbuilder.New( + vdsnapshotbuilder.WithName(name), + vdsnapshotbuilder.WithNamespace(vd.Namespace), + vdsnapshotbuilder.WithVirtualDiskName(vd.Name), + vdsnapshotbuilder.WithRequiredConsistency(true), + ) +} + +func checkVMUnfrozen(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine) { + GinkgoHelper() + + var currentVM v1alpha2.VirtualMachine + err := f.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(vm), ¤tVM) + Expect(err).NotTo(HaveOccurred()) + + _, frozenConditionExists := conditions.GetCondition(vmcondition.TypeFilesystemFrozen, currentVM.Status.Conditions) + Expect(frozenConditionExists).To(BeFalse(), "frozen condition must not exist") +} + +func concurentlyVDSnapshotsCreation(ctx context.Context, f *framework.Framework, vds []*v1alpha2.VirtualDisk, cnt int) []*v1alpha2.VirtualDiskSnapshot { + GinkgoHelper() + + var vdSnapshots []*v1alpha2.VirtualDiskSnapshot + + for i := 1; i <= cnt; i++ { + for _, vd := range vds { + vdSnapshots = append(vdSnapshots, generateVDSnapshot( + fmt.Sprintf("vdsnapshot-%s-%d", vd.Name, i), + vd, + )) + } + } + + var wg sync.WaitGroup + wg.Add(len(vds) * cnt) + + for _, vdSnapshot := range vdSnapshots { + go func(vdSnapshot *v1alpha2.VirtualDiskSnapshot) { + GinkgoRecover() + defer wg.Done() + err := f.GenericClient().Create(ctx, vdSnapshot) + Expect(err).NotTo(HaveOccurred()) + }(vdSnapshot) + } + + wg.Wait() + + for _, vdSnapshot := range vdSnapshots { + f.DeferDelete(vdSnapshot) + } + + return vdSnapshots +} diff --git a/test/e2e/vm/virtual_disk_resizing.go b/test/e2e/blockdevice/virtual_disk_resizing.go similarity index 90% rename from test/e2e/vm/virtual_disk_resizing.go rename to test/e2e/blockdevice/virtual_disk_resizing.go index 019e803309..d2c0620a04 100644 --- a/test/e2e/vm/virtual_disk_resizing.go +++ b/test/e2e/blockdevice/virtual_disk_resizing.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package vm +package blockdevice import ( "context" @@ -92,6 +92,7 @@ var _ = Describe("VirtualDiskResizing", Label(precheck.NoPrecheck), func() { var vdWasResizing atomic.Bool go func() { + GinkgoRecover() wasResizing, err := ensureVDWasResizing( ctxVDWatch, f.VirtClient().VirtualDisks(f.Namespace().Name), @@ -137,7 +138,7 @@ var _ = Describe("VirtualDiskResizing", Label(precheck.NoPrecheck), func() { Expect(newVDBlankLsblkSize.Cmp(vdBlankLsblkSize)).To(Equal(common.CmpGreater)) Expect(newVDAttachLsblkSize.Cmp(vdAttachLsblkSize)).To(Equal(common.CmpGreater)) - untilDisksArePresentAndAttachedInVMStatus(ctx, f, framework.ShortTimeout, vm, vdRoot, vdBlank, vdAttach) + util.UntilDisksAreAttachedInVMStatus(ctx, f, framework.ShortTimeout, vm, vdRoot, vdBlank, vdAttach) }) }) @@ -221,23 +222,3 @@ func ensureVDWasResizing(ctx context.Context, w util.Watcher, vds []*v1alpha2.Vi } } } - -func untilDisksArePresentAndAttachedInVMStatus(ctx context.Context, f *framework.Framework, timeout time.Duration, vm *v1alpha2.VirtualMachine, vds ...*v1alpha2.VirtualDisk) { - GinkgoHelper() - - Eventually(func(g Gomega) { - err := f.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(vm), vm) - g.Expect(err).NotTo(HaveOccurred()) - - statusActiveDisks := make(map[string]struct{}) - for _, bd := range vm.Status.BlockDeviceRefs { - if bd.Kind == v1alpha2.VirtualDiskKind && bd.Attached { - statusActiveDisks[bd.Name] = struct{}{} - } - } - - for _, vd := range vds { - g.Expect(statusActiveDisks).To(HaveKey(vd.Name)) - } - }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) -} diff --git a/test/e2e/default_config.yaml b/test/e2e/default_config.yaml index a5a34725d0..78b93d7e75 100644 --- a/test/e2e/default_config.yaml +++ b/test/e2e/default_config.yaml @@ -22,7 +22,6 @@ testData: sizingPolicy: "/tmp/testdata/sizing-policy" vmConfiguration: "/tmp/testdata/vm-configuration" vmMigration: "/tmp/testdata/vm-migration" - vdSnapshots: "/tmp/testdata/vd-snapshots" sshKey: "/tmp/testdata/sshkeys/id_ed" sshUser: "cloud" logFilter: diff --git a/test/e2e/internal/config/config.go b/test/e2e/internal/config/config.go index 7b51145108..feb5141fd0 100644 --- a/test/e2e/internal/config/config.go +++ b/test/e2e/internal/config/config.go @@ -90,7 +90,6 @@ type Config struct { type TestData struct { ImageHotplug string `yaml:"imageHotplug"` VMMigration string `yaml:"vmMigration"` - VdSnapshots string `yaml:"vdSnapshots"` Sshkey string `yaml:"sshKey"` SSHUser string `yaml:"sshUser"` } diff --git a/test/e2e/internal/util/block_device.go b/test/e2e/internal/util/block_device.go index 496628bb04..ddd2c14d0b 100644 --- a/test/e2e/internal/util/block_device.go +++ b/test/e2e/internal/util/block_device.go @@ -40,10 +40,10 @@ func GetBlockDevicePath(ctx context.Context, f *framework.Framework, vm *v1alpha GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) devicePath, err := GetBlockDeviceBySerial(f, vm, serial) - Expect(err).NotTo(HaveOccurred(), "failed to get device by serial") + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get device %s/%s by serial", bdKind, bdName)) return devicePath } @@ -51,58 +51,58 @@ func CreateBlockDeviceFilesystem(ctx context.Context, f *framework.Framework, vm GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) devicePath, err := GetBlockDeviceBySerial(f, vm, serial) - Expect(err).NotTo(HaveOccurred(), "failed to get device by serial") + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get device %s/%s by serial", bdKind, bdName)) _, err = f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo mkfs.%s %s", fsType, devicePath)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to create %s filesystem on block device %s/%s", fsType, bdKind, bdName)) } func MountBlockDevice(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, bdKind v1alpha2.BlockDeviceKind, bdName, mountPoint string) { GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) devicePath, err := GetBlockDeviceBySerial(f, vm, serial) - Expect(err).NotTo(HaveOccurred(), "failed to get device by serial") + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get device %s/%s by serial", bdKind, bdName)) _, err = f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo mount %s %s", devicePath, mountPoint)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to mount block device %s/%s to %s", bdKind, bdName, mountPoint)) } func UnmountBlockDevice(f *framework.Framework, vm *v1alpha2.VirtualMachine, mountPoint string) { GinkgoHelper() _, err := f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo umount %s", mountPoint)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to unmount %s", mountPoint)) } func RegisterFstabEntry(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, bdKind v1alpha2.BlockDeviceKind, bdName string) { GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) cmd := fmt.Sprintf(`UUID=$(lsblk -o SERIAL,UUID | grep %s | awk "{print \$2}"); echo "UUID=$UUID /mnt ext4 defaults 0 0" | sudo tee -a /etc/fstab`, serial) _, err := f.SSHCommand(vm.Name, vm.Namespace, cmd) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to register fstab entry for block device %s/%s", bdKind, bdName)) } func GetBlockDeviceHash(ctx context.Context, f *framework.Framework, vm *v1alpha2.VirtualMachine, bdKind v1alpha2.BlockDeviceKind, bdName string) string { GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) devicePath, err := GetBlockDeviceBySerial(f, vm, serial) - Expect(err).NotTo(HaveOccurred(), "failed to get device by serial") + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get device %s/%s by serial", bdKind, bdName)) // We use dd to ensure the entire disk is read. cmdOut, err := f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo dd if=%s bs=4M | sha256sum | awk \"{print \\$1}\"", devicePath)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get hash for block device %s/%s", bdKind, bdName)) return strings.TrimSpace(cmdOut) } @@ -110,13 +110,13 @@ func GetBlockDeviceLsblkSize(ctx context.Context, f *framework.Framework, vm *v1 GinkgoHelper() serial, ok := GetBlockDeviceSerialNumber(ctx, vm, bdKind, bdName) - Expect(ok).To(BeTrue(), "failed to get block device serial number") + Expect(ok).To(BeTrue(), fmt.Sprintf("failed to get block device %s/%s serial number", bdKind, bdName)) devicePath, err := GetBlockDeviceBySerial(f, vm, serial) - Expect(err).NotTo(HaveOccurred(), "failed to get device by serial") + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get device %s/%s by serial", bdKind, bdName)) cmdOut, err := f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo lsblk --json -o SIZE %s", devicePath)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get lsblk size for block device %s/%s", bdKind, bdName)) var disks Disks err = json.Unmarshal([]byte(cmdOut), &disks) @@ -155,11 +155,11 @@ func GetBlockDeviceSerialNumber(ctx context.Context, vm *v1alpha2.VirtualMachine Version: "v1", Resource: "internalvirtualizationvirtualmachineinstances", }).Namespace(vm.Namespace).Get(ctx, vm.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to get InternalVirtualizationVirtualMachineInstance %s/%s", vm.Namespace, vm.Name)) var kvvmi virtv1.VirtualMachineInstance err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredVMI.Object, &kvvmi) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to convert InternalVirtualizationVirtualMachineInstance %s/%s to kubevirt VMI", vm.Namespace, vm.Name)) var blockDeviceName string switch bdKind { @@ -188,14 +188,14 @@ func WriteFile(f *framework.Framework, vm *v1alpha2.VirtualMachine, path, value // Escape single quotes in value to prevent command injection. escapedValue := strings.ReplaceAll(value, "'", "'\"'\"'") _, err := f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo bash -c \"echo '%s' > %s\"", escapedValue, path)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to write file %s on vm %s/%s", path, vm.Namespace, vm.Name)) } func ReadFile(f *framework.Framework, vm *v1alpha2.VirtualMachine, path string) string { GinkgoHelper() cmdOut, err := f.SSHCommand(vm.Name, vm.Namespace, fmt.Sprintf("sudo cat %s", path)) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to read file %s on vm %s/%s", path, vm.Namespace, vm.Name)) return strings.TrimSpace(cmdOut) } diff --git a/test/e2e/internal/util/util.go b/test/e2e/internal/util/util.go index c177c6ff36..699535e07e 100644 --- a/test/e2e/internal/util/util.go +++ b/test/e2e/internal/util/util.go @@ -50,3 +50,11 @@ func UnmarshalResource(filePath string, obj client.Object) error { return nil } + +func ToObjects[T client.Object](objs []T) []client.Object { + out := make([]client.Object, len(objs)) + for i, o := range objs { + out[i] = o + } + return out +} diff --git a/test/e2e/internal/util/vm.go b/test/e2e/internal/util/vm.go index edb31193d0..f8e8a3218b 100644 --- a/test/e2e/internal/util/vm.go +++ b/test/e2e/internal/util/vm.go @@ -246,6 +246,25 @@ func UntilVMMigrationSucceeded(key client.ObjectKey, timeout time.Duration) { }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) } +func UntilDisksAreAttachedInVMStatus( + ctx context.Context, + f *framework.Framework, + timeout time.Duration, + vm *v1alpha2.VirtualMachine, + vds ...*v1alpha2.VirtualDisk, +) { + GinkgoHelper() + + Eventually(func(g Gomega) { + err := f.GenericClient().Get(ctx, client.ObjectKeyFromObject(vm), vm) + g.Expect(err).NotTo(HaveOccurred()) + + for _, vd := range vds { + g.Expect(IsVDAttached(vm, vd)).To(BeTrue()) + } + }).WithTimeout(timeout).WithPolling(time.Second).Should(Succeed()) +} + func MigrateVirtualMachine(f *framework.Framework, vm *v1alpha2.VirtualMachine, options ...vmopbuilder.Option) { GinkgoHelper() diff --git a/test/e2e/legacy/testdata/vd-snapshots/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/kustomization.yaml deleted file mode 100644 index 724e6e1681..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: testcases -namePrefix: commit- -resources: - - ns.yaml - - vd - - vm -configurations: - - transformer.yaml -labels: - - includeSelectors: true - pairs: - id: commit - testcase: vd-snapshots diff --git a/test/e2e/legacy/testdata/vd-snapshots/ns.yaml b/test/e2e/legacy/testdata/vd-snapshots/ns.yaml deleted file mode 100644 index 5efde875b6..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/ns.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: default diff --git a/test/e2e/legacy/testdata/vd-snapshots/transformer.yaml b/test/e2e/legacy/testdata/vd-snapshots/transformer.yaml deleted file mode 100644 index ec70d37fcd..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/transformer.yaml +++ /dev/null @@ -1,52 +0,0 @@ -namespace: - - kind: ClusterVirtualImage - path: spec/dataSource/objectRef/namespace -nameReference: - - kind: VirtualImage - version: v1alpha2 # optional - fieldSpecs: - - path: spec/dataSource/objectRef/name - kind: ClusterVirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualDisk - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - kind: ClusterVirtualImage - version: v1alpha2 # optional - fieldSpecs: - - path: spec/dataSource/objectRef/name - kind: ClusterVirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualDisk - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - kind: VirtualDisk - version: v1alpha2 # optional - fieldSpecs: - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - path: spec/blockDeviceRef/name - kind: VirtualMachineBlockDeviceAttachment - - kind: Secret - fieldSpecs: - - path: spec/provisioning/userDataRef/name - kind: VirtualMachine - - kind: VirtualMachineIPAddress - version: v1alpha2 - fieldSpecs: - - path: spec/virtualMachineIPAddressName - kind: VirtualMachine - - kind: VirtualMachine - version: v1alpha2 - fieldSpecs: - - path: spec/virtualMachineName - kind: VirtualMachineBlockDeviceAttachment - - kind: VirtualMachineClass - version: v1alpha3 - fieldSpecs: - - path: spec/virtualMachineClassName - kind: VirtualMachine diff --git a/test/e2e/legacy/testdata/vd-snapshots/vd/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/vd/kustomization.yaml deleted file mode 100644 index 7b38a7f143..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vd/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - vd-alpine-http.yaml -labels: - - includeSelectors: true - pairs: - hasNoConsumer: "vd-snapshots" diff --git a/test/e2e/legacy/testdata/vd-snapshots/vd/vd-alpine-http.yaml b/test/e2e/legacy/testdata/vd-snapshots/vd/vd-alpine-http.yaml deleted file mode 100644 index 1e06d9d929..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vd/vd-alpine-http.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: virtualization.deckhouse.io/v1alpha2 -kind: VirtualDisk -metadata: - name: vd-alpine-http -spec: - dataSource: - type: ObjectRef - objectRef: - kind: ClusterVirtualImage - name: v12n-e2e-alpine-bios - persistentVolumeClaim: - storageClassName: "{{ .STORAGE_CLASS_NAME }}" - size: 350Mi diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml deleted file mode 100644 index 2ec8f0c999..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/base/cfg/cloudinit.yaml +++ /dev/null @@ -1,23 +0,0 @@ -#cloud-config -package_update: true -packages: - - qemu-guest-agent - - curl - - bash - - sudo - - iputils -users: - - name: cloud - # passwd: cloud - passwd: $6$rounds=4096$vln/.aPHBOI7BMYR$bBMkqQvuGs5Gyd/1H5DP4m9HjQSy.kgrxpaGEHwkX7KEFV8BS.HZWPitAtZ2Vd8ZqIZRqmlykRCagTgPejt1i. - shell: /bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - lock_passwd: false - ssh_authorized_keys: - # testcases - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFxcXHmwaGnJ8scJaEN5RzklBPZpVSic4GdaAsKjQoeA your_email@example.com -final_message: "\U0001F525\U0001F525\U0001F525 The system is finally up, after ${updame} \U0001F525\U0001F525\U0001F525" -runcmd: - - "echo \"\U0001F7E1 Starting runcmd at $(date +%H:%M:%S)\"" - - "rc-update add qemu-guest-agent && rc-service qemu-guest-agent start" - - "echo \"\U0001F7E1 Finished runcmd at $(date +%H:%M:%S)\"" diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/base/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/base/kustomization.yaml deleted file mode 100644 index d820a3f566..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/base/kustomization.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ./vm.yaml - - ./vd-root.yaml -configurations: - - transformer.yaml -generatorOptions: - disableNameSuffixHash: true -secretGenerator: - - files: - - userData=cfg/cloudinit.yaml - name: cloud-init - type: provisioning.virtualization.deckhouse.io/cloud-init diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/base/transformer.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/base/transformer.yaml deleted file mode 100644 index 1dc146a3af..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/base/transformer.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# https://github.com/kubernetes-sigs/kustomize/blob/master/examples/transformerconfigs/README.md#transformer-configurations - -namespace: - - kind: ClusterVirtualImage - path: spec/dataSource/objectRef/namespace -nameReference: - - kind: VirtualImage - version: v1alpha2 # optional - fieldSpecs: - - path: spec/dataSource/objectRef/name - kind: ClusterVirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualDisk - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - kind: ClusterVirtualImage - version: v1alpha2 # optional - fieldSpecs: - - path: spec/dataSource/objectRef/name - kind: ClusterVirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualImage - - path: spec/dataSource/objectRef/name - kind: VirtualDisk - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - kind: VirtualDisk - version: v1alpha2 # optional - fieldSpecs: - - path: spec/blockDeviceRefs/name - kind: VirtualMachine - - path: spec/blockDeviceRef/name - kind: VirtualMachineBlockDeviceAttachment - - kind: Secret - fieldSpecs: - - path: spec/provisioning/userDataRef/name - kind: VirtualMachine - - kind: VirtualMachineIPAddress - version: v1alpha2 - fieldSpecs: - - path: spec/virtualMachineIPAddressName - kind: VirtualMachine - - kind: VirtualMachine - version: v1alpha2 - fieldSpecs: - - path: spec/virtualMachineName - kind: VirtualMachineBlockDeviceAttachment - - kind: VirtualMachineClass - version: v1alpha3 - fieldSpecs: - - path: spec/virtualMachineClassName - kind: VirtualMachine diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/base/vd-root.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/base/vd-root.yaml deleted file mode 100644 index 7442dd263a..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/base/vd-root.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: virtualization.deckhouse.io/v1alpha2 -kind: VirtualDisk -metadata: - name: vd-root -spec: - persistentVolumeClaim: - storageClassName: "{{ .STORAGE_CLASS_NAME }}" - size: 350Mi - dataSource: - type: ObjectRef - objectRef: - kind: ClusterVirtualImage - name: v12n-e2e-alpine-uefi diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/base/vm.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/base/vm.yaml deleted file mode 100644 index 8dac088b6b..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/base/vm.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: virtualization.deckhouse.io/v1alpha2 -kind: VirtualMachine -metadata: - name: vm -spec: - bootloader: EFI - virtualMachineClassName: generic - cpu: - cores: 1 - coreFraction: 50% - memory: - size: 256Mi - disruptions: - restartApprovalMode: Manual - provisioning: - type: UserDataRef - userDataRef: - kind: Secret - name: cloud-init - blockDeviceRefs: - - kind: VirtualDisk - name: vd-root diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/kustomization.yaml deleted file mode 100644 index ada7106a49..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - overlays/automatic - - overlays/automatic-with-hotplug diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml deleted file mode 100644 index e2fb6a2e8e..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/kustomization.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -nameSuffix: -automatic-with-hotplug -resources: - - ../../base - - ./vd-attach.yaml - - ./vmbda.yaml -patches: - - patch: |- - - op: replace - path: /spec/runPolicy - value: AlwaysOn - target: - kind: VirtualMachine - name: vm - - patch: |- - - op: replace - path: /spec/disruptions/restartApprovalMode - value: Automatic - target: - kind: VirtualMachine - name: vm -labels: - - includeSelectors: true - pairs: - vm: automatic-with-hotplug diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml deleted file mode 100644 index a68911289e..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vd-attach.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: virtualization.deckhouse.io/v1alpha2 -kind: VirtualDisk -metadata: - name: vd-attach -spec: - persistentVolumeClaim: - storageClassName: "{{ .STORAGE_CLASS_NAME }}" - size: 100Mi diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml deleted file mode 100644 index 30a7b6ba4a..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic-with-hotplug/vmbda.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: virtualization.deckhouse.io/v1alpha2 -kind: VirtualMachineBlockDeviceAttachment -metadata: - name: blank-disk-attachment -spec: - virtualMachineName: vm - blockDeviceRef: - kind: VirtualDisk - name: vd-attach diff --git a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml b/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml deleted file mode 100644 index 37f7a7cf08..0000000000 --- a/test/e2e/legacy/testdata/vd-snapshots/vm/overlays/automatic/kustomization.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -nameSuffix: -automatic -resources: - - ../../base -patches: - - patch: |- - - op: replace - path: /spec/runPolicy - value: AlwaysOn - target: - kind: VirtualMachine - name: vm - - patch: |- - - op: replace - path: /spec/disruptions/restartApprovalMode - value: Automatic - target: - kind: VirtualMachine - name: vm -labels: - - includeSelectors: true - pairs: - vm: automatic diff --git a/test/e2e/legacy/vd_snapshots.go b/test/e2e/legacy/vd_snapshots.go deleted file mode 100644 index a4e3caef76..0000000000 --- a/test/e2e/legacy/vd_snapshots.go +++ /dev/null @@ -1,461 +0,0 @@ -/* -Copyright 2024 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package legacy - -import ( - "errors" - "fmt" - "maps" - "strings" - "sync" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - - "github.com/deckhouse/virtualization/api/core/v1alpha2" - "github.com/deckhouse/virtualization/api/core/v1alpha2/vmcondition" - "github.com/deckhouse/virtualization/test/e2e/internal/config" - kc "github.com/deckhouse/virtualization/test/e2e/internal/kubectl" - "github.com/deckhouse/virtualization/test/e2e/internal/label" - "github.com/deckhouse/virtualization/test/e2e/internal/precheck" - "github.com/deckhouse/virtualization/test/e2e/internal/util" -) - -const ( - filesystemReadyTimeout = 60 * time.Second - filesystemReadyPollingInterval = 5 * time.Second - frozenReasonPollingInterval = 1 * time.Second -) - -var _ = Describe("VirtualDiskSnapshots", Ordered, label.Legacy(), Label(precheck.PrecheckImmediateStorageClass, precheck.PrecheckSnapshot), func() { - var ( - testCaseLabel map[string]string - attachedVirtualDiskLabel = map[string]string{"attachedVirtualDisk": ""} - hasNoConsumerLabel = map[string]string{"hasNoConsumer": "vd-snapshots"} - vmAutomaticWithHotplug = map[string]string{"vm": "automatic-with-hotplug"} - ns string - ) - - BeforeAll(func() { - // Initialize testCaseLabel after Init() has set namePrefix - testCaseLabel = map[string]string{"testcase": "vd-snapshots", "id": namePrefix} - - if conf.StorageClass.TemplateStorageClass != nil && conf.StorageClass.TemplateStorageClass.Provisioner == config.NFS { - Skip("Concurrent snapshotting is not supported on NFS on the volumesnapshot side, skipping") - } - - kustomization := fmt.Sprintf("%s/%s", conf.TestData.VdSnapshots, "kustomization.yaml") - var err error - ns, err = kustomize.GetNamespace(kustomization) - Expect(err).NotTo(HaveOccurred(), "%w", err) - - Expect(conf.StorageClass.ImmediateStorageClass).NotTo(BeNil(), "immediate storage class cannot be nil; please set up the immediate storage class in the cluster") - setDiskImmediateStorageClass() - - CreateNamespace(ns) - }) - - AfterEach(func() { - if CurrentSpecReport().Failed() { - SaveTestCaseDump(testCaseLabel, CurrentSpecReport().LeafNodeText, ns) - } - }) - - AfterAll(func() { - if conf.IsCleanupNeeded { - DeleteTestCaseResources(ns, ResourcesToDelete{ - KustomizationDir: conf.TestData.VdSnapshots, - }) - } - }) - - Context("When virtualization resources are applied:", func() { - It("result should be succeeded", func() { - res := kubectl.Apply(kc.ApplyOptions{ - Filename: []string{conf.TestData.VdSnapshots}, - FilenameOption: kc.Kustomize, - }) - Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) - }) - }) - - Context("When virtual disks are applied:", func() { - It("checks VDs phases", func() { - By(fmt.Sprintf("VDs should be in %s phases", PhaseReady)) - waitOpts := kc.WaitOptions{ - Labels: testCaseLabel, - Namespace: ns, - Timeout: MaxWaitTimeout, - } - WaitPhaseByLabel(kc.ResourceVD, PhaseReady, waitOpts) - }) - }) - - Context("When virtual machines are applied:", func() { - It("checks VMs phases", func() { - By("Virtual machine agents should be ready") - WaitVMAgentReady(kc.WaitOptions{ - Labels: testCaseLabel, - Namespace: ns, - Timeout: MaxWaitTimeout, - }) - }) - }) - - Context("When virtual machine block device attachments are applied:", func() { - It("checks VMBDAs phases", func() { - By(fmt.Sprintf("VMBDAs should be in %s phases", PhaseAttached)) - WaitPhaseByLabel(kc.ResourceVMBDA, PhaseAttached, kc.WaitOptions{ - Labels: testCaseLabel, - Namespace: ns, - Timeout: MaxWaitTimeout, - }) - }) - }) - - Context(fmt.Sprintf("When unattached VDs in phase %s:", PhaseReady), func() { - It("creates VDs snapshots with `requiredConsistency`", func() { - res := kubectl.List(kc.ResourceVD, kc.GetOptions{ - Labels: hasNoConsumerLabel, - Namespace: ns, - Output: "jsonpath='{.items[*].metadata.name}'", - }) - Expect(res.Error()).NotTo(HaveOccurred(), "cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()) - - vds := strings.Split(res.StdOut(), " ") - - for _, vdName := range vds { - By(fmt.Sprintf("Create snapshot for %q", vdName)) - labels := make(map[string]string) - maps.Copy(labels, hasNoConsumerLabel) - maps.Copy(labels, testCaseLabel) - err := CreateVirtualDiskSnapshot(vdName, vdName, ns, true, labels) - Expect(err).NotTo(HaveOccurred(), "%s", err) - } - }) - - It("checks snapshots of unattached VDs", func() { - By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady)) - WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{ - Labels: hasNoConsumerLabel, - Namespace: ns, - Timeout: MaxWaitTimeout, - }) - - By("Snapshots should be consistent", func() { - vdSnapshots := v1alpha2.VirtualDiskSnapshotList{} - err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{Namespace: ns, Labels: hasNoConsumerLabel}) - Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err) - - for _, snapshot := range vdSnapshots.Items { - Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name) - } - }) - }) - }) - - Context(fmt.Sprintf("When virtual machines in %s phase", PhaseRunning), func() { - It("creates snapshots with `requiredConsistency` of attached VDs", func() { - vmObjects := v1alpha2.VirtualMachineList{} - err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{Namespace: ns}) - Expect(err).NotTo(HaveOccurred(), "cannot get virtual machines\nstderr: %s", err) - - for _, vm := range vmObjects.Items { - Eventually(func() error { - frozen, err := CheckFileSystemFrozen(vm.Name, ns) - if frozen { - return errors.New("file system of the Virtual Machine is frozen") - } - return err - }).WithTimeout( - filesystemReadyTimeout, - ).WithPolling( - filesystemReadyPollingInterval, - ).Should(Succeed()) - - blockDevices := vm.Status.BlockDeviceRefs - for _, blockDevice := range blockDevices { - if blockDevice.Kind == v1alpha2.VirtualDiskKind { - By(fmt.Sprintf("Create snapshot for %q", blockDevice.Name)) - labels := make(map[string]string) - maps.Copy(labels, attachedVirtualDiskLabel) - maps.Copy(labels, testCaseLabel) - err := CreateVirtualDiskSnapshot(blockDevice.Name, blockDevice.Name, ns, true, labels) - Expect(err).NotTo(HaveOccurred(), "%s", err) - - Eventually(func() error { - frozen, err := CheckFileSystemFrozen(vm.Name, ns) - if !frozen { - return fmt.Errorf("`Filesystem` should be frozen when controller is snapshotting the attached virtual disk") - } - return err - }).WithTimeout( - filesystemReadyTimeout, - ).WithPolling( - frozenReasonPollingInterval, - ).Should(Succeed()) - } - } - } - }) - - It("creates `vdSnapshots` concurrently", func() { - vmObjects := v1alpha2.VirtualMachineList{} - err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{ - Namespace: ns, - Labels: vmAutomaticWithHotplug, - }) - Expect(err).NotTo(HaveOccurred(), "cannot get vmObject with label %q\nstderr: %s", vmAutomaticWithHotplug, err) - - for _, vm := range vmObjects.Items { - Eventually(func() error { - frozen, err := CheckFileSystemFrozen(vm.Name, ns) - if frozen { - return errors.New("filesystem of the Virtual Machine is frozen") - } - return err - }).WithTimeout( - filesystemReadyTimeout, - ).WithPolling( - filesystemReadyPollingInterval, - ).Should(Succeed()) - - blockDevices := vm.Status.BlockDeviceRefs - for _, blockDevice := range blockDevices { - if blockDevice.Kind == v1alpha2.VirtualDiskKind { - By(fmt.Sprintf("Create five snapshots for %q of %q", blockDevice.Name, vm.Name)) - errs := make([]error, 0, 5) - wg := sync.WaitGroup{} - for i := range 5 { - wg.Add(1) - go func(index int) { - defer wg.Done() - snapshotName := fmt.Sprintf("%s-%d", blockDevice.Name, index) - - labels := make(map[string]string) - maps.Copy(labels, attachedVirtualDiskLabel) - maps.Copy(labels, testCaseLabel) - err := CreateVirtualDiskSnapshot(blockDevice.Name, snapshotName, ns, true, labels) - if err != nil { - errs = append(errs, err) - } - }(i) - } - wg.Wait() - Expect(errs).To(BeEmpty(), "should not face concurrent snapshotting error") - - Eventually(func() error { - frozen, err := CheckFileSystemFrozen(vm.Name, ns) - if !frozen { - return fmt.Errorf("`Filesystem` should be frozen when controller is snapshotting the attached virtual disk") - } - return err - }).WithTimeout( - filesystemReadyTimeout, - ).WithPolling( - frozenReasonPollingInterval, - ).Should(Succeed()) - } - } - } - }) - - It("checks snapshots", func() { - By("Snapshots should be `Ready`") - labels := make(map[string]string) - maps.Copy(labels, attachedVirtualDiskLabel) - maps.Copy(labels, testCaseLabel) - - noopGomega := NewGomega(func(string, ...int) {}) - allReady := noopGomega.Eventually(func() error { - vdSnapshots := GetVirtualDiskSnapshots(ns, labels) - for _, snapshot := range vdSnapshots.Items { - if snapshot.Status.Phase == v1alpha2.VirtualDiskSnapshotPhaseReady || snapshot.DeletionTimestamp != nil { - continue - } - return errors.New("still wait for all snapshots either in ready or in deletion state") - } - return nil - }).WithTimeout( - ShortWaitDuration, - ).WithPolling( - Interval, - ).Should(Succeed(), "all snapshots should be in ready state after creation") - - if allReady { - return - } - - allVSReady, err := isAllVolumeSnapshotsReadyToUse(ns, labels) - Expect(err).NotTo(HaveOccurred(), "cannot get VolumeSnapshots by labels") - if !allVSReady { - // TODO: Remove this skip and revert using `g` (Gomega) to the global `Eventually(...)` with `LongWaitDuration` once the new `sds-replicated-volume` arrives - Skip("TODO: remove skip when new sds-replicated-volume arrives. Snapshots are not ready within 60s due to known limitations.") - } - Fail("not all snapshots are ready after creation") - }) - - It("checks snapshots of attached VDs", func() { - By(fmt.Sprintf("Snapshots should be in %s phase", PhaseReady)) - WaitPhaseByLabel(kc.ResourceVDSnapshot, PhaseReady, kc.WaitOptions{ - Labels: attachedVirtualDiskLabel, - Namespace: ns, - Timeout: MaxWaitTimeout, - }) - By("Snapshots should be consistent", func() { - vdSnapshots := v1alpha2.VirtualDiskSnapshotList{} - err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{ - ExcludedLabels: []string{"hasNoConsumer"}, - Namespace: ns, - Labels: attachedVirtualDiskLabel, - }) - Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err) - - for _, snapshot := range vdSnapshots.Items { - Expect(snapshot.Status.Consistent).ToNot(BeNil()) - Expect(*snapshot.Status.Consistent).To(BeTrue(), "consistent field should be `true`: %s", snapshot.Name) - } - }) - }) - - It("checks `FileSystemFrozen` status of VMs", func() { - By("Status should not be `Frozen`") - vmObjects := v1alpha2.VirtualMachineList{} - err := GetObjects(kc.ResourceVM, &vmObjects, kc.GetOptions{Namespace: ns}) - Expect(err).NotTo(HaveOccurred(), "cannot get virtual machines\nstderr: %s", err) - - for _, vm := range vmObjects.Items { - Eventually(func() error { - frozen, err := CheckFileSystemFrozen(vm.Name, vm.Namespace) - if err != nil { - return nil - } - if frozen { - return fmt.Errorf("the filesystem of the virtual machine %s/%s is still frozen", vm.Namespace, vm.Name) - } - return nil - }).WithTimeout( - filesystemReadyTimeout, - ).WithPolling( - filesystemReadyPollingInterval, - ).Should(Succeed()) - } - }) - }) -}) - -func CreateVirtualDiskSnapshot(vdName, snapshotName, namespace string, requiredConsistency bool, labels map[string]string) error { - GinkgoHelper() - vdSnapshot := v1alpha2.VirtualDiskSnapshot{ - TypeMeta: metav1.TypeMeta{ - APIVersion: APIVersion, - Kind: v1alpha2.VirtualDiskSnapshotKind, - }, - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Name: snapshotName, - Namespace: namespace, - }, - Spec: v1alpha2.VirtualDiskSnapshotSpec{ - RequiredConsistency: requiredConsistency, - VirtualDiskName: vdName, - }, - } - - filePath := fmt.Sprintf("%s/snapshots/%s.yaml", conf.TestData.VdSnapshots, snapshotName) - err := util.WriteYamlObject(filePath, &vdSnapshot) - if err != nil { - return fmt.Errorf("cannot write file with virtual disk snapshot: %s\nstderr: %w", snapshotName, err) - } - - res := kubectl.Apply(kc.ApplyOptions{ - Filename: []string{filePath}, - FilenameOption: kc.Filename, - }) - if res.Error() != nil { - return fmt.Errorf("cannot create virtual disk snapshot: %s\nstderr: %s", snapshotName, res.StdErr()) - } - return nil -} - -func isAllVolumeSnapshotsReadyToUse(namespace string, labels map[string]string) (bool, error) { - GinkgoHelper() - list := &unstructured.UnstructuredList{} - err := GetObjects(kc.ResourceVolumeSnapshot, list, kc.GetOptions{ - Namespace: namespace, - Labels: labels, - }) - if err != nil { - return false, err - } - - for _, item := range list.Items { - if item.GetDeletionTimestamp() != nil { - continue - } - ready, found, nestedErr := unstructured.NestedBool(item.Object, "status", "readyToUse") - if nestedErr != nil { - return false, nestedErr - } - if !found || !ready { - return false, nil - } - } - return true, nil -} - -func GetVirtualDiskSnapshots(namespace string, labels map[string]string) v1alpha2.VirtualDiskSnapshotList { - GinkgoHelper() - vdSnapshots := v1alpha2.VirtualDiskSnapshotList{} - err := GetObjects(kc.ResourceVDSnapshot, &vdSnapshots, kc.GetOptions{ - ExcludedLabels: []string{"hasNoConsumer"}, - Namespace: namespace, - Labels: labels, - }) - Expect(err).NotTo(HaveOccurred(), "cannot get `vdSnapshots`\nstderr: %s", err) - return vdSnapshots -} - -func CheckFileSystemFrozen(vmName, vmNamespace string) (bool, error) { - vmObj := v1alpha2.VirtualMachine{} - err := GetObject(kc.ResourceVM, vmName, &vmObj, kc.GetOptions{Namespace: vmNamespace}) - if err != nil { - return false, fmt.Errorf("cannot get `VirtualMachine`: %q\nstderr: %w", vmName, err) - } - - for _, condition := range vmObj.Status.Conditions { - if condition.Type == vmcondition.TypeFilesystemFrozen.String() { - return condition.Status == metav1.ConditionTrue, nil - } - } - - return false, nil -} - -func setDiskImmediateStorageClass() { - virtualDiskWithoutConsumer := v1alpha2.VirtualDisk{} - vdWithoutConsumerFilePath := fmt.Sprintf("%s/vd/vd-alpine-http.yaml", conf.TestData.VdSnapshots) - err := util.UnmarshalResource(vdWithoutConsumerFilePath, &virtualDiskWithoutConsumer) - Expect(err).NotTo(HaveOccurred(), "cannot get object from file: %s\nstderr: %s", vdWithoutConsumerFilePath, err) - - virtualDiskWithoutConsumer.Spec.PersistentVolumeClaim.StorageClass = &conf.StorageClass.ImmediateStorageClass.Name - err = util.WriteYamlObject(vdWithoutConsumerFilePath, &virtualDiskWithoutConsumer) - Expect(err).NotTo(HaveOccurred(), "cannot update virtual disk with custom storage class: %s\nstderr: %s", vdWithoutConsumerFilePath, err) -} diff --git a/test/e2e/vm/migration.go b/test/e2e/vm/migration.go index 7d6b59e993..20ea65cb28 100644 --- a/test/e2e/vm/migration.go +++ b/test/e2e/vm/migration.go @@ -201,7 +201,7 @@ var _ = Describe("VirtualMachineMigration", Label(precheck.NoPrecheck), func() { allObjects = append([]crclient.Object{ vdRootBIOS, vdBlankBIOS, vmBIOS, vdRootUEFI, vdBlankUEFI, vmUEFI, vdHotplugBIOS, vdHotplugUEFI, viHotplugBIOS, viHotplugUEFI, - }, toObjects(vmbdas)...) + }, util.ToObjects(vmbdas)...) err := f.CreateWithDeferredDeletion(ctx, allObjects...) Expect(err).NotTo(HaveOccurred()) @@ -209,7 +209,7 @@ var _ = Describe("VirtualMachineMigration", Label(precheck.NoPrecheck), func() { util.UntilObjectPhase( ctx, string(v1alpha2.BlockDeviceAttachmentPhaseAttached), framework.LongTimeout, - toObjects(vmbdas)..., + util.ToObjects(vmbdas)..., ) util.UntilSSHReady(f, vmBIOS, framework.LongTimeout) @@ -353,11 +353,3 @@ func ensureVMBDAsStayAttached(ctx context.Context, w util.Watcher, names []strin } } } - -func toObjects[T crclient.Object](objs []T) []crclient.Object { - out := make([]crclient.Object, len(objs)) - for i, o := range objs { - out[i] = o - } - return out -}