Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 49 additions & 0 deletions images/virtualization-artifact/pkg/common/vd/vd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package vd

import (
"context"
"errors"
"fmt"
"log/slog"

Expand All @@ -27,6 +28,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/deckhouse/virtualization-controller/pkg/common/object"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
"github.com/deckhouse/virtualization-controller/pkg/featuregates"
"github.com/deckhouse/virtualization/api/core/v1alpha2"
)
Expand Down Expand Up @@ -75,6 +77,53 @@ func StorageClassChanged(vd *v1alpha2.VirtualDisk) bool {
return *specSc != "" && statusSc != ""
}

type VirtualDiskStorageClassResolver interface {
GetModuleStorageClass(ctx context.Context) (*storagev1.StorageClass, error)
GetDefaultStorageClass(ctx context.Context) (*storagev1.StorageClass, error)
}

// ResolveStorageClassName resolves storage class name for a VirtualDisk
// with the same precedence as VD handlers:
// 1. vd.Status.StorageClassName
// 2. vd.Spec.PersistentVolumeClaim.StorageClass
// 3. module default storage class (if resolver is provided)
// 4. cluster default storage class (if resolver is provided)
func ResolveStorageClassName(ctx context.Context, vd *v1alpha2.VirtualDisk, resolver VirtualDiskStorageClassResolver) (string, error) {
if vd == nil {
return "", nil
}

if vd.Status.StorageClassName != "" {
return vd.Status.StorageClassName, nil
}

if vd.Spec.PersistentVolumeClaim.StorageClass != nil && *vd.Spec.PersistentVolumeClaim.StorageClass != "" {
return *vd.Spec.PersistentVolumeClaim.StorageClass, nil
}

if resolver == nil {
return "", nil
}

moduleStorageClass, err := resolver.GetModuleStorageClass(ctx)
if err != nil {
return "", err
}
if moduleStorageClass != nil {
return moduleStorageClass.Name, nil
}

defaultStorageClass, err := resolver.GetDefaultStorageClass(ctx)
if err != nil && !errors.Is(err, service.ErrDefaultStorageClassNotFound) {
return "", err
}
if defaultStorageClass != nil {
return defaultStorageClass.Name, nil
}

return "", fmt.Errorf("storage class for VirtualDisk %q cannot be determined", vd.Name)
}

func ValidateVirtualImageStorageClassProvisionerCompatibility(ctx context.Context, vd *v1alpha2.VirtualDisk, client client.Client) error {
if vd.Spec.DataSource == nil || vd.Spec.DataSource.Type != v1alpha2.DataSourceTypeObjectRef {
return nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,13 @@ package validator

import (
"context"
"errors"
"fmt"
"reflect"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"

commonvd "github.com/deckhouse/virtualization-controller/pkg/common/vd"
"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
intsvc "github.com/deckhouse/virtualization-controller/pkg/controller/vd/internal/service"
"github.com/deckhouse/virtualization-controller/pkg/controller/vd/internal/source"
"github.com/deckhouse/virtualization/api/core/v1alpha2"
Expand All @@ -47,7 +44,7 @@ func NewVirtualImagePVCStorageClassValidator(client client.Client, scService *in
}

func (v *VirtualImagePVCStorageClassValidator) ValidateCreate(ctx context.Context, vd *v1alpha2.VirtualDisk) (admission.Warnings, error) {
scName, err := v.extractVDStorageClassName(ctx, vd)
scName, err := commonvd.ResolveStorageClassName(ctx, vd, v.scService)
if err != nil {
return nil, err
}
Expand All @@ -70,33 +67,3 @@ func (v *VirtualImagePVCStorageClassValidator) ValidateUpdate(ctx context.Contex

return nil, commonvd.ValidateVirtualImageStorageClassProvisionerCompatibility(ctx, newVD, v.client)
}

func (v *VirtualImagePVCStorageClassValidator) extractVDStorageClassName(ctx context.Context, vd *v1alpha2.VirtualDisk) (string, error) {
if vd.Status.StorageClassName != "" {
return vd.Status.StorageClassName, nil
}

if vd.Spec.PersistentVolumeClaim.StorageClass != nil {
return *vd.Spec.PersistentVolumeClaim.StorageClass, nil
}

moduleStorageClass, err := v.scService.GetModuleStorageClass(ctx)
if err != nil {
return "", err
}

if moduleStorageClass != nil {
return moduleStorageClass.Name, nil
}

defaultStorageClass, err := v.scService.GetDefaultStorageClass(ctx)
if err != nil && !errors.Is(err, service.ErrDefaultStorageClassNotFound) {
return "", err
}

if defaultStorageClass != nil {
return defaultStorageClass.Name, nil
}

return "", fmt.Errorf("storage class for VirtualDisk %q cannot be determined", vd.Name)
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,22 @@ import (
"fmt"

corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
virtv1 "kubevirt.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/yaml"

"github.com/deckhouse/virtualization-controller/pkg/common/annotations"
kvvmutil "github.com/deckhouse/virtualization-controller/pkg/common/kvvm"
"github.com/deckhouse/virtualization-controller/pkg/common/nodeaffinity"
"github.com/deckhouse/virtualization-controller/pkg/common/object"
commonvd "github.com/deckhouse/virtualization-controller/pkg/common/vd"
"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/indexer"
"github.com/deckhouse/virtualization-controller/pkg/controller/powerstate"
Expand Down Expand Up @@ -410,7 +415,7 @@ func (s *state) PVNodeAffinityTerms(ctx context.Context) ([]corev1.NodeSelectorT
continue
}

terms, err := s.pvNodeAffinityTermsForPVC(ctx, pvcName, namespace)
terms, err := s.pvNodeAffinityTermsForPVC(ctx, ref.Kind, ref.Name, pvcName, namespace)
if err != nil {
return nil, fmt.Errorf("get PV node affinity for PVC %s: %w", pvcName, err)
}
Expand Down Expand Up @@ -495,19 +500,63 @@ func (s *state) resolvePVCName(ctx context.Context, kind v1alpha2.BlockDeviceKin
}
}

func (s *state) pvNodeAffinityTermsForPVC(ctx context.Context, pvcName, namespace string) ([]corev1.NodeSelectorTerm, error) {
func (s *state) resolveStorageClassName(ctx context.Context, kind v1alpha2.BlockDeviceKind, name, pvcName string) (string, error) {
switch kind {
case v1alpha2.DiskDevice:
vd, err := s.VirtualDisk(ctx, name)
if err != nil {
return "", err
}
if vd == nil {
return "", nil
}
// During migration, status.StorageClassName can still point to the old PVC's class.
// Avoid using it for the target PVC; rely on target PVC fields instead.
if vd.Status.MigrationState.TargetPVC == pvcName {
return "", nil
}
return commonvd.ResolveStorageClassName(ctx, vd, nil)
case v1alpha2.ImageDevice:
vi, err := s.VirtualImage(ctx, name)
if err != nil {
return "", err
}
if vi == nil {
return "", nil
}
if vi.Status.StorageClassName != "" {
return vi.Status.StorageClassName, nil
}
if vi.Spec.PersistentVolumeClaim.StorageClass != nil {
return *vi.Spec.PersistentVolumeClaim.StorageClass, nil
}
return "", nil
default:
return "", nil
}
}

func (s *state) pvNodeAffinityTermsForPVC(ctx context.Context, kind v1alpha2.BlockDeviceKind, name, pvcName, namespace string) ([]corev1.NodeSelectorTerm, error) {
pvc, err := object.FetchObject(ctx, types.NamespacedName{
Name: pvcName, Namespace: namespace,
}, s.client, &corev1.PersistentVolumeClaim{})
if err != nil {
return nil, err
}
if pvc == nil || pvc.Spec.VolumeName == "" {
if pvc == nil {
return nil, nil
}

if pvc.Spec.VolumeName != "" {
return s.nodeAffinityTermsFromBoundPV(ctx, pvc.Spec.VolumeName)
}

return s.nodeAffinityTermsFromUnboundPVC(ctx, kind, name, pvcName, pvc)
}

func (s *state) nodeAffinityTermsFromBoundPV(ctx context.Context, pvName string) ([]corev1.NodeSelectorTerm, error) {
pv, err := object.FetchObject(ctx, types.NamespacedName{
Name: pvc.Spec.VolumeName,
Name: pvName,
}, s.client, &corev1.PersistentVolume{})
if err != nil {
return nil, err
Expand All @@ -522,6 +571,126 @@ func (s *state) pvNodeAffinityTermsForPVC(ctx context.Context, pvcName, namespac
return nil, nil
}

// nodeAffinityTermsFromUnboundPVC determines node affinity for an unbound PVC by:
// 1. Looking for pre-provisioned Available PVs (static provisioning case).
// 2. If none found, deriving topology from StorageClass + LVMVolumeGroup (dynamic provisioning case).
func (s *state) nodeAffinityTermsFromUnboundPVC(ctx context.Context, kind v1alpha2.BlockDeviceKind, name, pvcName string, pvc *corev1.PersistentVolumeClaim) ([]corev1.NodeSelectorTerm, error) {
storageClassName, err := s.resolveStorageClassName(ctx, kind, name, pvcName)
if err != nil {
return nil, fmt.Errorf("resolve StorageClass for %s/%s: %w", kind, name, err)
}
// During migration we intentionally do not trust VD status storage class for target PVC,
// because it can still point to the source PVC class while the source VM pod is running.
// In this case use target PVC storage class if it is already set.
if storageClassName == "" && pvc.Spec.StorageClassName != nil {
storageClassName = *pvc.Spec.StorageClassName
}
if storageClassName == "" {
return nil, nil
}

var pvList corev1.PersistentVolumeList
if err := s.client.List(ctx, &pvList); err != nil {
return nil, fmt.Errorf("list PVs: %w", err)
}

var allTerms []corev1.NodeSelectorTerm
for i := range pvList.Items {
pv := &pvList.Items[i]
if pv.Status.Phase != corev1.VolumeAvailable {
continue
}
if pv.Spec.StorageClassName != storageClassName {
continue
}
if pv.Spec.NodeAffinity == nil || pv.Spec.NodeAffinity.Required == nil {
continue
}
allTerms = append(allTerms, pv.Spec.NodeAffinity.Required.NodeSelectorTerms...)
}

if len(allTerms) > 0 {
return allTerms, nil
}

return s.nodeAffinityTermsFromStorageClassTopology(ctx, storageClassName)
}

const (
localCSIProvisioner = "local.csi.storage.deckhouse.io"
lvmVolumeGroupsParam = "local.csi.storage.deckhouse.io/lvm-volume-groups"
)

// nodeAffinityTermsFromStorageClassTopology resolves node topology for dynamic provisioning
// by reading the StorageClass parameters and looking up LVMVolumeGroup resources to determine
// which nodes can provision volumes for this StorageClass.
func (s *state) nodeAffinityTermsFromStorageClassTopology(ctx context.Context, storageClassName string) ([]corev1.NodeSelectorTerm, error) {
sc, err := object.FetchObject(ctx, types.NamespacedName{Name: storageClassName}, s.client, &storagev1.StorageClass{})
if err != nil {
return nil, fmt.Errorf("get StorageClass %s: %w", storageClassName, err)
}
if sc == nil || sc.Provisioner != localCSIProvisioner {
return nil, nil
}

lvgParam, ok := sc.Parameters[lvmVolumeGroupsParam]
if !ok || lvgParam == "" {
return nil, nil
}

var lvgs []struct {
Name string `json:"name"`
}
if err := yaml.Unmarshal([]byte(lvgParam), &lvgs); err != nil {
return nil, nil
}

var nodeNames []string
for _, lvg := range lvgs {
nodeName, err := s.getNodeNameFromLVMVolumeGroup(ctx, lvg.Name)
if err != nil {
return nil, err
}
if nodeName != "" {
nodeNames = append(nodeNames, nodeName)
}
}

if len(nodeNames) == 0 {
return nil, nil
}

return []corev1.NodeSelectorTerm{{
MatchExpressions: []corev1.NodeSelectorRequirement{{
Key: corev1.LabelHostname,
Operator: corev1.NodeSelectorOpIn,
Values: nodeNames,
}},
}}, nil
}

var lvmVolumeGroupGVK = schema.GroupVersionKind{
Group: "storage.deckhouse.io",
Version: "v1alpha1",
Kind: "LVMVolumeGroup",
}

func (s *state) getNodeNameFromLVMVolumeGroup(ctx context.Context, name string) (string, error) {
lvg := &unstructured.Unstructured{}
lvg.SetGroupVersionKind(lvmVolumeGroupGVK)

err := s.client.Get(ctx, types.NamespacedName{Name: name}, lvg)
if err != nil {
if k8serrors.IsNotFound(err) {
return "", nil
}
return "", fmt.Errorf("get LVMVolumeGroup %s: %w", name, err)
}

nodeName, _, _ := unstructured.NestedString(lvg.Object, "spec", "local", "nodeName")
return nodeName, nil
}

func (s *state) USBDevice(ctx context.Context, name string) (*v1alpha2.USBDevice, error) {
return object.FetchObject(ctx, types.NamespacedName{
Name: name,
Expand Down
Loading
Loading