Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/core/v1alpha2/vmopcondition/condition.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ const (
// ReasonNotReadyToBeExecuted is a ReasonCompleted indicating that the operation is not ready to be executed.
ReasonNotReadyToBeExecuted ReasonCompleted = "NotReadyToBeExecuted"

// ReasonSuperseded is a ReasonCompleted indicating that the operation has been superseded by another operation.
ReasonSuperseded ReasonCompleted = "Superseded"

// ReasonRestartInProgress is a ReasonCompleted indicating that the restart signal has been sent and restart is in progress.
ReasonRestartInProgress ReasonCompleted = "RestartInProgress"

Expand Down
4 changes: 4 additions & 0 deletions docs/USER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -1756,6 +1756,10 @@ A list of possible operations is given in the table below:
| `d8 v evict` | `Evict` | Evict the VM to another host |
| `d8 v migrate` | `Migrate` | Migrate the VM to another host |

Only one active operation is executed for a VM at a time. If a new operation is compatible with an already active operation, it can supersede the older operation. The older operation is completed with `status.phase: Completed` and the `Completed` condition reason `Superseded`, while the new operation continues execution. For example, `Stop` can supersede an active `Start`, `Stop` with `force: true` can supersede a regular `Stop`, and `Restart` can supersede an active `Migrate` or `Evict`.

If operations are incompatible, the new operation is rejected until the active operation finishes. Restore and clone operations do not supersede other VM operations.

How to perform the operation in the web interface:

- Go to the "Projects" tab and select the desired project.
Expand Down
4 changes: 4 additions & 0 deletions docs/USER_GUIDE.ru.md
Original file line number Diff line number Diff line change
Expand Up @@ -1774,6 +1774,10 @@ d8 v restart linux-vm
| `d8 v evict` | `Evict` | Выселить ВМ на другой узел |
| `d8 v migrate` | `Migrate` | Мигрировать ВМ на другой узел |

Для одной ВМ одновременно выполняется только одна активная операция. Если новая операция совместима с уже активной операцией, она может вытеснить более старую операцию. Более старая операция завершается с `status.phase: Completed` и причиной `Superseded` в условии `Completed`, а новая операция продолжает выполнение. Например, `Stop` может вытеснить активную операцию `Start`, `Stop` с `force: true` может вытеснить обычную операцию `Stop`, а `Restart` может вытеснить активную операцию `Migrate` или `Evict`.

Если операции несовместимы, новая операция отклоняется до завершения активной операции. Операции восстановления и клонирования не вытесняют другие операции ВМ.

Как выполнить операцию в веб-интерфейсе:

- Перейдите на вкладку «Проекты» и выберите нужный проект.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package service
import (
"context"

apierrors "k8s.io/apimachinery/pkg/api/errors"
virtv1 "kubevirt.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -53,6 +54,9 @@ func (o RestartOperation) Execute(ctx context.Context) error {
if o.vmop.Spec.Force != nil && *o.vmop.Spec.Force {
kvvmi := &virtv1.VirtualMachineInstance{}
err = o.client.Get(ctx, key, kvvmi)
if apierrors.IsNotFound(err) {
return kvvmutil.AddStartAnnotation(ctx, o.client, kvvm)
}
if err != nil {
return err
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package service
import (
"context"

apierrors "k8s.io/apimachinery/pkg/api/errors"
virtv1 "kubevirt.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand All @@ -42,6 +43,9 @@ type StopOperation struct {
func (o StopOperation) Execute(ctx context.Context) error {
kvvmi := &virtv1.VirtualMachineInstance{}
err := o.client.Get(ctx, virtualMachineKeyByVmop(o.vmop), kvvmi)
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
Expand Down
138 changes: 121 additions & 17 deletions images/virtualization-artifact/pkg/controller/vmop/service/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,22 @@ package service
import (
"context"
"fmt"
"sort"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/ptr"
"k8s.io/client-go/util/retry"
virtv1 "kubevirt.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/deckhouse/virtualization-controller/pkg/common/annotations"
kvvmutil "github.com/deckhouse/virtualization-controller/pkg/common/kvvm"
"github.com/deckhouse/virtualization-controller/pkg/common/object"
commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop"
"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/vmop/supersede"
"github.com/deckhouse/virtualization-controller/pkg/eventrecord"
"github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vmopcondition"
Expand All @@ -49,32 +53,54 @@ func NewBaseVMOPService(client client.Client, recorder eventrecord.EventRecorder
}

func (s *BaseVMOPService) ShouldExecuteOrSetFailedPhase(ctx context.Context, vmop *v1alpha2.VirtualMachineOperation) (bool, error) {
should, err := s.ShouldExecute(ctx, vmop)
return s.ShouldExecuteOrSupersedeOrSetFailedPhase(ctx, vmop)
}

func (s *BaseVMOPService) ShouldExecuteOrSupersedeOrSetFailedPhase(ctx context.Context, vmop *v1alpha2.VirtualMachineOperation) (bool, error) {
blockers, err := s.findOlderActiveVMOPs(ctx, vmop)
if err != nil {
return false, err
}
if should {
if len(blockers) == 0 {
return true, nil
}

vmop.Status.Phase = v1alpha2.VMOPPhaseFailed
conditions.SetCondition(
conditions.NewConditionBuilder(vmopcondition.TypeCompleted).
Generation(vmop.GetGeneration()).
Reason(vmopcondition.ReasonNotReadyToBeExecuted).
Message("VMOP cannot be executed now. Previously created operation should finish first.").
Status(metav1.ConditionFalse),
&vmop.Status.Conditions)
return false, nil
for i := range blockers {
oldVMOP := &blockers[i]
if !supersede.CanSupersede(oldVMOP, vmop) {
s.setNotReadyToBeExecuted(vmop)
return false, nil
}
}

for i := range blockers {
oldVMOP := &blockers[i]
if err := s.cleanupSupersededOperation(ctx, oldVMOP); err != nil {
return false, err
}
if err := s.markSuperseded(ctx, oldVMOP, vmop); err != nil {
return false, err
}
}

return true, nil
}

func (s *BaseVMOPService) ShouldExecute(ctx context.Context, vmop *v1alpha2.VirtualMachineOperation) (bool, error) {
var vmopList v1alpha2.VirtualMachineOperationList
err := s.client.List(ctx, &vmopList, client.InNamespace(vmop.GetNamespace()))
blockers, err := s.findOlderActiveVMOPs(ctx, vmop)
if err != nil {
return false, err
}
return len(blockers) == 0, nil
}

func (s *BaseVMOPService) findOlderActiveVMOPs(ctx context.Context, vmop *v1alpha2.VirtualMachineOperation) ([]v1alpha2.VirtualMachineOperation, error) {
var vmopList v1alpha2.VirtualMachineOperationList
if err := s.client.List(ctx, &vmopList, client.InNamespace(vmop.GetNamespace())); err != nil {
return nil, err
}

blockers := make([]v1alpha2.VirtualMachineOperation, 0)
for _, other := range vmopList.Items {
if other.Spec.VirtualMachine != vmop.Spec.VirtualMachine {
continue
Expand All @@ -85,12 +111,90 @@ func (s *BaseVMOPService) ShouldExecute(ctx context.Context, vmop *v1alpha2.Virt
if other.GetUID() == vmop.GetUID() {
continue
}
if other.CreationTimestamp.Before(ptr.To(vmop.CreationTimestamp)) {
return false, nil
if isOlderVMOP(&other, vmop) {
blockers = append(blockers, other)
}
}

return true, nil
sort.SliceStable(blockers, func(i, j int) bool {
return isOlderVMOP(&blockers[i], &blockers[j])
})

return blockers, nil
}

func isOlderVMOP(a, b *v1alpha2.VirtualMachineOperation) bool {
if !a.CreationTimestamp.Equal(&b.CreationTimestamp) {
return a.CreationTimestamp.Before(&b.CreationTimestamp)
}
return a.GetName() < b.GetName()
}
Comment thread
danilrwx marked this conversation as resolved.

func (s *BaseVMOPService) setNotReadyToBeExecuted(vmop *v1alpha2.VirtualMachineOperation) {
vmop.Status.Phase = v1alpha2.VMOPPhaseFailed
conditions.SetCondition(
conditions.NewConditionBuilder(vmopcondition.TypeCompleted).
Generation(vmop.GetGeneration()).
Reason(vmopcondition.ReasonNotReadyToBeExecuted).
Message("VMOP cannot be executed now. Previously created operation should finish first.").
Status(metav1.ConditionFalse),
&vmop.Status.Conditions)
}

func (s *BaseVMOPService) cleanupSupersededOperation(ctx context.Context, oldVMOP *v1alpha2.VirtualMachineOperation) error {
key := types.NamespacedName{Name: oldVMOP.Spec.VirtualMachine, Namespace: oldVMOP.Namespace}

switch oldVMOP.Spec.Type {
case v1alpha2.VMOPTypeStart:
kvvm := &virtv1.VirtualMachine{}
if err := s.client.Get(ctx, key, kvvm); err != nil {
return client.IgnoreNotFound(err)
}
return kvvmutil.RemoveStartAnnotation(ctx, s.client, kvvm)
case v1alpha2.VMOPTypeRestart:
kvvm := &virtv1.VirtualMachine{}
if err := s.client.Get(ctx, key, kvvm); err != nil {
return client.IgnoreNotFound(err)
}
if err := kvvmutil.RemoveRestartAnnotation(ctx, s.client, kvvm); err != nil {
return err
}
return kvvmutil.RemoveStartAnnotation(ctx, s.client, kvvm)
case v1alpha2.VMOPTypeMigrate, v1alpha2.VMOPTypeEvict:
mig := &virtv1.VirtualMachineInstanceMigration{}
if err := s.client.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("vmop-%s", oldVMOP.Name), Namespace: oldVMOP.Namespace}, mig); err != nil {
return client.IgnoreNotFound(err)
}
return client.IgnoreNotFound(s.client.Delete(ctx, mig))
case v1alpha2.VMOPTypeStop:
return nil
default:
return nil
}
}

func (s *BaseVMOPService) markSuperseded(ctx context.Context, oldVMOP, newVMOP *v1alpha2.VirtualMachineOperation) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
current := &v1alpha2.VirtualMachineOperation{}
if err := s.client.Get(ctx, types.NamespacedName{Name: oldVMOP.Name, Namespace: oldVMOP.Namespace}, current); err != nil {
return client.IgnoreNotFound(err)
}
if commonvmop.IsFinished(current) {
return nil
}

base := current.DeepCopy()
current.Status.Phase = v1alpha2.VMOPPhaseCompleted
conditions.SetCondition(
conditions.NewConditionBuilder(vmopcondition.TypeCompleted).
Generation(current.GetGeneration()).
Reason(vmopcondition.ReasonSuperseded).
Message(fmt.Sprintf("Superseded by %s with type %s", newVMOP.Name, newVMOP.Spec.Type)).
Status(metav1.ConditionTrue),
&current.Status.Conditions)

return s.client.Status().Patch(ctx, current, client.MergeFrom(base))
})
}

func (s *BaseVMOPService) Init(vmop *v1alpha2.VirtualMachineOperation) {
Expand Down
Loading
Loading