diff --git a/api/core/v1alpha2/vmopcondition/condition.go b/api/core/v1alpha2/vmopcondition/condition.go index 9234b68ced..0a70f00d3d 100644 --- a/api/core/v1alpha2/vmopcondition/condition.go +++ b/api/core/v1alpha2/vmopcondition/condition.go @@ -142,6 +142,9 @@ const ( // ReasonOperationCompleted is a ReasonCompleted indicating that operation is completed. ReasonOperationCompleted ReasonCompleted = "OperationCompleted" + + // ReasonMigrationNetworkUnavailable indicates that liveMigration.systemNetworkName is configured in the ModuleConfig but the source node has no resolved migration interface. + ReasonMigrationNetworkUnavailable ReasonCompleted = "MigrationNetworkUnavailable" ) // ReasonCompleted represents specific reasons for the 'SignalSent' condition type. diff --git a/build/components/versions.yml b/build/components/versions.yml index 6fa7777554..e506030f00 100644 --- a/build/components/versions.yml +++ b/build/components/versions.yml @@ -3,7 +3,7 @@ firmware: libvirt: v10.9.0 edk2: stable202411 core: - 3p-kubevirt: v1.6.2-v12n.38 + 3p-kubevirt: feat/network/dedicated-migration-network 3p-containerized-data-importer: v1.60.3-v12n.19 distribution: 2.8.3 package: diff --git a/docs/ADMIN_GUIDE.md b/docs/ADMIN_GUIDE.md index 6bb53b0ee5..8ad6c77ed9 100644 --- a/docs/ADMIN_GUIDE.md +++ b/docs/ADMIN_GUIDE.md @@ -1203,6 +1203,17 @@ How to start VM migration in the web interface: - Select `Migrate` from the pop-up menu. - Confirm or cancel the migration in the pop-up window. +#### Dedicated migration network + +By default, live migration traffic flows over the node's default network and competes with workload traffic. You can also route it over a dedicated VLAN provisioned by the [`sdn`](https://deckhouse.io/modules/sdn/) module. + +Prerequisites: + +- The `sdn` module is enabled. +- A `SystemNetwork` resource exists and is in the `Ready` state. + +To enable the feature, set `spec.settings.liveMigration.systemNetworkName` on the `virtualization` ModuleConfig to the name of the prepared `SystemNetwork`. Once configured, every VM migration in the cluster runs over the specified `SystemNetwork` VLAN. + #### Maintenance mode When working on nodes with virtual machines running, there is a risk of disrupting their performance. To avoid this, you can put a node into the maintenance mode and migrate the virtual machines to other free nodes. diff --git a/docs/ADMIN_GUIDE.ru.md b/docs/ADMIN_GUIDE.ru.md index 7c8609d701..4c6f66ed8b 100644 --- a/docs/ADMIN_GUIDE.ru.md +++ b/docs/ADMIN_GUIDE.ru.md @@ -1213,6 +1213,17 @@ spec: - Во всплывающем меню выберите `Мигрировать`. - Во всплывающем окне подтвердите или отмените миграцию. +#### Выделенная сеть для миграции + +По умолчанию трафик живой миграции идёт по основной сети узла и конкурирует за полосу пропускания с пользовательскими нагрузками. Его можно направить через выделенный VLAN, предоставляемый модулем [`sdn`](https://deckhouse.ru/modules/sdn/). + +Необходимые условия: + +- Включён модуль `sdn`. +- Ресурс `SystemNetwork` создан и находится в состоянии `Ready`. + +Чтобы включить возможность, укажите имя подготовленного `SystemNetwork` в поле `spec.settings.liveMigration.systemNetworkName` ресурса ModuleConfig `virtualization`. После настройки каждая миграция виртуальных машин в кластере выполняется по VLAN указанного `SystemNetwork`. + #### Режим обслуживания При выполнении работ на узлах с запущенными виртуальными машинами существует риск нарушения их работоспособности. Чтобы этого избежать, узел можно перевести в режим обслуживания и мигрировать виртуальные машины на другие свободные узлы. diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 542d241165..5a1ef995a2 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -7,6 +7,7 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }}-src-artifact +fromCacheVersion: "{{ now | date "Mon Jan 2 15:04:05 MST 2006" }}" final: false fromImage: builder/src secrets: @@ -42,6 +43,7 @@ packages: {{ $builderDependencies := include "$name" . | fromYaml }} image: {{ .ModuleNamePrefix }}{{ .ImageName }} +fromCacheVersion: "{{ now | date "Mon Jan 2 15:04:05 MST 2006" }}" final: false fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-alt-1.25" "builder/golang-alt-svace-1.25" }} mount: diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go index d69a9d8d55..9bb70d7ab3 100644 --- a/images/virtualization-artifact/cmd/virtualization-controller/main.go +++ b/images/virtualization-artifact/cmd/virtualization-controller/main.go @@ -46,6 +46,7 @@ import ( "github.com/deckhouse/virtualization-controller/pkg/controller/evacuation" "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" "github.com/deckhouse/virtualization-controller/pkg/controller/livemigration" + "github.com/deckhouse/virtualization-controller/pkg/controller/migrationiface" mc "github.com/deckhouse/virtualization-controller/pkg/controller/moduleconfig" mcapi "github.com/deckhouse/virtualization-controller/pkg/controller/moduleconfig/api" "github.com/deckhouse/virtualization-controller/pkg/controller/nodeusbdevice" @@ -96,6 +97,8 @@ const ( SdnEnabledEnv = "SDN_ENABLED" clusterUUIDEnv = "CLUSTER_UUID" + + migrationSystemNetworkNameEnv = "MIGRATION_SYSTEM_NETWORK_NAME" ) func main() { @@ -259,6 +262,11 @@ func main() { BindAddress: metricsBindAddr, }, HealthProbeBindAddress: healthProbeBindAddr, + // Route unstructured reads through the cache so field-index lookups are served locally + // instead of hitting the apiserver. + Client: client.Options{ + Cache: &client.CacheOptions{Unstructured: true}, + }, } if pprofBindAddr != "" { managerOpts.PprofBindAddress = pprofBindAddr @@ -402,6 +410,12 @@ func main() { os.Exit(1) } + migrationIfaceLogger := logger.NewControllerLogger(migrationiface.ControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if _, err = migrationiface.NewController(ctx, mgr, migrationIfaceLogger, os.Getenv(migrationSystemNetworkNameEnv)); err != nil { + log.Error(err.Error()) + os.Exit(1) + } + resourceSliceLogger := logger.NewControllerLogger(resourceslice.ControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) if _, err = resourceslice.NewController(ctx, mgr, resourceSliceLogger); err != nil { log.Error(err.Error()) @@ -427,7 +441,7 @@ func main() { } vmopLogger := logger.NewControllerLogger(vmop.ControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) - if err = vmop.SetupController(ctx, mgr, vmopLogger); err != nil { + if err = vmop.SetupController(ctx, mgr, vmopLogger, os.Getenv(migrationSystemNetworkNameEnv)); err != nil { log.Error(err.Error()) os.Exit(1) } diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index 10ab494bbd..fdf8291bf2 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -192,6 +192,12 @@ const ( // AnnNetworksStatus is the annotation for view current network configuration into Pod. AnnNetworksStatus = "network.deckhouse.io/networks-status" + // AnnMigrationIface names the kernel interface that virt-handler binds + // live-migration traffic to. Written on Nodes by the migrationiface + // controller from a SystemNetwork CR (sdn module); read by virt-handler + // on startup. + AnnMigrationIface = AnnAPIGroupV + "/migration-iface" + // AnnVirtualDiskOriginalAnnotations is the annotation for storing original VirtualDisk annotations. AnnVirtualDiskOriginalAnnotations = AnnAPIGroupV + "/vd-original-annotations" // AnnVirtualDiskOriginalLabels is the annotation for storing original VirtualDisk labels. diff --git a/images/virtualization-artifact/pkg/controller/indexer/indexer.go b/images/virtualization-artifact/pkg/controller/indexer/indexer.go index c9991df5be..ed31bde1c4 100644 --- a/images/virtualization-artifact/pkg/controller/indexer/indexer.go +++ b/images/virtualization-artifact/pkg/controller/indexer/indexer.go @@ -75,6 +75,9 @@ const ( IndexFieldResourceSliceByPoolName = "spec.pool.name" IndexFieldResourceSliceByDriver = "spec.driver" + + IndexFieldSNNNIAByNodeName = "snnnia.status.nodeName" + IndexFieldSNNNIABySystemNetworkName = "snnnia.spec.systemNetworkName" ) var IndexGetters = []IndexGetter{ @@ -113,6 +116,11 @@ var IndexGettersUSB = []IndexGetter{ IndexResourceSliceByDriver, } +var IndexGettersSDN = []IndexGetter{ + IndexSNNNIAByNodeName, + IndexSNNNIABySystemNetworkName, +} + type IndexGetter func() (obj client.Object, field string, extractValue client.IndexerFunc) func IndexALL(ctx context.Context, mgr manager.Manager) error { @@ -125,6 +133,15 @@ func IndexALL(ctx context.Context, mgr manager.Manager) error { } } + if featuregates.Default().Enabled(featuregates.SDN) { + for _, fn := range IndexGettersSDN { + obj, field, indexFunc := fn() + if err := mgr.GetFieldIndexer().IndexField(ctx, obj, field, indexFunc); err != nil { + return err + } + } + } + for _, fn := range IndexGetters { obj, field, indexFunc := fn() if err := mgr.GetFieldIndexer().IndexField(ctx, obj, field, indexFunc); err != nil { diff --git a/images/virtualization-artifact/pkg/controller/indexer/snnnia_indexer.go b/images/virtualization-artifact/pkg/controller/indexer/snnnia_indexer.go new file mode 100644 index 0000000000..f38a9e3dc5 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/indexer/snnnia_indexer.go @@ -0,0 +1,57 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package indexer + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var snnniaGVK = schema.GroupVersionKind{ + Group: "network.deckhouse.io", + Version: "v1alpha1", + Kind: "SystemNetworkNodeNetworkInterfaceAttachment", +} + +func snnniaSeed() client.Object { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(snnniaGVK) + return u +} + +func snnniaIndexer(path ...string) client.IndexerFunc { + return func(o client.Object) []string { + u, ok := o.(*unstructured.Unstructured) + if !ok || u == nil { + return nil + } + v, _, _ := unstructured.NestedString(u.Object, path...) + if v == "" { + return nil + } + return []string{v} + } +} + +func IndexSNNNIAByNodeName() (client.Object, string, client.IndexerFunc) { + return snnniaSeed(), IndexFieldSNNNIAByNodeName, snnniaIndexer("status", "nodeName") +} + +func IndexSNNNIABySystemNetworkName() (client.Object, string, client.IndexerFunc) { + return snnniaSeed(), IndexFieldSNNNIABySystemNetworkName, snnniaIndexer("spec", "systemNetworkName") +} diff --git a/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_controller.go b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_controller.go new file mode 100644 index 0000000000..920f3160b3 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_controller.go @@ -0,0 +1,68 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package migrationiface annotates each Node with the kernel interface name +// of a dedicated live-migration network, resolved from sdn's +// SystemNetworkNodeNetworkInterfaceAttachment + NodeNetworkInterface. +// virt-handler reads the annotation (see pkg/common/annotations.AnnMigrationIface) +// at startup to bind migration traffic to that interface. +package migrationiface + +import ( + "context" + "time" + + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/featuregates" + "github.com/deckhouse/virtualization-controller/pkg/logger" +) + +const ControllerName = "migrationiface-controller" + +func NewController( + ctx context.Context, + mgr manager.Manager, + log *log.Logger, + systemNetworkName string, +) (controller.Controller, error) { + if !featuregates.Default().Enabled(featuregates.SDN) { + log.Info("SDN feature gate is disabled, migrationiface controller is disabled") + return nil, nil + } + + r := NewReconciler(mgr.GetClient(), systemNetworkName, log) + + c, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: r, + RecoverPanic: ptr.To(true), + LogConstructor: logger.NewConstructor(log), + CacheSyncTimeout: 10 * time.Minute, + }) + if err != nil { + return nil, err + } + + if err = r.SetupController(ctx, mgr, c); err != nil { + return nil, err + } + + log.Info("Initialized migrationiface controller", "systemNetwork", systemNetworkName) + return c, nil +} diff --git a/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler.go b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler.go new file mode 100644 index 0000000000..aae1a12ffc --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler.go @@ -0,0 +1,207 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationiface + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" +) + +const ( + sdnGroup = "network.deckhouse.io" + sdnVersion = "v1alpha1" + sdnNodeNameLabel = sdnGroup + "/node-name" + sdnInterfaceType = sdnGroup + "/interface-type" + sdnInterfaceVLAN = "VLAN" +) + +var ( + snnniaGVK = schema.GroupVersionKind{Group: sdnGroup, Version: sdnVersion, Kind: "SystemNetworkNodeNetworkInterfaceAttachment"} + nniGVK = schema.GroupVersionKind{Group: sdnGroup, Version: sdnVersion, Kind: "NodeNetworkInterface"} +) + +func NewReconciler(c client.Client, systemNetworkName string, log *log.Logger) *Reconciler { + return &Reconciler{ + client: c, + systemNetworkName: systemNetworkName, + log: log, + } +} + +type Reconciler struct { + client client.Client + systemNetworkName string + log *log.Logger +} + +func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr controller.Controller) error { + nodePredicate := predicate.TypedFuncs[*corev1.Node]{ + CreateFunc: func(event.TypedCreateEvent[*corev1.Node]) bool { return true }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Node]) bool { + return e.ObjectOld.Annotations[annotations.AnnMigrationIface] != + e.ObjectNew.Annotations[annotations.AnnMigrationIface] + }, + DeleteFunc: func(event.TypedDeleteEvent[*corev1.Node]) bool { return false }, + GenericFunc: func(event.TypedGenericEvent[*corev1.Node]) bool { return false }, + } + if err := ctr.Watch(source.Kind(mgr.GetCache(), + &corev1.Node{}, + &handler.TypedEnqueueRequestForObject[*corev1.Node]{}, + nodePredicate, + )); err != nil { + return fmt.Errorf("watch Node: %w", err) + } + + r.watchSdnKind(mgr, ctr, snnniaGVK, func(obj *unstructured.Unstructured) string { + n, _, _ := unstructured.NestedString(obj.Object, "status", "nodeName") + return n + }) + + r.watchSdnKind(mgr, ctr, nniGVK, func(obj *unstructured.Unstructured) string { + if obj.GetLabels()[sdnInterfaceType] != sdnInterfaceVLAN { + return "" + } + if n := obj.GetLabels()[sdnNodeNameLabel]; n != "" { + return n + } + n, _, _ := unstructured.NestedString(obj.Object, "spec", "nodeName") + return n + }) + + return nil +} + +func (r *Reconciler) watchSdnKind( + mgr manager.Manager, + ctr controller.Controller, + gvk schema.GroupVersionKind, + toNodeName func(*unstructured.Unstructured) string, +) { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + err := ctr.Watch(source.Kind(mgr.GetCache(), obj, + handler.TypedEnqueueRequestsFromMapFunc(func(_ context.Context, o *unstructured.Unstructured) []reconcile.Request { + if n := toNodeName(o); n != "" { + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: n}}} + } + return nil + }), + )) + if err != nil { + r.log.Warn("sdn watch failed; migration interface annotation will not track sdn changes", + "kind", gvk.Kind, "err", err.Error()) + } +} + +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + var node corev1.Node + if err := r.client.Get(ctx, req.NamespacedName, &node); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + desired, err := r.resolveInterfaceForNode(ctx, node.Name) + if err != nil { + return reconcile.Result{}, err + } + + if node.Annotations[annotations.AnnMigrationIface] == desired { + return reconcile.Result{}, nil + } + + var value any + if desired != "" { + value = desired + } + patch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "annotations": map[string]any{annotations.AnnMigrationIface: value}, + }, + }) + if err != nil { + return reconcile.Result{}, err + } + if err := r.client.Patch(ctx, &node, client.RawPatch(types.StrategicMergePatchType, patch)); err != nil { + return reconcile.Result{}, fmt.Errorf("patch node %q annotation: %w", node.Name, err) + } + + r.log.Info("updated migration interface annotation", + "node", node.Name, + "systemNetwork", r.systemNetworkName, + "interface", desired, + ) + return reconcile.Result{}, nil +} + +func (r *Reconciler) resolveInterfaceForNode(ctx context.Context, nodeName string) (string, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{Group: sdnGroup, Version: sdnVersion, Kind: snnniaGVK.Kind + "List"}) + err := r.client.List(ctx, list, client.MatchingFields{ + indexer.IndexFieldSNNNIAByNodeName: nodeName, + indexer.IndexFieldSNNNIABySystemNetworkName: r.systemNetworkName, + }) + if err != nil { + if meta.IsNoMatchError(err) { + return "", nil + } + return "", fmt.Errorf("list %s: %w", snnniaGVK.Kind, err) + } + + for i := range list.Items { + nniName, _, _ := unstructured.NestedString(list.Items[i].Object, "status", "nodeNetworkInterfaceName") + if nniName == "" { + continue + } + return r.ifNameFromNNI(ctx, nniName) + } + return "", nil +} + +func (r *Reconciler) ifNameFromNNI(ctx context.Context, nniName string) (string, error) { + nni := &unstructured.Unstructured{} + nni.SetGroupVersionKind(nniGVK) + if err := r.client.Get(ctx, client.ObjectKey{Name: nniName}, nni); err != nil { + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + return "", nil + } + return "", fmt.Errorf("get %s %q: %w", nniGVK.Kind, nniName, err) + } + ifName, _, _ := unstructured.NestedString(nni.Object, "status", "ifName") + return ifName, nil +} diff --git a/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler_test.go b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler_test.go new file mode 100644 index 0000000000..9ab716bd95 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/migrationiface/migrationiface_reconciler_test.go @@ -0,0 +1,206 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationiface + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" + "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" +) + +const ( + testSystemNetworkName = "migration" + testNode = "node-1" + testIfName = "eth0.999" + testNNIName = "node-1-migration-vlan-999-abcdef" +) + +func newFakeClient(objs ...client.Object) client.WithWatch { + GinkgoHelper() + scheme := runtime.NewScheme() + Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) + + b := fake.NewClientBuilder().WithScheme(scheme) + for _, fn := range indexer.IndexGettersSDN { + b.WithIndex(fn()) + } + if len(objs) > 0 { + b.WithObjects(objs...) + } + return b.Build() +} + +func newNode(annotationVal string) *corev1.Node { + n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: testNode}} + if annotationVal != "" { + n.Annotations = map[string]string{annotations.AnnMigrationIface: annotationVal} + } + return n +} + +func newSNNNIA(name, sn, nodeName, nniName string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(snnniaGVK) + u.SetName(name) + _ = unstructured.SetNestedField(u.Object, sn, "spec", "systemNetworkName") + if nodeName != "" { + _ = unstructured.SetNestedField(u.Object, nodeName, "status", "nodeName") + } + if nniName != "" { + _ = unstructured.SetNestedField(u.Object, nniName, "status", "nodeNetworkInterfaceName") + } + return u +} + +func newNNI(ifName string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(nniGVK) + u.SetName(testNNIName) + if ifName != "" { + _ = unstructured.SetNestedField(u.Object, ifName, "status", "ifName") + } + return u +} + +var _ = Describe("Reconciler", func() { + var ( + ctx context.Context + r *Reconciler + ) + + BeforeEach(func() { + ctx = context.Background() + }) + + reconcileNode := func(c client.Client, nodeName string) { + GinkgoHelper() + r = NewReconciler(c, testSystemNetworkName, log.NewNop()) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKey{Name: nodeName}}) + Expect(err).NotTo(HaveOccurred()) + } + + getNode := func(c client.Client) *corev1.Node { + GinkgoHelper() + n := &corev1.Node{} + Expect(c.Get(context.Background(), client.ObjectKey{Name: testNode}, n)).To(Succeed()) + return n + } + + It("sets the annotation from SNNNIA + NNI when the node has none", func() { + c := newFakeClient( + newNode(""), + newSNNNIA("attachment-1", testSystemNetworkName, testNode, testNNIName), + newNNI(testIfName), + ) + reconcileNode(c, testNode) + Expect(getNode(c).Annotations[annotations.AnnMigrationIface]).To(Equal(testIfName)) + }) + + It("does not patch when annotation already matches the resolved value", func() { + node := newNode(testIfName) + c := newFakeClient( + node, + newSNNNIA("attachment-1", testSystemNetworkName, testNode, testNNIName), + newNNI(testIfName), + ) + rvBefore := node.ResourceVersion + + reconcileNode(c, testNode) + + got := getNode(c) + Expect(got.Annotations[annotations.AnnMigrationIface]).To(Equal(testIfName)) + Expect(got.ResourceVersion).To(Equal(rvBefore), "ResourceVersion bump indicates a spurious patch") + }) + + It("clears a stale annotation when no matching SNNNIA exists", func() { + c := newFakeClient(newNode("stale-iface")) + reconcileNode(c, testNode) + _, present := getNode(c).Annotations[annotations.AnnMigrationIface] + Expect(present).To(BeFalse()) + }) + + It("is a no-op when node has no annotation and resolver returns empty", func() { + c := newFakeClient(newNode("")) + reconcileNode(c, testNode) + _, present := getNode(c).Annotations[annotations.AnnMigrationIface] + Expect(present).To(BeFalse()) + }) + + It("returns no error when the Node has been deleted", func() { + c := newFakeClient() + r = NewReconciler(c, testSystemNetworkName, log.NewNop()) + _, err := r.Reconcile(context.Background(), reconcile.Request{NamespacedName: client.ObjectKey{Name: "nonexistent"}}) + Expect(err).NotTo(HaveOccurred()) + }) + + It("skips SNNNIA for a different SystemNetwork", func() { + c := newFakeClient( + newNode(""), + newSNNNIA("attachment-other", "other-sn", testNode, testNNIName), + newNNI(testIfName), + ) + reconcileNode(c, testNode) + _, present := getNode(c).Annotations[annotations.AnnMigrationIface] + Expect(present).To(BeFalse(), "must not annotate from an attachment belonging to another SystemNetwork") + }) + + It("skips SNNNIA without status.nodeName (IPAM-failed attachments)", func() { + c := newFakeClient( + newNode(""), + newSNNNIA("attachment-ipam-failed", testSystemNetworkName, "", ""), + ) + reconcileNode(c, testNode) + _, present := getNode(c).Annotations[annotations.AnnMigrationIface] + Expect(present).To(BeFalse()) + }) + + Describe("resolveInterfaceForNode", func() { + It("returns empty when the referenced NNI doesn't exist", func() { + c := newFakeClient( + newSNNNIA("attachment-1", testSystemNetworkName, testNode, "missing-nni"), + ) + r = NewReconciler(c, testSystemNetworkName, log.NewNop()) + got, err := r.resolveInterfaceForNode(context.Background(), testNode) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeEmpty()) + }) + + It("returns empty when the NNI has no status.ifName yet", func() { + c := newFakeClient( + newSNNNIA("attachment-1", testSystemNetworkName, testNode, testNNIName), + newNNI(""), + ) + r = NewReconciler(c, testSystemNetworkName, log.NewNop()) + got, err := r.resolveInterfaceForNode(context.Background(), testNode) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(BeEmpty()) + }) + }) +}) diff --git a/images/virtualization-artifact/pkg/controller/migrationiface/suite_test.go b/images/virtualization-artifact/pkg/controller/migrationiface/suite_test.go new file mode 100644 index 0000000000..bf9dd4e9a5 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/migrationiface/suite_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationiface + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMigrationIface(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "MigrationIface controller Suite") +} diff --git a/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator.go b/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator.go new file mode 100644 index 0000000000..17188f282e --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator.go @@ -0,0 +1,112 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package moduleconfig + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + mcapi "github.com/deckhouse/virtualization-controller/pkg/controller/moduleconfig/api" +) + +const ( + liveMigrationField = "liveMigration" + systemNetworkNameKey = "systemNetworkName" + conditionTypeReady = "Ready" + conditionStatusTrue = "True" +) + +var systemNetworkGVK = schema.GroupVersionKind{ + Group: "network.deckhouse.io", + Version: "v1alpha1", + Kind: "SystemNetwork", +} + +type liveMigrationValidator struct { + client client.Client +} + +func newLiveMigrationValidator(c client.Client) *liveMigrationValidator { + return &liveMigrationValidator{client: c} +} + +func (v liveMigrationValidator) ValidateCreate(ctx context.Context, mc *mcapi.ModuleConfig) (admission.Warnings, error) { + return v.validate(ctx, mc) +} + +func (v liveMigrationValidator) ValidateUpdate(ctx context.Context, _, newMC *mcapi.ModuleConfig) (admission.Warnings, error) { + return v.validate(ctx, newMC) +} + +func (v liveMigrationValidator) validate(ctx context.Context, mc *mcapi.ModuleConfig) (admission.Warnings, error) { + name := parseLiveMigrationSystemNetworkName(mc.Spec.Settings) + if name == "" { + return nil, nil + } + + sn := &unstructured.Unstructured{} + sn.SetGroupVersionKind(systemNetworkGVK) + err := v.client.Get(ctx, client.ObjectKey{Name: name}, sn) + switch { + case meta.IsNoMatchError(err): + return nil, fmt.Errorf("liveMigration.systemNetworkName=%q: SDN module is not enabled", name) + case apierrors.IsNotFound(err): + return nil, fmt.Errorf("liveMigration.systemNetworkName=%q: SystemNetwork not found", name) + case err != nil: + return nil, fmt.Errorf("liveMigration.systemNetworkName=%q: %w", name, err) + } + + if !isSystemNetworkReady(sn) { + return nil, fmt.Errorf("liveMigration.systemNetworkName=%q: SystemNetwork is not Ready", name) + } + return nil, nil +} + +func parseLiveMigrationSystemNetworkName(settings mcapi.SettingsValues) string { + lm, ok := settings[liveMigrationField].(map[string]any) + if !ok { + return "" + } + name, _ := lm[systemNetworkNameKey].(string) + return name +} + +func isSystemNetworkReady(sn *unstructured.Unstructured) bool { + conds, found, err := unstructured.NestedSlice(sn.Object, "status", "conditions") + if err != nil || !found { + return false + } + for _, c := range conds { + m, ok := c.(map[string]any) + if !ok { + continue + } + t, _ := m["type"].(string) + s, _ := m["status"].(string) + if t == conditionTypeReady && s == conditionStatusTrue { + return true + } + } + return false +} diff --git a/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator_test.go b/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator_test.go new file mode 100644 index 0000000000..dbd4572e21 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/moduleconfig/livemigration_validator_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package moduleconfig + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + mcapi "github.com/deckhouse/virtualization-controller/pkg/controller/moduleconfig/api" +) + +var _ = Describe("parseLiveMigrationSystemNetworkName", func() { + DescribeTable("extracts the configured name", + func(settings mcapi.SettingsValues, expected string) { + Expect(parseLiveMigrationSystemNetworkName(settings)).To(Equal(expected)) + }, + Entry("nil settings", mcapi.SettingsValues(nil), ""), + Entry("missing liveMigration", mcapi.SettingsValues{}, ""), + Entry("liveMigration without systemNetworkName", + mcapi.SettingsValues{"liveMigration": map[string]any{}}, ""), + Entry("happy path", + mcapi.SettingsValues{"liveMigration": map[string]any{"systemNetworkName": "migration"}}, + "migration"), + Entry("non-string value treated as absent", + mcapi.SettingsValues{"liveMigration": map[string]any{"systemNetworkName": 42}}, ""), + ) +}) + +var _ = Describe("liveMigrationValidator", func() { + var ctx context.Context + + BeforeEach(func() { + ctx = context.Background() + }) + + mcWith := func(name string) *mcapi.ModuleConfig { + mc := &mcapi.ModuleConfig{} + mc.Name = moduleConfigName + if name != "" { + mc.Spec.Settings = mcapi.SettingsValues{ + "liveMigration": map[string]any{"systemNetworkName": name}, + } + } + return mc + } + + systemNetwork := func(name string, ready bool) *unstructured.Unstructured { + sn := &unstructured.Unstructured{} + sn.SetGroupVersionKind(systemNetworkGVK) + sn.SetName(name) + status := "False" + if ready { + status = "True" + } + _ = unstructured.SetNestedSlice(sn.Object, []any{ + map[string]any{ + "type": conditionTypeReady, + "status": status, + }, + }, "status", "conditions") + return sn + } + + It("accepts MC without liveMigration block", func() { + v := liveMigrationValidator{client: fake.NewClientBuilder().Build()} + warns, err := v.validate(ctx, mcWith("")) + Expect(err).NotTo(HaveOccurred()) + Expect(warns).To(BeNil()) + }) + + It("rejects when SystemNetwork is not found", func() { + scheme := runtime.NewScheme() + v := liveMigrationValidator{client: fake.NewClientBuilder().WithScheme(scheme).Build()} + _, err := v.validate(ctx, mcWith("missing")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("SystemNetwork not found")) + }) + + It("accepts when SystemNetwork is Ready", func() { + c := fake.NewClientBuilder().WithObjects(systemNetwork("migration", true)).Build() + v := liveMigrationValidator{client: c} + warns, err := v.validate(ctx, mcWith("migration")) + Expect(err).NotTo(HaveOccurred()) + Expect(warns).To(BeNil()) + }) + + It("rejects when SystemNetwork is not Ready", func() { + c := fake.NewClientBuilder().WithObjects(systemNetwork("migration", false)).Build() + v := liveMigrationValidator{client: c} + _, err := v.validate(ctx, mcWith("migration")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("is not Ready")) + }) + + It("rejects when SystemNetwork has no Ready condition at all", func() { + sn := &unstructured.Unstructured{} + sn.SetGroupVersionKind(systemNetworkGVK) + sn.SetName("migration") + c := fake.NewClientBuilder().WithObjects(sn).Build() + v := liveMigrationValidator{client: c} + _, err := v.validate(ctx, mcWith("migration")) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("is not Ready")) + }) +}) + +var _ = Describe("isSystemNetworkReady", func() { + build := func(conditions []any) *unstructured.Unstructured { + sn := &unstructured.Unstructured{} + sn.SetGroupVersionKind(systemNetworkGVK) + _ = unstructured.SetNestedSlice(sn.Object, conditions, "status", "conditions") + return sn + } + + DescribeTable("reads the Ready condition", + func(obj *unstructured.Unstructured, expected bool) { + Expect(isSystemNetworkReady(obj)).To(Equal(expected)) + }, + Entry("Ready=True", + build([]any{map[string]any{"type": conditionTypeReady, "status": conditionStatusTrue}}), + true), + Entry("Ready=False", + build([]any{map[string]any{"type": conditionTypeReady, "status": "False"}}), + false), + Entry("no Ready condition", + build([]any{map[string]any{"type": "Other", "status": "True"}}), + false), + Entry("empty conditions list", build(nil), false), + Entry("no status.conditions key", + &unstructured.Unstructured{Object: map[string]any{}}, false), + ) +}) diff --git a/images/virtualization-artifact/pkg/controller/moduleconfig/moduleconfig_webhook.go b/images/virtualization-artifact/pkg/controller/moduleconfig/moduleconfig_webhook.go index 66e3198cbb..c0ac79ce2e 100644 --- a/images/virtualization-artifact/pkg/controller/moduleconfig/moduleconfig_webhook.go +++ b/images/virtualization-artifact/pkg/controller/moduleconfig/moduleconfig_webhook.go @@ -49,13 +49,18 @@ func NewModuleConfigValidator(client client.Client, clusterSubnets *appconfig.Cl reduceCIDRs := newRemoveCIDRsValidator(client) viStorageClasses := newViStorageClassValidator(client) dvcrValidator := newDvcrValidator(client) + liveMigration := newLiveMigrationValidator(client) return validator.NewValidator[*mcapi.ModuleConfig](logger). WithPredicate(&validator.Predicate[*mcapi.ModuleConfig]{ + Create: func(newMC *mcapi.ModuleConfig) bool { + return newMC.GetName() == moduleConfigName + }, Update: func(oldMC, newMC *mcapi.ModuleConfig) bool { return newMC.GetName() == moduleConfigName && oldMC.GetGeneration() != newMC.GetGeneration() }, }). - WithUpdateValidators(cidrs, reduceCIDRs, viStorageClasses, dvcrValidator) + WithCreateValidators(liveMigration). + WithUpdateValidators(cidrs, reduceCIDRs, viStorageClasses, dvcrValidator, liveMigration) } diff --git a/images/virtualization-artifact/pkg/controller/moduleconfig/suite_test.go b/images/virtualization-artifact/pkg/controller/moduleconfig/suite_test.go new file mode 100644 index 0000000000..ed2b9879ab --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/moduleconfig/suite_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package moduleconfig + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestModuleConfigValidators(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ModuleConfig validators Suite") +} diff --git a/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle.go b/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle.go index 9bb21ce859..f7db8cb9d1 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle.go +++ b/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/object" commonvmop "github.com/deckhouse/virtualization-controller/pkg/common/vmop" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" @@ -77,20 +78,22 @@ type Base interface { IsApplicableOrSetFailedPhase(checker genericservice.ApplicableChecker, vmop *v1alpha2.VirtualMachineOperation, vm *v1alpha2.VirtualMachine) bool } type LifecycleHandler struct { - client client.Client - migration *migrationservice.MigrationService - base Base - recorder eventrecord.EventRecorderLogger - progressStrategy migrationprogress.Strategy + client client.Client + migration *migrationservice.MigrationService + base Base + recorder eventrecord.EventRecorderLogger + progressStrategy migrationprogress.Strategy + systemNetworkName string } -func NewLifecycleHandler(client client.Client, migration *migrationservice.MigrationService, base Base, recorder eventrecord.EventRecorderLogger) *LifecycleHandler { +func NewLifecycleHandler(client client.Client, migration *migrationservice.MigrationService, base Base, recorder eventrecord.EventRecorderLogger, systemNetworkName string) *LifecycleHandler { return &LifecycleHandler{ - client: client, - migration: migration, - base: base, - recorder: recorder, - progressStrategy: migrationprogress.NewProgress(), + client: client, + migration: migration, + base: base, + recorder: recorder, + progressStrategy: migrationprogress.NewProgress(), + systemNetworkName: systemNetworkName, } } @@ -202,6 +205,21 @@ func (h LifecycleHandler) Handle(ctx context.Context, vmop *v1alpha2.VirtualMach return reconcile.Result{}, nil } + // Check if SystemNetwork is configured and ready + if h.systemNetworkName != "" { + if msg, ok := h.checkMigrationNetwork(ctx, vm); !ok { + vmop.Status.Phase = v1alpha2.VMOPPhaseFailed + h.recorder.Event(vmop, corev1.EventTypeWarning, v1alpha2.ReasonErrVMOPFailed, msg) + conditions.SetCondition( + completedCond. + Reason(vmopcondition.ReasonMigrationNetworkUnavailable). + Status(metav1.ConditionFalse). + Message(msg), + &vmop.Status.Conditions) + return reconcile.Result{}, nil + } + } + // 6.1 Check if force flag is applicable for effective liveMigrationPolicy. msg, isApplicable := h.isApplicableForLiveMigrationPolicy(vmop, vm) if !isApplicable { @@ -729,6 +747,24 @@ func isContainerCreating(pod *corev1.Pod) bool { return false } +func (h *LifecycleHandler) checkMigrationNetwork(ctx context.Context, vm *v1alpha2.VirtualMachine) (string, bool) { + nodeName := vm.Status.Node + if nodeName == "" { + return "Virtual machine is not scheduled to any node", false + } + var node corev1.Node + if err := h.client.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + return fmt.Sprintf("failed to get source node %q: %v", nodeName, err), false + } + if node.Annotations[annotations.AnnMigrationIface] == "" { + return fmt.Sprintf( + "source node %q has no dedicated migration interface for network %q", + nodeName, h.systemNetworkName, + ), false + } + return "", true +} + func getPodPendingUnschedulableMessage(pod *corev1.Pod) (string, bool) { if pod == nil { return "", false diff --git a/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle_test.go b/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle_test.go index e821efab49..3d4c7fef39 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle_test.go +++ b/images/virtualization-artifact/pkg/controller/vmop/migration/internal/handler/lifecycle_test.go @@ -33,6 +33,7 @@ import ( vmbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vm" vmopbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vmop" + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" "github.com/deckhouse/virtualization-controller/pkg/common/testutil" "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" "github.com/deckhouse/virtualization-controller/pkg/controller/reconciler" @@ -135,7 +136,7 @@ var _ = Describe("LifecycleHandler", func() { migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -235,7 +236,7 @@ var _ = Describe("LifecycleHandler", func() { migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -275,7 +276,7 @@ var _ = Describe("LifecycleHandler", func() { migrationService := service.NewMigrationService(fakeClient, featureGate) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err = h.Handle(ctx, vmop) if targetMigrationEnabled { @@ -481,7 +482,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -506,7 +507,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -525,7 +526,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -549,7 +550,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -572,7 +573,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -597,7 +598,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") h.progressStrategy = stub _, err := h.Handle(ctx, srv.Changed()) @@ -626,7 +627,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") h.progressStrategy = stub _, err := h.Handle(ctx, srv.Changed()) @@ -661,7 +662,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -686,7 +687,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -708,7 +709,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -733,7 +734,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -759,7 +760,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -794,7 +795,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -832,7 +833,7 @@ var _ = Describe("LifecycleHandler", func() { fakeClient, srv = setupEnvironment(vmop, vm, mig) migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) - h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock) + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") _, err := h.Handle(ctx, srv.Changed()) Expect(err).NotTo(HaveOccurred()) @@ -976,6 +977,150 @@ var _ = Describe("LifecycleHandler", func() { ), ) + Describe("checkMigrationNetwork", func() { + const ( + sourceNode = "node-1" + ifName = "eth0.999" + ) + + makeNode := func(name, annotationVal string) *corev1.Node { + n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: name}} + if annotationVal != "" { + n.Annotations = map[string]string{annotations.AnnMigrationIface: annotationVal} + } + return n + } + + It("returns ok when source node has the annotation", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + fakeClient, err := testutil.NewFakeClientWithObjects(vm, makeNode(sourceNode, ifName)) + Expect(err).NotTo(HaveOccurred()) + + h := NewLifecycleHandler(fakeClient, nil, nil, recorderMock, "migration") + msg, ok := h.checkMigrationNetwork(ctx, vm) + Expect(ok).To(BeTrue()) + Expect(msg).To(BeEmpty()) + }) + + It("refuses when source node has no annotation", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + fakeClient, err := testutil.NewFakeClientWithObjects(vm, makeNode(sourceNode, "")) + Expect(err).NotTo(HaveOccurred()) + + h := NewLifecycleHandler(fakeClient, nil, nil, recorderMock, "migration") + msg, ok := h.checkMigrationNetwork(ctx, vm) + Expect(ok).To(BeFalse()) + Expect(msg).To(ContainSubstring(sourceNode)) + Expect(msg).To(ContainSubstring("migration")) + }) + + It("refuses when VM is not scheduled to any node", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = "" + fakeClient, err := testutil.NewFakeClientWithObjects(vm) + Expect(err).NotTo(HaveOccurred()) + + h := NewLifecycleHandler(fakeClient, nil, nil, recorderMock, "migration") + msg, ok := h.checkMigrationNetwork(ctx, vm) + Expect(ok).To(BeFalse()) + Expect(msg).To(ContainSubstring("not scheduled")) + }) + + It("refuses when source node lookup fails", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + // no Node object in the fake client → Get returns NotFound + fakeClient, err := testutil.NewFakeClientWithObjects(vm) + Expect(err).NotTo(HaveOccurred()) + + h := NewLifecycleHandler(fakeClient, nil, nil, recorderMock, "migration") + msg, ok := h.checkMigrationNetwork(ctx, vm) + Expect(ok).To(BeFalse()) + Expect(msg).To(ContainSubstring(sourceNode)) + }) + }) + + Describe("Handle migration network pre-flight", func() { + const ( + sourceNode = "node-1" + ifName = "eth0.999" + ) + + makeNode := func(annotationVal string) *corev1.Node { + n := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: sourceNode}} + if annotationVal != "" { + n.Annotations = map[string]string{annotations.AnnMigrationIface: annotationVal} + } + return n + } + + It("fails the VMOP with MigrationNetworkUnavailable when source node lacks the annotation", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + vmop := newVMOPMigrate() + + var err error + fakeClient, srv = setupEnvironment(vmop, vm, makeNode("")) + migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) + base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) + + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "migration") + _, err = h.Handle(ctx, srv.Changed()) + Expect(err).NotTo(HaveOccurred()) + + Expect(srv.Changed().Status.Phase).To(Equal(v1alpha2.VMOPPhaseFailed)) + completed, found := conditions.GetCondition(vmopcondition.TypeCompleted, srv.Changed().Status.Conditions) + Expect(found).To(BeTrue()) + Expect(completed.Status).To(Equal(metav1.ConditionFalse)) + Expect(completed.Reason).To(Equal(vmopcondition.ReasonMigrationNetworkUnavailable.String())) + Expect(completed.Message).To(ContainSubstring(sourceNode)) + }) + + It("skips the pre-flight entirely when systemNetworkName is empty", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + vmop := newVMOPMigrate() + + // no migration-iface annotation on the node, but systemNetworkName="" + // means pre-flight is bypassed, so the VMOP must NOT fail + fakeClient, srv = setupEnvironment(vmop, vm, makeNode("")) + migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) + base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) + + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "") + _, err := h.Handle(ctx, srv.Changed()) + Expect(err).NotTo(HaveOccurred()) + + completed, found := conditions.GetCondition(vmopcondition.TypeCompleted, srv.Changed().Status.Conditions) + if found { + Expect(completed.Reason).NotTo(Equal(vmopcondition.ReasonMigrationNetworkUnavailable.String()), + "pre-flight must be skipped when systemNetworkName is empty") + } + }) + + It("passes the pre-flight when source node has the annotation", func() { + vm := newVM(v1alpha2.AlwaysSafeMigrationPolicy) + vm.Status.Node = sourceNode + vmop := newVMOPMigrate() + + fakeClient, srv = setupEnvironment(vmop, vm, makeNode(ifName)) + migrationService := service.NewMigrationService(fakeClient, featuregates.Default()) + base := genericservice.NewBaseVMOPService(fakeClient, recorderMock) + + h := NewLifecycleHandler(fakeClient, migrationService, base, recorderMock, "migration") + _, err := h.Handle(ctx, srv.Changed()) + Expect(err).NotTo(HaveOccurred()) + + completed, found := conditions.GetCondition(vmopcondition.TypeCompleted, srv.Changed().Status.Conditions) + if found { + Expect(completed.Reason).NotTo(Equal(vmopcondition.ReasonMigrationNetworkUnavailable.String()), + "pre-flight must pass when annotation is present") + } + }) + }) + It("should use humanized message for migration failed condition", func() { mig := newSimpleMigration("test", name) mig.Status.Conditions = []virtv1.VirtualMachineInstanceMigrationCondition{{ diff --git a/images/virtualization-artifact/pkg/controller/vmop/migration/migration_controller.go b/images/virtualization-artifact/pkg/controller/vmop/migration/migration_controller.go index ff387d7934..a74d331f18 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/migration/migration_controller.go +++ b/images/virtualization-artifact/pkg/controller/vmop/migration/migration_controller.go @@ -34,7 +34,7 @@ const ( controllerName = "vmop-migration-controller" ) -func NewController(client client.Client, mgr manager.Manager, featureGate featuregate.FeatureGate) *Controller { +func NewController(client client.Client, mgr manager.Manager, featureGate featuregate.FeatureGate, systemNetworkName string) *Controller { recorder := eventrecord.NewEventRecorderLogger(mgr, controllerName) baseSvc := genericservice.NewBaseVMOPService(client, recorder) migration := service.NewMigrationService(client, featureGate) @@ -46,7 +46,7 @@ func NewController(client client.Client, mgr manager.Manager, featureGate featur }, handlers: []reconciler.Handler[*v1alpha2.VirtualMachineOperation]{ handler.NewDeletionHandler(migration), - handler.NewLifecycleHandler(client, migration, baseSvc, recorder), + handler.NewLifecycleHandler(client, migration, baseSvc, recorder, systemNetworkName), }, } } diff --git a/images/virtualization-artifact/pkg/controller/vmop/vmop_controller.go b/images/virtualization-artifact/pkg/controller/vmop/vmop_controller.go index e1ebf0f76e..98793263eb 100644 --- a/images/virtualization-artifact/pkg/controller/vmop/vmop_controller.go +++ b/images/virtualization-artifact/pkg/controller/vmop/vmop_controller.go @@ -52,12 +52,13 @@ func SetupController( ctx context.Context, mgr manager.Manager, log *log.Logger, + systemNetworkName string, ) error { client := mgr.GetClient() controllers := []SubController{ powerstate.NewController(client, mgr), - migration.NewController(client, mgr, featuregates.Default()), + migration.NewController(client, mgr, featuregates.Default(), systemNetworkName), snapshot.NewController(client, mgr), } diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index e46d2362fc..9e0278ad16 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -252,6 +252,19 @@ properties: type: string minLength: 1 x-examples: ["sc-1", "sc-2"] + liveMigration: + type: object + description: | + Live migration network configuration. + properties: + systemNetworkName: + type: string + minLength: 1 + description: | + Name of a `SystemNetwork` (sdn module) whose per-host IP addresses are used for VM live migration traffic between `virt-handler` pods. + + Requires the `sdn` module. When unset, live migration traffic flows over the default node network. + x-examples: ["migration"] logLevel: type: string description: | diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 5aaf65aae6..cca08db35c 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -157,6 +157,15 @@ properties: default: false description: | Включение контроллера аудита. + liveMigration: + description: | + Параметры сети живой миграции виртуальных машин. + properties: + systemNetworkName: + description: | + Имя ресурса `SystemNetwork` (из модуля sdn), IP-адреса которого на узлах будут использоваться для трафика живой миграции между подами `virt-handler`. + + Требуется включенный модуль `sdn`. Если параметр не задан, трафик миграции идёт по сети узла по умолчанию. logLevel: type: string description: | diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index 3d799bf0fd..46462a6e65 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -88,16 +88,10 @@ true value: "24h" - name: GC_VM_POD_SCHEDULE value: "0 0 * * *" -{{- if (hasKey .Values.virtualization.internal.moduleConfig "liveMigration") }} -- name: LIVE_MIGRATION_BANDWIDTH_PER_NODE - value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.bandwidthPerNode | quote }} -- name: LIVE_MIGRATION_MAX_MIGRATIONS_PER_NODE - value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.maxMigrationsPerNode | quote }} -- name: LIVE_MIGRATION_NETWORK - value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.network | quote }} -{{- if (hasKey .Values.virtualization.internal.moduleConfig.liveMigration "dedicated") }} -- name: LIVE_MIGRATION_DEDICATED_INTERFACE_NAME - value: {{ .Values.virtualization.internal.moduleConfig.liveMigration.dedicated.interfaceName | quote }} +{{- if (hasKey (.Values.virtualization | default dict) "liveMigration") }} +{{- if .Values.virtualization.liveMigration.systemNetworkName }} +- name: MIGRATION_SYSTEM_NETWORK_NAME + value: {{ .Values.virtualization.liveMigration.systemNetworkName | quote }} {{- end }} {{- end }} - name: METRICS_BIND_ADDRESS diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index 75e1297afe..d1ac276578 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -108,6 +108,18 @@ rules: resources: - nodes verbs: + - get + - list + - watch + - patch +- apiGroups: + - network.deckhouse.io + resources: + - systemnetworks + - systemnetworknodenetworkinterfaceattachments + - nodenetworkinterfaces + verbs: + - get - list - watch - apiGroups: diff --git a/test/e2e/vm/migration_dedicated_network.go b/test/e2e/vm/migration_dedicated_network.go new file mode 100644 index 0000000000..876c561f89 --- /dev/null +++ b/test/e2e/vm/migration_dedicated_network.go @@ -0,0 +1,289 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vm + +import ( + "context" + "encoding/json" + "fmt" + "net" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization-controller/pkg/builder/vd" + "github.com/deckhouse/virtualization-controller/pkg/builder/vm" + vmopbuilder "github.com/deckhouse/virtualization-controller/pkg/builder/vmop" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/test/e2e/internal/framework" + "github.com/deckhouse/virtualization/test/e2e/internal/object" + "github.com/deckhouse/virtualization/test/e2e/internal/precheck" + "github.com/deckhouse/virtualization/test/e2e/internal/rewrite" + "github.com/deckhouse/virtualization/test/e2e/internal/util" +) + +// migrationIfaceAnnotation must match the constant in pkg/common/annotations. +const migrationIfaceAnnotation = "virtualization.deckhouse.io/migration-iface" + +var ( + systemNetworkGVK = schema.GroupVersionKind{ + Group: "network.deckhouse.io", + Version: "v1alpha1", + Kind: "SystemNetwork", + } + clusterIPAddressPoolGVK = schema.GroupVersionKind{ + Group: "network.deckhouse.io", + Version: "v1alpha1", + Kind: "ClusterIPAddressPool", + } +) + +var _ = Describe("VirtualMachineMigrationDedicatedNetwork", Label(precheck.PrecheckSDN), func() { + var ( + f *framework.Framework + ctx context.Context + + systemNetworkName string + poolCIDRs []*net.IPNet + + vdRoot *v1alpha2.VirtualDisk + vmObj *v1alpha2.VirtualMachine + vmop *v1alpha2.VirtualMachineOperation + ) + + BeforeEach(func() { + ctx = context.Background() + f = framework.NewFramework("vm-migration-dedicated-network") + DeferCleanup(f.After) + + f.Before() + }) + + It("routes live migration traffic over the configured SystemNetwork", func() { + By("Reading liveMigration.systemNetworkName from the virtualization ModuleConfig", func() { + systemNetworkName = getConfiguredSystemNetworkName(ctx, f) + if systemNetworkName == "" { + Skip("ModuleConfig virtualization has no spec.settings.liveMigration.systemNetworkName; configure it to run this test") + } + }) + + By(fmt.Sprintf("Verifying SystemNetwork %q is Ready and discovering its IPAM pool CIDRs", systemNetworkName), func() { + sn := &unstructured.Unstructured{} + sn.SetGroupVersionKind(systemNetworkGVK) + err := f.GenericClient().Get(ctx, crclient.ObjectKey{Name: systemNetworkName}, sn) + Expect(err).NotTo(HaveOccurred(), + "SystemNetwork %q is referenced by the ModuleConfig but not present on the cluster", + systemNetworkName) + Expect(isSystemNetworkReady(sn)).To(BeTrue(), + "SystemNetwork %q must be Ready", systemNetworkName) + + poolName, found, err := unstructured.NestedString(sn.Object, "spec", "ipam", "clusterIPAddressPoolName") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue(), + "SystemNetwork %q must declare spec.ipam.clusterIPAddressPoolName for IP-pool verification", + systemNetworkName) + + poolCIDRs = getClusterIPAddressPoolCIDRs(ctx, f, poolName) + Expect(poolCIDRs).NotTo(BeEmpty(), + "ClusterIPAddressPool %q has no parsable spec.pools[].network entries", poolName) + }) + + By("Waiting for migration-iface annotations to populate on every node", func() { + Eventually(func(g Gomega) { + nodes := &corev1.NodeList{} + g.Expect(f.GenericClient().List(ctx, nodes)).To(Succeed()) + g.Expect(nodes.Items).NotTo(BeEmpty()) + for _, n := range nodes.Items { + if !nodeIsReady(&n) { + continue + } + g.Expect(n.Annotations).To(HaveKey(migrationIfaceAnnotation), + "node %q must carry the migration-iface annotation", n.Name) + g.Expect(n.Annotations[migrationIfaceAnnotation]).NotTo(BeEmpty(), + "node %q annotation must not be empty", n.Name) + } + }).WithPolling(2 * time.Second).WithTimeout(framework.MiddleTimeout).Should(Succeed()) + }) + + By("Creating a VM", func() { + vdRoot = object.NewVDFromCVI("vd-root-alpine", f.Namespace().Name, object.PrecreatedCVIAlpineBIOS, + vd.WithSize(ptr.To(resource.MustParse("10Gi"))), + ) + vmObj = object.NewMinimalVM("vm-migration-dn", f.Namespace().Name, + vm.WithBlockDeviceRefs(v1alpha2.BlockDeviceSpecRef{ + Kind: v1alpha2.VirtualDiskKind, + Name: vdRoot.Name, + }), + vm.WithCPU(1, ptr.To("100%")), + vm.WithBootloader(v1alpha2.BIOS), + vm.WithProvisioningUserData(object.AlpineCloudInit), + vm.WithLiveMigrationPolicy(v1alpha2.PreferSafeMigrationPolicy), + ) + Expect(f.CreateWithDeferredDeletion(ctx, vdRoot, vmObj)).To(Succeed()) + util.UntilObjectPhase(ctx, string(v1alpha2.MachineRunning), framework.LongTimeout, vmObj) + util.UntilSSHReady(f, vmObj, framework.LongTimeout) + }) + + By("Triggering live migration via VMOP", func() { + vmop = vmopbuilder.New( + vmopbuilder.WithGenerateName("vmop-migrate-dn-"), + vmopbuilder.WithNamespace(f.Namespace().Name), + vmopbuilder.WithType(v1alpha2.VMOPTypeMigrate), + vmopbuilder.WithVirtualMachine(vmObj.Name), + ) + Expect(f.CreateWithDeferredDeletion(ctx, vmop)).To(Succeed()) + + Eventually(func(g Gomega) { + g.Expect(f.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(vmObj), vmObj)).To(Succeed()) + util.SkipIfKnownMigrationFailure(vmObj) + + g.Expect(f.GenericClient().Get(ctx, crclient.ObjectKeyFromObject(vmop), vmop)).To(Succeed()) + g.Expect(vmop.Status.Phase).To(Equal(v1alpha2.VMOPPhaseCompleted)) + }).WithPolling(time.Second).WithTimeout(framework.LongTimeout).Should(Succeed()) + }) + + By("Verifying the migration target address is in the SystemNetwork IPAM pool", func() { + intvmi := &rewrite.VirtualMachineInstance{} + Expect(framework.GetClients().RewriteClient().Get( + ctx, vmObj.Name, intvmi, rewrite.InNamespace(f.Namespace().Name), + )).To(Succeed()) + + state := intvmi.Status.MigrationState + Expect(state).NotTo(BeNil(), "VMI must have a MigrationState after a completed migration") + Expect(state.Completed).To(BeTrue()) + Expect(state.TargetNode).NotTo(BeEmpty()) + Expect(state.TargetNodeAddress).NotTo(BeEmpty(), + "target node address must be advertised by virt-handler") + + ip := net.ParseIP(state.TargetNodeAddress) + Expect(ip).NotTo(BeNil(), + "target node address %q is not a parsable IP", state.TargetNodeAddress) + Expect(cidrsContain(poolCIDRs, ip)).To(BeTrue(), + "migration target address %s is not within SystemNetwork %q IPAM pool CIDRs %v - feature did not route over SystemNetwork", + state.TargetNodeAddress, systemNetworkName, formatCIDRs(poolCIDRs)) + + node := &corev1.Node{} + Expect(f.GenericClient().Get(ctx, crclient.ObjectKey{Name: state.TargetNode}, node)).To(Succeed()) + Expect(node.Annotations[migrationIfaceAnnotation]).NotTo(BeEmpty(), + "target node %q has no migration-iface annotation", state.TargetNode) + }) + }) +}) + +func getConfiguredSystemNetworkName(ctx context.Context, f *framework.Framework) string { + GinkgoHelper() + mc, err := f.GetModuleConfig(ctx, "virtualization") + if k8serrors.IsNotFound(err) { + return "" + } + Expect(err).NotTo(HaveOccurred()) + + raw, err := json.Marshal(mc.Spec.Settings) + Expect(err).NotTo(HaveOccurred()) + settings := map[string]any{} + Expect(json.Unmarshal(raw, &settings)).To(Succeed()) + + lm, ok := settings["liveMigration"].(map[string]any) + if !ok { + return "" + } + name, _ := lm["systemNetworkName"].(string) + return name +} + +func isSystemNetworkReady(obj *unstructured.Unstructured) bool { + conds, found, err := unstructured.NestedSlice(obj.Object, "status", "conditions") + if err != nil || !found { + return false + } + for _, c := range conds { + m, ok := c.(map[string]any) + if !ok { + continue + } + t, _ := m["type"].(string) + s, _ := m["status"].(string) + if t == "Ready" && s == "True" { + return true + } + } + return false +} + +func nodeIsReady(n *corev1.Node) bool { + for _, c := range n.Status.Conditions { + if c.Type == corev1.NodeReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +func getClusterIPAddressPoolCIDRs(ctx context.Context, f *framework.Framework, poolName string) []*net.IPNet { + GinkgoHelper() + pool := &unstructured.Unstructured{} + pool.SetGroupVersionKind(clusterIPAddressPoolGVK) + Expect(f.GenericClient().Get(ctx, crclient.ObjectKey{Name: poolName}, pool)).To(Succeed(), + "ClusterIPAddressPool %q must exist", poolName) + + pools, found, err := unstructured.NestedSlice(pool.Object, "spec", "pools") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue(), "ClusterIPAddressPool %q must have spec.pools", poolName) + + cidrs := make([]*net.IPNet, 0, len(pools)) + for _, p := range pools { + m, ok := p.(map[string]any) + if !ok { + continue + } + network, _ := m["network"].(string) + if network == "" { + continue + } + _, ipnet, err := net.ParseCIDR(network) + if err != nil { + continue + } + cidrs = append(cidrs, ipnet) + } + return cidrs +} + +func cidrsContain(cidrs []*net.IPNet, ip net.IP) bool { + for _, c := range cidrs { + if c.Contains(ip) { + return true + } + } + return false +} + +func formatCIDRs(cidrs []*net.IPNet) []string { + out := make([]string, 0, len(cidrs)) + for _, c := range cidrs { + out = append(out, c.String()) + } + return out +}