From 9397cb87f08ef446b7f93e455a77148c07303206 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Sat, 5 Jul 2025 12:43:09 +0300 Subject: [PATCH 01/32] t Signed-off-by: Valeriy Khorunzhin --- templates/admission-policy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/admission-policy.yaml b/templates/admission-policy.yaml index d59c0bc61d..baae8568fd 100644 --- a/templates/admission-policy.yaml +++ b/templates/admission-policy.yaml @@ -36,6 +36,7 @@ spec: resources: ["*"] validations: - expression: | + true || request.userInfo.username.startsWith("system:serviceaccount:kube-system:") || request.userInfo.username.startsWith("system:serviceaccount:d8-system:") || request.userInfo.username in [ From a5931e278a72920c0b5f0edab7745125888f07be Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Wed, 9 Jul 2025 18:57:22 +0300 Subject: [PATCH 02/32] ttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- .../pkg/controller/vm/internal/sync_kvvm.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index ae40def477..962e7b2330 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ $.ImageName }} final: true -fromImage: distroless +fromImage: ubuntu import: - image: {{ $.ImageName }}-binaries add: /relocate diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 20c916592a..b3c9c64626 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -302,6 +302,11 @@ func (h *SyncKvvmHandler) createKVVM(ctx context.Context, s state.VirtualMachine return fmt.Errorf("failed to make the internal virtual machine: %w", err) } + kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ + DeviceName: "kubevirt.io/usbstore", + Name: "usb322", + }) + err = h.client.Create(ctx, kvvm) if err != nil { if k8serrors.IsAlreadyExists(err) { From e12d6eff7ca2ecb88832a070ec18784cd34dc1a6 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Wed, 9 Jul 2025 19:01:08 +0300 Subject: [PATCH 03/32] tttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 962e7b2330..50efec09c3 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ $.ImageName }} final: true -fromImage: ubuntu +fromImage: ubuntu-latest import: - image: {{ $.ImageName }}-binaries add: /relocate From ad6f60cd2ea748153a41a1167db364883510fbcd Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Wed, 9 Jul 2025 19:03:50 +0300 Subject: [PATCH 04/32] ttttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 50efec09c3..bc5ae823ac 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ $.ImageName }} final: true -fromImage: ubuntu-latest +fromImage: BASE_ALT_P11 import: - image: {{ $.ImageName }}-binaries add: /relocate From b404878ac62dc2a40f1c0dd265bd2a3f3b3c97c3 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Wed, 9 Jul 2025 19:07:46 +0300 Subject: [PATCH 05/32] tttttttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index bc5ae823ac..f4cb01cb01 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ $.ImageName }} final: true -fromImage: BASE_ALT_P11 +fromImage: builder/alpine import: - image: {{ $.ImageName }}-binaries add: /relocate From 1f379a3df0a6956abfe23aa0f0e6a29084b29370 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 01:10:11 +0300 Subject: [PATCH 06/32] distroless with ls Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index f4cb01cb01..8246eeb1f1 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ $.ImageName }} final: true -fromImage: builder/alpine +fromImage: distroless import: - image: {{ $.ImageName }}-binaries add: /relocate @@ -128,6 +128,7 @@ binaries: - /usr/bin/ocsptool - /usr/bin/p11tool - /usr/bin/psktool + - /usr/bin/ls # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 9da73852995c228c8648771717c152f1ad8964cd Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 01:17:31 +0300 Subject: [PATCH 07/32] lsusb Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 8246eeb1f1..47e0cda1d3 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -129,6 +129,7 @@ binaries: - /usr/bin/p11tool - /usr/bin/psktool - /usr/bin/ls + - /usr/bin/lsusb # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From d65559b454b2c75edbc34a8e8d35861c3b7c8b1a Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 01:22:33 +0300 Subject: [PATCH 08/32] whoami Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 47e0cda1d3..c55ed88b8c 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -129,7 +129,7 @@ binaries: - /usr/bin/p11tool - /usr/bin/psktool - /usr/bin/ls - - /usr/bin/lsusb + - /usr/bin/whoami # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 119a2f8009a94d071c3435ab5f45deea6ffc37e2 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 11:49:24 +0300 Subject: [PATCH 09/32] lsusb Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index c55ed88b8c..36f77bb1de 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -120,6 +120,7 @@ packages: - policycoreutils - psmisc - msulogin + - usbutils - iproute2 binaries: # Gnu utils (requared for swtpm) @@ -128,8 +129,13 @@ binaries: - /usr/bin/ocsptool - /usr/bin/p11tool - /usr/bin/psktool + # Debug - /usr/bin/ls - /usr/bin/whoami + - /usr/bin/lsusb + - /usr/bin/usb-devices + - /usr/bin/usbhid-dump + - /usr/bin/usbreset # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 2f3987a342b6a91896253750572ffb2d694ef509 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 12:49:36 +0300 Subject: [PATCH 10/32] tm1 Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 36f77bb1de..f0e4c6868a 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -131,7 +131,7 @@ binaries: - /usr/bin/psktool # Debug - /usr/bin/ls - - /usr/bin/whoami + # - /usr/bin/whoami - /usr/bin/lsusb - /usr/bin/usb-devices - /usr/bin/usbhid-dump From 2fd3a6cf7cbd337a86ab885ed4bed35ba94cb966 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 12:54:58 +0300 Subject: [PATCH 11/32] tmls Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index f0e4c6868a..7b34556d3d 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -130,7 +130,7 @@ binaries: - /usr/bin/p11tool - /usr/bin/psktool # Debug - - /usr/bin/ls + # - /usr/bin/ls # - /usr/bin/whoami - /usr/bin/lsusb - /usr/bin/usb-devices From ba2da07b0442f6af480d8bc276ebc16f2ab3f39d Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 13:01:09 +0300 Subject: [PATCH 12/32] tdellsusb Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 7b34556d3d..459841a1d4 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -132,7 +132,7 @@ binaries: # Debug # - /usr/bin/ls # - /usr/bin/whoami - - /usr/bin/lsusb + # - /usr/bin/lsusb - /usr/bin/usb-devices - /usr/bin/usbhid-dump - /usr/bin/usbreset From 30af70727f203326a196e1b407245ae56532ab97 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 13:08:44 +0300 Subject: [PATCH 13/32] tdelusbdevices Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 459841a1d4..1f97ebf08e 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -133,7 +133,7 @@ binaries: # - /usr/bin/ls # - /usr/bin/whoami # - /usr/bin/lsusb - - /usr/bin/usb-devices + # - /usr/bin/usb-devices - /usr/bin/usbhid-dump - /usr/bin/usbreset # Xorriso (Creates an image of an ISO9660 filesystem) From 2b0f2cc632b5083b592f3f3cbc372608230310fa Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 13:14:53 +0300 Subject: [PATCH 14/32] tdelusbhiddump Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 1f97ebf08e..0daf758d65 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -134,7 +134,7 @@ binaries: # - /usr/bin/whoami # - /usr/bin/lsusb # - /usr/bin/usb-devices - - /usr/bin/usbhid-dump + # - /usr/bin/usbhid-dump - /usr/bin/usbreset # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso From 95692d14d95a04af9f80773d595b8861c3abbaee Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 10 Jul 2025 13:21:39 +0300 Subject: [PATCH 15/32] backhiddumpdelreset Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 0daf758d65..a0876bc2f3 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -134,8 +134,8 @@ binaries: # - /usr/bin/whoami # - /usr/bin/lsusb # - /usr/bin/usb-devices - # - /usr/bin/usbhid-dump - - /usr/bin/usbreset + - /usr/bin/usbhid-dump + # - /usr/bin/usbreset # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 38b6d61b8855561f9e92194675c3f2c42a8fbe6c Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 14 Jul 2025 10:34:18 +0300 Subject: [PATCH 16/32] tttttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 5 ----- .../pkg/controller/vm/internal/sync_kvvm.go | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index a0876bc2f3..041339935e 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -130,12 +130,7 @@ binaries: - /usr/bin/p11tool - /usr/bin/psktool # Debug - # - /usr/bin/ls - # - /usr/bin/whoami - # - /usr/bin/lsusb - # - /usr/bin/usb-devices - /usr/bin/usbhid-dump - # - /usr/bin/usbreset # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index b3c9c64626..b936bbf82c 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -307,6 +307,11 @@ func (h *SyncKvvmHandler) createKVVM(ctx context.Context, s state.VirtualMachine Name: "usb322", }) + kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ + DeviceName: "devices.kubevirt.io/samsung-pm9a", + Name: "nvme-passthrough", + }) + err = h.client.Create(ctx, kvvm) if err != nil { if k8serrors.IsAlreadyExists(err) { From c8107594267bcc17b2156c1c74141052ceeb988c Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 17 Jul 2025 17:13:00 +0300 Subject: [PATCH 17/32] tttttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 041339935e..de4a409f98 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -131,6 +131,10 @@ binaries: - /usr/bin/psktool # Debug - /usr/bin/usbhid-dump + # Debug2 + - /usr/bin/bash + - /usr/bin/ls + - /usr/bin/lsusb # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 6fee4ef203b1f012be8cd6646555cbdf9074cecf Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 17 Jul 2025 18:42:45 +0300 Subject: [PATCH 18/32] pgrep Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index de4a409f98..c525ee2177 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -132,6 +132,7 @@ binaries: # Debug - /usr/bin/usbhid-dump # Debug2 + - /usr/bin/pgrep - /usr/bin/bash - /usr/bin/ls - /usr/bin/lsusb From 1b5b1a261a0e7bbcf81488b29d4827bc2b401f95 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 17 Jul 2025 19:30:47 +0300 Subject: [PATCH 19/32] htop Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index c525ee2177..6ab0cac45e 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -121,6 +121,7 @@ packages: - psmisc - msulogin - usbutils + - htop - iproute2 binaries: # Gnu utils (requared for swtpm) @@ -136,6 +137,7 @@ binaries: - /usr/bin/bash - /usr/bin/ls - /usr/bin/lsusb + - /usr/bin/htop # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 5aede0fda0ebd7cd176de8bc13f67df950f434da Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 17 Jul 2025 19:44:08 +0300 Subject: [PATCH 20/32] ps Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 6ab0cac45e..e9633aaaca 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -138,6 +138,7 @@ binaries: - /usr/bin/ls - /usr/bin/lsusb - /usr/bin/htop + - /usr/bin/ps # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 0b5e5852cb72e59ef5084951ab2c51bdb4437d7d Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 17 Jul 2025 19:52:46 +0300 Subject: [PATCH 21/32] cat Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index e9633aaaca..3a6a8caa8f 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -139,6 +139,7 @@ binaries: - /usr/bin/lsusb - /usr/bin/htop - /usr/bin/ps + - /usr/bin/cat # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm From 71a5c2319878ff0d7becf0294174f109bad1ca42 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Fri, 18 Jul 2025 12:24:17 +0300 Subject: [PATCH 22/32] comment passthrough hardcode Signed-off-by: Valeriy Khorunzhin --- .../pkg/controller/vm/internal/sync_kvvm.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index b936bbf82c..68df6ac0b2 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -302,15 +302,15 @@ func (h *SyncKvvmHandler) createKVVM(ctx context.Context, s state.VirtualMachine return fmt.Errorf("failed to make the internal virtual machine: %w", err) } - kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ - DeviceName: "kubevirt.io/usbstore", - Name: "usb322", - }) - - kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ - DeviceName: "devices.kubevirt.io/samsung-pm9a", - Name: "nvme-passthrough", - }) + // kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ + // DeviceName: "kubevirt.io/usbstore", + // Name: "usb322", + // }) + + // kvvm.Spec.Template.Spec.Domain.Devices.HostDevices = append(kvvm.Spec.Template.Spec.Domain.Devices.HostDevices, virtv1.HostDevice{ + // DeviceName: "devices.kubevirt.io/samsung-pm9a", + // Name: "nvme-passthrough", + // }) err = h.client.Create(ctx, kvvm) if err != nil { From 7ae6767ea58ab6df3538f58a56e89aaa1f51181e Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 24 Jul 2025 16:12:06 +0300 Subject: [PATCH 23/32] debug Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 3 + .../pkg/controller/vm/internal/sync_kvvm.go | 2 + usbredirbridgesample/go.mod | 37 ++++++ usbredirbridgesample/main.go | 113 ++++++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 usbredirbridgesample/go.mod create mode 100644 usbredirbridgesample/main.go diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 3a6a8caa8f..be85b7ebe0 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -123,6 +123,9 @@ packages: - usbutils - htop - iproute2 + # debug + - libgbm + - libusbredir binaries: # Gnu utils (requared for swtpm) - /usr/bin/certtool diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 68df6ac0b2..23bc610f1e 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -312,6 +312,8 @@ func (h *SyncKvvmHandler) createKVVM(ctx context.Context, s state.VirtualMachine // Name: "nvme-passthrough", // }) + // kvvm.Spec.Template.Spec.Domain.Devices.ClientPassthrough = &virtv1.ClientPassthroughDevices{} + err = h.client.Create(ctx, kvvm) if err != nil { if k8serrors.IsAlreadyExists(err) { diff --git a/usbredirbridgesample/go.mod b/usbredirbridgesample/go.mod new file mode 100644 index 0000000000..f9af24ebef --- /dev/null +++ b/usbredirbridgesample/go.mod @@ -0,0 +1,37 @@ +module usbredirbridge + +go 1.24.0 + +toolchain go1.24.5 + +require ( + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 + k8s.io/client-go v0.33.3 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.9.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apimachinery v0.33.3 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/usbredirbridgesample/main.go b/usbredirbridgesample/main.go new file mode 100644 index 0000000000..ffe6ed0d81 --- /dev/null +++ b/usbredirbridgesample/main.go @@ -0,0 +1,113 @@ +package main + +import ( + "fmt" + "log" + "net" + "net/http" + "net/url" + "os" + "strings" + + "github.com/gorilla/websocket" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func main() { + //----------------------------------------------------------------- + // 1. kube‑config from ~/.kube/config + cfgFile := fmt.Sprintf("%s/.kube/config", os.Getenv("HOME")) + kcfg, err := clientcmd.BuildConfigFromFlags("", cfgFile) + if err != nil { + log.Fatalf("kubeconfig: %v", err) + } + + //----------------------------------------------------------------- + // 2. Собираем wss‑URL к subresource usbredir + ns, name := "default", "ubuntu-vm" + host := strings.TrimPrefix(kcfg.Host, "https://") + wsURL := url.URL{ + Scheme: "wss", + Host: host, + Path: fmt.Sprintf("/apis/subresources.kubevirt.io/v1/namespaces/%s/"+ + "virtualmachineinstances/%s/usbredir", ns, name), + } + + //----------------------------------------------------------------- + // 3. Создаём TLS‑конфиг из kube‑config  (функция rest.TLSConfigFor) + tlsCfg, err := rest.TLSConfigFor(kcfg) // :contentReference[oaicite:0]{index=0} + if err != nil { + log.Fatalf("TLS config: %v", err) + } + + dialer := websocket.Dialer{ + TLSClientConfig: tlsCfg, + Subprotocols: []string{"binary.kubevirt.io"}, + } + + //----------------------------------------------------------------- + // 4. TCP‑порт 4000 — сюда подключается usbredirect‑клиент + l, err := net.Listen("tcp", "127.0.0.1:4000") + if err != nil { + log.Fatalf("listen 4000: %v", err) + } + log.Printf("usbredir proxy ready on localhost:4000") + + for { + tcpConn, err := l.Accept() + if err != nil { + log.Printf("accept: %v", err) + continue + } + go handle(tcpConn, dialer, wsURL, kcfg.BearerToken) + } +} + +// handle: на каждое TCP‑подключение открываем свой WebSocket +func handle(tcp net.Conn, dialer websocket.Dialer, wsURL url.URL, token string) { + defer tcp.Close() + + hdr := http.Header{} + if token != "" { + hdr.Set("Authorization", "Bearer "+token) + } + + ws, _, err := dialer.Dial(wsURL.String(), hdr) + if err != nil { + log.Printf("dial websocket: %v", err) + return + } + defer ws.Close() + + // WebSocket <‑‑> TCP копируем параллельно + go func() { + // TCP → WS + for { + buf := make([]byte, 32*1024) + n, err := tcp.Read(buf) + if n > 0 { + if werr := ws.WriteMessage(websocket.BinaryMessage, buf[:n]); werr != nil { + break + } + } + if err != nil { + break + } + } + ws.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage( + websocket.CloseNormalClosure, "")) + }() + + // WS → TCP + for { + _, data, err := ws.ReadMessage() + if err != nil { + break + } + if _, err := tcp.Write(data); err != nil { + break + } + } + log.Printf("client disconnected") +} From f489514e599bfb9a40a40d0368a9740386f76cfc Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 24 Jul 2025 16:26:32 +0300 Subject: [PATCH 24/32] debug Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 6 +++--- .../pkg/controller/vm/internal/sync_kvvm.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index be85b7ebe0..db2b14b335 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -107,6 +107,9 @@ libs: - libclocale - libLLVMSPIRVLib-devel - libisofs + # debug + - libgbm + - libusbredir packages: - ethtool - fdisk @@ -123,9 +126,6 @@ packages: - usbutils - htop - iproute2 - # debug - - libgbm - - libusbredir binaries: # Gnu utils (requared for swtpm) - /usr/bin/certtool diff --git a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go index 23bc610f1e..742613fa96 100644 --- a/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go +++ b/images/virtualization-artifact/pkg/controller/vm/internal/sync_kvvm.go @@ -312,7 +312,7 @@ func (h *SyncKvvmHandler) createKVVM(ctx context.Context, s state.VirtualMachine // Name: "nvme-passthrough", // }) - // kvvm.Spec.Template.Spec.Domain.Devices.ClientPassthrough = &virtv1.ClientPassthroughDevices{} + kvvm.Spec.Template.Spec.Domain.Devices.ClientPassthrough = &virtv1.ClientPassthroughDevices{} err = h.client.Create(ctx, kvvm) if err != nil { From f11b393e373d5e4880a1a800cb07f306fdb75d9a Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 24 Jul 2025 17:08:39 +0300 Subject: [PATCH 25/32] is it? Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index db2b14b335..fd7b5eb14b 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -107,9 +107,6 @@ libs: - libclocale - libLLVMSPIRVLib-devel - libisofs - # debug - - libgbm - - libusbredir packages: - ethtool - fdisk @@ -325,6 +322,7 @@ shell: LIBS+=" /usr/lib64/libtpms* /usr/lib64/libjson* /usr/lib64/libfuse*" LIBS+=" /usr/lib64/libxml2.s* /usr/lib64/libgcc_s* /usr/lib64/libaudit*" LIBS+=" /usr/lib64/libisoburn.s*" + LIBS+=" /usr/lib64/libgbm.s*" echo "Relocate additional libs for files in /VBINS" ./relocate_binaries.sh -i "$FILES" -o /VBINS From 8ae81b18b3c47f63bbd39117c1a7ccf4238dbf1f Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Thu, 24 Jul 2025 17:19:35 +0300 Subject: [PATCH 26/32] debug Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index fd7b5eb14b..d5d5a8c019 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -140,6 +140,7 @@ binaries: - /usr/bin/htop - /usr/bin/ps - /usr/bin/cat + - /usr/bin/grep # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm @@ -323,6 +324,7 @@ shell: LIBS+=" /usr/lib64/libxml2.s* /usr/lib64/libgcc_s* /usr/lib64/libaudit*" LIBS+=" /usr/lib64/libisoburn.s*" LIBS+=" /usr/lib64/libgbm.s*" + LIBS+=" /usr/lib64/libusbredirparser.s*" echo "Relocate additional libs for files in /VBINS" ./relocate_binaries.sh -i "$FILES" -o /VBINS From e6b96b7334320d560c80c20cf917de77ec24b487 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Fri, 25 Jul 2025 01:36:16 +0300 Subject: [PATCH 27/32] usbredir api Signed-off-by: Valeriy Khorunzhin --- .../generated/openapi/zz_generated.openapi.go | 27 +++++ api/subresources/register.go | 1 + api/subresources/types.go | 8 ++ api/subresources/v1alpha2/register.go | 1 + api/subresources/v1alpha2/types.go | 9 ++ .../v1alpha2/zz_generated.conversion.go | 44 ++++++++ .../v1alpha2/zz_generated.deepcopy.go | 25 +++++ api/subresources/zz_generated.deepcopy.go | 25 +++++ .../pkg/apiserver/api/install.go | 1 + .../apiserver/registry/vm/rest/usbredir.go | 103 ++++++++++++++++++ .../apiserver/registry/vm/storage/storage.go | 6 + .../kubevirt/virt-operator/rbac-for-us.yaml | 2 + .../use/capabilities/access_console.yaml | 1 + templates/user-authz-cluster-roles.yaml | 1 + 14 files changed, 254 insertions(+) create mode 100644 images/virtualization-artifact/pkg/apiserver/registry/vm/rest/usbredir.go diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go index 6b9f96c4b8..f8a7ce8033 100644 --- a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go +++ b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go @@ -151,6 +151,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/deckhouse/virtualization/api/subresources/v1alpha2.VirtualMachinePortForward": schema_virtualization_api_subresources_v1alpha2_VirtualMachinePortForward(ref), "github.com/deckhouse/virtualization/api/subresources/v1alpha2.VirtualMachineRemoveVolume": schema_virtualization_api_subresources_v1alpha2_VirtualMachineRemoveVolume(ref), "github.com/deckhouse/virtualization/api/subresources/v1alpha2.VirtualMachineUnfreeze": schema_virtualization_api_subresources_v1alpha2_VirtualMachineUnfreeze(ref), + "github.com/deckhouse/virtualization/api/subresources/v1alpha2.VirtualMachineUsbRedir": schema_virtualization_api_subresources_v1alpha2_VirtualMachineUsbRedir(ref), "github.com/deckhouse/virtualization/api/subresources/v1alpha2.VirtualMachineVNC": schema_virtualization_api_subresources_v1alpha2_VirtualMachineVNC(ref), "k8s.io/api/core/v1.AWSElasticBlockStoreVolumeSource": schema_k8sio_api_core_v1_AWSElasticBlockStoreVolumeSource(ref), "k8s.io/api/core/v1.Affinity": schema_k8sio_api_core_v1_Affinity(ref), @@ -5632,6 +5633,32 @@ func schema_virtualization_api_subresources_v1alpha2_VirtualMachineUnfreeze(ref } } +func schema_virtualization_api_subresources_v1alpha2_VirtualMachineUsbRedir(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_virtualization_api_subresources_v1alpha2_VirtualMachineVNC(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/api/subresources/register.go b/api/subresources/register.go index 8d19e4c34c..aeb438374d 100644 --- a/api/subresources/register.go +++ b/api/subresources/register.go @@ -52,6 +52,7 @@ var ( func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &VirtualMachineConsole{}, + &VirtualMachineUsbRedir{}, &VirtualMachineVNC{}, &VirtualMachinePortForward{}, &VirtualMachineAddVolume{}, diff --git a/api/subresources/types.go b/api/subresources/types.go index 6a8530d0a5..9c6a06924f 100644 --- a/api/subresources/types.go +++ b/api/subresources/types.go @@ -24,6 +24,14 @@ import ( // +genclient:readonly // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type VirtualMachineUsbRedir struct { + metav1.TypeMeta +} + +// +genclient +// +genclient:readonly +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + type VirtualMachineConsole struct { metav1.TypeMeta } diff --git a/api/subresources/v1alpha2/register.go b/api/subresources/v1alpha2/register.go index 4df9e743ab..cef52ab1e0 100644 --- a/api/subresources/v1alpha2/register.go +++ b/api/subresources/v1alpha2/register.go @@ -52,6 +52,7 @@ var ( func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, &VirtualMachineConsole{}, + &VirtualMachineUsbRedir{}, &VirtualMachineVNC{}, &VirtualMachinePortForward{}, &VirtualMachineAddVolume{}, diff --git a/api/subresources/v1alpha2/types.go b/api/subresources/v1alpha2/types.go index 5121840479..7bf7707310 100644 --- a/api/subresources/v1alpha2/types.go +++ b/api/subresources/v1alpha2/types.go @@ -23,6 +23,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +k8s:conversion-gen:explicit-from=net/url.Values +type VirtualMachineUsbRedir struct { + metav1.TypeMeta `json:",inline"` +} + +// +genclient +// +genclient:readonly +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +k8s:conversion-gen:explicit-from=net/url.Values + type VirtualMachineConsole struct { metav1.TypeMeta `json:",inline"` } diff --git a/api/subresources/v1alpha2/zz_generated.conversion.go b/api/subresources/v1alpha2/zz_generated.conversion.go index 9aa6dff88a..af6104b02f 100644 --- a/api/subresources/v1alpha2/zz_generated.conversion.go +++ b/api/subresources/v1alpha2/zz_generated.conversion.go @@ -108,6 +108,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*VirtualMachineUsbRedir)(nil), (*subresources.VirtualMachineUsbRedir)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_VirtualMachineUsbRedir_To_subresources_VirtualMachineUsbRedir(a.(*VirtualMachineUsbRedir), b.(*subresources.VirtualMachineUsbRedir), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*subresources.VirtualMachineUsbRedir)(nil), (*VirtualMachineUsbRedir)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_subresources_VirtualMachineUsbRedir_To_v1alpha2_VirtualMachineUsbRedir(a.(*subresources.VirtualMachineUsbRedir), b.(*VirtualMachineUsbRedir), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*VirtualMachineVNC)(nil), (*subresources.VirtualMachineVNC)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_VirtualMachineVNC_To_subresources_VirtualMachineVNC(a.(*VirtualMachineVNC), b.(*subresources.VirtualMachineVNC), scope) }); err != nil { @@ -153,6 +163,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*VirtualMachineUsbRedir)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_url_Values_To_v1alpha2_VirtualMachineUsbRedir(a.(*url.Values), b.(*VirtualMachineUsbRedir), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*url.Values)(nil), (*VirtualMachineVNC)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_url_Values_To_v1alpha2_VirtualMachineVNC(a.(*url.Values), b.(*VirtualMachineVNC), scope) }); err != nil { @@ -459,6 +474,35 @@ func Convert_url_Values_To_v1alpha2_VirtualMachineUnfreeze(in *url.Values, out * return autoConvert_url_Values_To_v1alpha2_VirtualMachineUnfreeze(in, out, s) } +func autoConvert_v1alpha2_VirtualMachineUsbRedir_To_subresources_VirtualMachineUsbRedir(in *VirtualMachineUsbRedir, out *subresources.VirtualMachineUsbRedir, s conversion.Scope) error { + return nil +} + +// Convert_v1alpha2_VirtualMachineUsbRedir_To_subresources_VirtualMachineUsbRedir is an autogenerated conversion function. +func Convert_v1alpha2_VirtualMachineUsbRedir_To_subresources_VirtualMachineUsbRedir(in *VirtualMachineUsbRedir, out *subresources.VirtualMachineUsbRedir, s conversion.Scope) error { + return autoConvert_v1alpha2_VirtualMachineUsbRedir_To_subresources_VirtualMachineUsbRedir(in, out, s) +} + +func autoConvert_subresources_VirtualMachineUsbRedir_To_v1alpha2_VirtualMachineUsbRedir(in *subresources.VirtualMachineUsbRedir, out *VirtualMachineUsbRedir, s conversion.Scope) error { + return nil +} + +// Convert_subresources_VirtualMachineUsbRedir_To_v1alpha2_VirtualMachineUsbRedir is an autogenerated conversion function. +func Convert_subresources_VirtualMachineUsbRedir_To_v1alpha2_VirtualMachineUsbRedir(in *subresources.VirtualMachineUsbRedir, out *VirtualMachineUsbRedir, s conversion.Scope) error { + return autoConvert_subresources_VirtualMachineUsbRedir_To_v1alpha2_VirtualMachineUsbRedir(in, out, s) +} + +func autoConvert_url_Values_To_v1alpha2_VirtualMachineUsbRedir(in *url.Values, out *VirtualMachineUsbRedir, s conversion.Scope) error { + // WARNING: Field TypeMeta does not have json tag, skipping. + + return nil +} + +// Convert_url_Values_To_v1alpha2_VirtualMachineUsbRedir is an autogenerated conversion function. +func Convert_url_Values_To_v1alpha2_VirtualMachineUsbRedir(in *url.Values, out *VirtualMachineUsbRedir, s conversion.Scope) error { + return autoConvert_url_Values_To_v1alpha2_VirtualMachineUsbRedir(in, out, s) +} + func autoConvert_v1alpha2_VirtualMachineVNC_To_subresources_VirtualMachineVNC(in *VirtualMachineVNC, out *subresources.VirtualMachineVNC, s conversion.Scope) error { return nil } diff --git a/api/subresources/v1alpha2/zz_generated.deepcopy.go b/api/subresources/v1alpha2/zz_generated.deepcopy.go index 4334ca31f6..62983bd785 100644 --- a/api/subresources/v1alpha2/zz_generated.deepcopy.go +++ b/api/subresources/v1alpha2/zz_generated.deepcopy.go @@ -211,6 +211,31 @@ func (in *VirtualMachineUnfreeze) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachineUsbRedir) DeepCopyInto(out *VirtualMachineUsbRedir) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineUsbRedir. +func (in *VirtualMachineUsbRedir) DeepCopy() *VirtualMachineUsbRedir { + if in == nil { + return nil + } + out := new(VirtualMachineUsbRedir) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachineUsbRedir) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineVNC) DeepCopyInto(out *VirtualMachineVNC) { *out = *in diff --git a/api/subresources/zz_generated.deepcopy.go b/api/subresources/zz_generated.deepcopy.go index dbb6d0bec1..191dc58f92 100644 --- a/api/subresources/zz_generated.deepcopy.go +++ b/api/subresources/zz_generated.deepcopy.go @@ -211,6 +211,31 @@ func (in *VirtualMachineUnfreeze) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VirtualMachineUsbRedir) DeepCopyInto(out *VirtualMachineUsbRedir) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineUsbRedir. +func (in *VirtualMachineUsbRedir) DeepCopy() *VirtualMachineUsbRedir { + if in == nil { + return nil + } + out := new(VirtualMachineUsbRedir) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VirtualMachineUsbRedir) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineVNC) DeepCopyInto(out *VirtualMachineVNC) { *out = *in diff --git a/images/virtualization-artifact/pkg/apiserver/api/install.go b/images/virtualization-artifact/pkg/apiserver/api/install.go index 071c333b55..de19597abb 100644 --- a/images/virtualization-artifact/pkg/apiserver/api/install.go +++ b/images/virtualization-artifact/pkg/apiserver/api/install.go @@ -60,6 +60,7 @@ func Build(store *storage.VirtualMachineStorage) genericapiserver.APIGroupInfo { resources := map[string]rest.Storage{ "virtualmachines": store, "virtualmachines/console": store.ConsoleREST(), + "virtualmachines/usbredir": store.UsbRedirREST(), "virtualmachines/vnc": store.VncREST(), "virtualmachines/portforward": store.PortForwardREST(), "virtualmachines/addvolume": store.AddVolumeREST(), diff --git a/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/usbredir.go b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/usbredir.go new file mode 100644 index 0000000000..6f23908833 --- /dev/null +++ b/images/virtualization-artifact/pkg/apiserver/registry/vm/rest/usbredir.go @@ -0,0 +1,103 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rest + +import ( + "context" + "fmt" + "net/http" + "net/url" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/registry/rest" + + "github.com/deckhouse/virtualization-controller/pkg/tls/certmanager" + virtlisters "github.com/deckhouse/virtualization/api/client/generated/listers/core/v1alpha2" + "github.com/deckhouse/virtualization/api/subresources" +) + +type UsbRedirREST struct { + vmLister virtlisters.VirtualMachineLister + proxyCertManager certmanager.CertificateManager + kubevirt KubevirtAPIServerConfig +} + +var ( + _ rest.Storage = &UsbRedirREST{} + _ rest.Connecter = &UsbRedirREST{} +) + +func NewUsbRedirREST(vmLister virtlisters.VirtualMachineLister, kubevirt KubevirtAPIServerConfig, proxyCertManager certmanager.CertificateManager) *UsbRedirREST { + return &UsbRedirREST{ + vmLister: vmLister, + kubevirt: kubevirt, + proxyCertManager: proxyCertManager, + } +} + +// New implements rest.Storage interface +func (r UsbRedirREST) New() runtime.Object { + return &subresources.VirtualMachineUsbRedir{} +} + +// Destroy implements rest.Storage interface +func (r UsbRedirREST) Destroy() { +} + +func (r UsbRedirREST) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) { + usbRedirOpts, ok := opts.(*subresources.VirtualMachineUsbRedir) + if !ok { + return nil, fmt.Errorf("invalid options object: %#v", opts) + } + location, transport, err := UsbRedirLocation(ctx, r.vmLister, name, usbRedirOpts, r.kubevirt, r.proxyCertManager) + transport.ReadBufferSize = 32 * 1024 + transport.WriteBufferSize = 32 * 1024 + if err != nil { + return nil, err + } + handler := newThrottledUpgradeAwareProxyHandler(location, transport, true, responder, r.kubevirt.ServiceAccount) + return handler, nil +} + +// NewConnectOptions implements rest.Connecter interface +func (r UsbRedirREST) NewConnectOptions() (runtime.Object, bool, string) { + return &subresources.VirtualMachineUsbRedir{}, false, "" +} + +// ConnectMethods implements rest.Connecter interface +func (r UsbRedirREST) ConnectMethods() []string { + return upgradeableMethods +} + +func UsbRedirLocation( + ctx context.Context, + getter virtlisters.VirtualMachineLister, + name string, + opts *subresources.VirtualMachineUsbRedir, + kubevirt KubevirtAPIServerConfig, + proxyCertManager certmanager.CertificateManager, +) (*url.URL, *http.Transport, error) { + return streamLocation( + ctx, + getter, + name, + newKVVMIPather("usbredir"), + kubevirt, + proxyCertManager, + virtualMachineNeedRunning, + ) +} diff --git a/images/virtualization-artifact/pkg/apiserver/registry/vm/storage/storage.go b/images/virtualization-artifact/pkg/apiserver/registry/vm/storage/storage.go index fe13f2887d..b5750fea64 100644 --- a/images/virtualization-artifact/pkg/apiserver/registry/vm/storage/storage.go +++ b/images/virtualization-artifact/pkg/apiserver/registry/vm/storage/storage.go @@ -43,6 +43,7 @@ type VirtualMachineStorage struct { groupResource schema.GroupResource vmLister virtlisters.VirtualMachineLister console *vmrest.ConsoleREST + usbredir *vmrest.UsbRedirREST vnc *vmrest.VNCREST portforward *vmrest.PortForwardREST addVolume *vmrest.AddVolumeREST @@ -93,6 +94,7 @@ func NewStorage( vmLister: vmLister, console: vmrest.NewConsoleREST(vmLister, kubevirt, proxyCertManager), vnc: vmrest.NewVNCREST(vmLister, kubevirt, proxyCertManager), + usbredir: vmrest.NewUsbRedirREST(vmLister, kubevirt, proxyCertManager), portforward: vmrest.NewPortForwardREST(vmLister, kubevirt, proxyCertManager), addVolume: vmrest.NewAddVolumeREST(vmLister, kubevirt, proxyCertManager), removeVolume: vmrest.NewRemoveVolumeREST(vmLister, kubevirt, proxyCertManager), @@ -136,6 +138,10 @@ func (store VirtualMachineStorage) CancelEvacuationREST() *vmrest.CancelEvacuati return store.cancelEvacuation } +func (store VirtualMachineStorage) UsbRedirREST() *vmrest.UsbRedirREST { + return store.usbredir +} + // New implements rest.Storage interface func (store VirtualMachineStorage) New() runtime.Object { return &virtv2.VirtualMachine{} diff --git a/templates/kubevirt/virt-operator/rbac-for-us.yaml b/templates/kubevirt/virt-operator/rbac-for-us.yaml index d52cec41b0..5d0b777919 100644 --- a/templates/kubevirt/virt-operator/rbac-for-us.yaml +++ b/templates/kubevirt/virt-operator/rbac-for-us.yaml @@ -826,6 +826,7 @@ rules: - subresources.kubevirt.io resources: - virtualmachineinstances/console + - virtualmachineinstances/usbredir - virtualmachineinstances/vnc - virtualmachineinstances/vnc/screenshot - virtualmachineinstances/portforward @@ -975,6 +976,7 @@ rules: - subresources.kubevirt.io resources: - virtualmachineinstances/console + - virtualmachineinstances/usbredir - virtualmachineinstances/vnc - virtualmachineinstances/vnc/screenshot - virtualmachineinstances/portforward diff --git a/templates/rbacv2/use/capabilities/access_console.yaml b/templates/rbacv2/use/capabilities/access_console.yaml index f0c30052c1..269c707bfa 100644 --- a/templates/rbacv2/use/capabilities/access_console.yaml +++ b/templates/rbacv2/use/capabilities/access_console.yaml @@ -12,6 +12,7 @@ rules: - subresources.virtualization.deckhouse.io resources: - virtualmachines/console + - virtualmachines/usbredir verbs: - get - create diff --git a/templates/user-authz-cluster-roles.yaml b/templates/user-authz-cluster-roles.yaml index 260538aa39..53de479c80 100644 --- a/templates/user-authz-cluster-roles.yaml +++ b/templates/user-authz-cluster-roles.yaml @@ -52,6 +52,7 @@ rules: - subresources.virtualization.deckhouse.io resources: - virtualmachines/console + - virtualmachines/usbredir - virtualmachines/vnc - virtualmachines/portforward - virtualmachines/addvolume From 886208f8e9cda34cd2947c9e37738c326b0c1a3c Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 28 Jul 2025 00:03:02 +0300 Subject: [PATCH 28/32] usbredirhostlib Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index d5d5a8c019..9fd1cddfcd 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -325,6 +325,7 @@ shell: LIBS+=" /usr/lib64/libisoburn.s*" LIBS+=" /usr/lib64/libgbm.s*" LIBS+=" /usr/lib64/libusbredirparser.s*" + LIBS+=" /usr/lib64/libusbredirhost.s*" echo "Relocate additional libs for files in /VBINS" ./relocate_binaries.sh -i "$FILES" -o /VBINS From c408f45059a46c95c721c4d8dd09e86c373d5cdb Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Mon, 28 Jul 2025 12:03:34 +0300 Subject: [PATCH 29/32] f Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 1 - usbredirbridgesample/main.go | 13 +++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index 9fd1cddfcd..d5d5a8c019 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -325,7 +325,6 @@ shell: LIBS+=" /usr/lib64/libisoburn.s*" LIBS+=" /usr/lib64/libgbm.s*" LIBS+=" /usr/lib64/libusbredirparser.s*" - LIBS+=" /usr/lib64/libusbredirhost.s*" echo "Relocate additional libs for files in /VBINS" ./relocate_binaries.sh -i "$FILES" -o /VBINS diff --git a/usbredirbridgesample/main.go b/usbredirbridgesample/main.go index ffe6ed0d81..cb600ca564 100644 --- a/usbredirbridgesample/main.go +++ b/usbredirbridgesample/main.go @@ -25,14 +25,19 @@ func main() { //----------------------------------------------------------------- // 2. Собираем wss‑URL к subresource usbredir - ns, name := "default", "ubuntu-vm" + ns, name := "test-project", "linux-vm" host := strings.TrimPrefix(kcfg.Host, "https://") wsURL := url.URL{ Scheme: "wss", Host: host, - Path: fmt.Sprintf("/apis/subresources.kubevirt.io/v1/namespaces/%s/"+ - "virtualmachineinstances/%s/usbredir", ns, name), + // Path: fmt.Sprintf("/apis/subresources.kubevirt.io/v1/namespaces/%s/"+ + // "virtualmachineinstances/%s/usbredir", ns, name), + Path: fmt.Sprintf("/apis/subresources.virtualization.deckhouse.io/v1alpha2/namespaces/%s/"+ + "virtualmachines/%s/usbredir", ns, name), + // Path: "/apis/subresources.virtualization.deckhouse.io/v1alpha2/namespaces/test-project/virtualmachines/linux-vm/usbredir", + // kvvmiPathTmpl = "/apis/subresources.kubevirt.io/v1/namespaces/%s/virtualmachineinstances/%s/%s" } + fmt.Println(wsURL) //----------------------------------------------------------------- // 3. Создаём TLS‑конфиг из kube‑config  (функция rest.TLSConfigFor) @@ -43,7 +48,7 @@ func main() { dialer := websocket.Dialer{ TLSClientConfig: tlsCfg, - Subprotocols: []string{"binary.kubevirt.io"}, + Subprotocols: []string{""}, } //----------------------------------------------------------------- From 11a56a1dd66f1ac1b90155f7365757338b176d96 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 29 Jul 2025 13:01:18 +0300 Subject: [PATCH 30/32] test patch Signed-off-by: Valeriy Khorunzhin --- images/virt-artifact/werf.inc.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/images/virt-artifact/werf.inc.yaml b/images/virt-artifact/werf.inc.yaml index 6c719ab784..bd29b1d661 100644 --- a/images/virt-artifact/werf.inc.yaml +++ b/images/virt-artifact/werf.inc.yaml @@ -1,7 +1,7 @@ --- # Source https://github.com/kubevirt/kubevirt/blob/v1.3.1/hack/dockerized#L15 {{- $version := "v1.3.1" }} -{{- $tag := print $version "-v12n.7"}} +{{- $tag := print $version "-virtualization-hotplug-experiments"}} {{- $name := print $.ImageName "-dependencies" -}} {{- define "$name" -}} @@ -43,8 +43,10 @@ shell: {{- include "alt packages clean" . | nindent 2 }} + installCacheVersion: "{{ now | date "Mon Jan 2 15:04:05 MST 2006" }}" # <--- для пересборки install: - | + echo $date mkdir -p ~/.ssh && echo "StrictHostKeyChecking accept-new" > ~/.ssh/config git config --global --add advice.detachedHead false git clone --depth=1 $(cat /run/secrets/SOURCE_REPO)/deckhouse/3p-kubevirt --branch {{ $tag }} /kubevirt From ff04024d8237b0c412f727980bf1578e6a5816d0 Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Tue, 29 Jul 2025 18:44:15 +0300 Subject: [PATCH 31/32] ttt Signed-off-by: Valeriy Khorunzhin --- images/virt-handler/werf.inc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virt-handler/werf.inc.yaml b/images/virt-handler/werf.inc.yaml index 2ef4fe73a3..8f295037e2 100644 --- a/images/virt-handler/werf.inc.yaml +++ b/images/virt-handler/werf.inc.yaml @@ -1,6 +1,6 @@ --- image: {{ $.ImageName }} -fromImage: distroless +fromImage: BASE_ALT_P11 import: - image: {{ $.ImageName }}-bins add: /relocate From 3b8b02b94b07b84c1a68299d1a30bf4199cb1fdc Mon Sep 17 00:00:00 2001 From: Valeriy Khorunzhin Date: Wed, 30 Jul 2025 16:39:43 +0300 Subject: [PATCH 32/32] tttttttt Signed-off-by: Valeriy Khorunzhin --- images/virt-launcher/werf.inc.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/images/virt-launcher/werf.inc.yaml b/images/virt-launcher/werf.inc.yaml index d5d5a8c019..561da9eefc 100644 --- a/images/virt-launcher/werf.inc.yaml +++ b/images/virt-launcher/werf.inc.yaml @@ -137,10 +137,15 @@ binaries: - /usr/bin/bash - /usr/bin/ls - /usr/bin/lsusb + - /usr/bin/lspci - /usr/bin/htop - /usr/bin/ps - /usr/bin/cat - /usr/bin/grep + - /usr/bin/mkdir + - /usr/bin/mount + - /usr/bin/chmod + - /usr/bin/mknod # Xorriso (Creates an image of an ISO9660 filesystem) - /usr/bin/xorriso-dd-target /usr/bin/xorrisofs /usr/bin/xorriso # Swtpm