From 189a5f78f3f694368b1b954fe54384ac1a834dac Mon Sep 17 00:00:00 2001 From: spencercjh Date: Tue, 25 Aug 2026 07:08:03 +0000 Subject: [PATCH] feat(iluvatar): mock configured GPU resources Signed-off-by: spencercjh --- README.md | 106 +++++-- internal/pkg/api/device/iluvatar/device.go | 264 +++++++++++++++++- .../pkg/api/device/iluvatar/device_test.go | 239 ++++++++++++++++ internal/pkg/config/config.go | 82 +++++- internal/pkg/config/iluvatar_test.go | 140 ++++++++++ 5 files changed, 799 insertions(+), 32 deletions(-) create mode 100644 internal/pkg/api/device/iluvatar/device_test.go create mode 100644 internal/pkg/config/iluvatar_test.go diff --git a/README.md b/README.md index b024359..f91205e 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,30 @@ After deployment these resources show up under `node.status.allocatable` and `no ## How it works (read this first) -The mock plugin **does not detect hardware**. It does the following every ~30s: +The mock plugin **does not detect hardware**. It reads the HAMi node-register +annotation every ~30s and uses one of two resource-registration patterns: ```text - node annotation: hami.io/node--register = [ {devmem, devcore, ...}, ... ] (1) - node capacity: (e.g. nvidia.com/gpu) > 0 (2) <- health gate - | mock reads - v - registers into allocatable: /...mem , /...cores , ... + supplemental: annotation + existing count capacity > 0 -> memory/core resources + fully mocked: annotation -> count + memory + core resources ``` -On a **real** cluster, (1) and (2) are produced by the real device plugin. In a **mock-only** (no hardware) environment you provide them yourself: +NVIDIA, Ascend, and Hygon use the supplemental pattern: their existing count +resource is a health gate. Iluvatar uses the fully mocked pattern because its +count resource is one of the virtual resources derived from the annotation. -- **(1) the `node--register` annotation** describing the fake cards -- `kubectl annotate`. -- **(2) the count extended resource** (e.g. `nvidia.com/gpu`) -- patched onto the node `status`. +On a **real** cluster, the annotation and any external count capacity are +produced by the real device plugin. In a **mock-only** (no hardware) environment +you provide the inputs yourself: -> There is **no auto-labeller** in this repo, so (1) and (2) are manual today. Forgetting them is the usual cause of `device xxx is unhealthy` / `no allocation` -- see issues #14 / #16. +- The `node--register` annotation describing the fake cards -- + `kubectl annotate`. +- For supplemental devices only, the count extended resource (for example, + `nvidia.com/gpu`) -- patched onto the node `status`. + +> There is **no auto-labeller** in this repo, so these inputs are manual today. +> For a supplemental device, forgetting either input is the usual cause of +> `device xxx is unhealthy` / `no allocation` -- see issues #14 / #16. ## Prerequisites @@ -42,23 +50,32 @@ kubectl apply -f k8s-mock-plugin.yaml ## Understanding the values -The mock **derives the registered resources from the annotation, not from the count resource.** This trips people up, so to be explicit: - -- **Number of fake cards = the number of entries in the annotation array** (not the count value). -- Registered `...-memory` = **sum of `devmem`** over all entries. -- Registered `...-cores` / `...-core` = **sum of `devcore`** over all entries. -- The **count extended resource is only a health gate**: its value just needs to be `> 0`. It does **not** affect the registered memory/cores. By convention it is set to `cards x splits-per-card` (e.g. Ascend `2 x VDeviceCount(4) = 8`), but `1` would work equally well for the memory/cores to appear. +For supplemental devices, the mock derives the registered memory/core resources +from the annotation, not from the external count resource. For Iluvatar, all +three resources are derived from the annotation. In detail: + +- **Number of fake cards = the number of entries in the annotation** (not the count value). +- Supplemental devices register **the sum of `devmem`** over annotation entries. + Iluvatar counts only healthy entries and converts their total from MiB to + 256-MiB vMem units with + `floor(sum(devmem) / 256)`. +- Registered cores are normally **the sum of `devcore`** over annotation + entries. Iluvatar excludes unhealthy entries; Ascend soft mode ignores + `devcore` and contributes 100 per annotation entry. +- For NVIDIA, Ascend, and Hygon, the **external count resource is only a health + gate**: its value just needs to be `> 0`. For Iluvatar, the mock instead + registers the count resource as **the sum of `count`** over healthy entries. Annotation entry fields: | field | meaning | | :-- | :-- | | `id` | unique device UUID (any string) | -| `devmem` | per-card memory in MB -- **summed** into `...-memory` | -| `devcore` | per-card cores. **NVIDIA/Hygon:** summed into `...-cores` (NVIDIA: percentage, 100 = a whole card). **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. | -| `count` | per-card split count (informational for the mock) | +| `devmem` | per-card memory in MiB. NVIDIA/Hygon/Ascend sum it directly; Iluvatar divides the healthy-device total by 256 to produce vMem units. | +| `devcore` | per-card cores. **NVIDIA/Hygon/Iluvatar:** summed into the configured core resource. **Ascend:** ignored -- `huawei.com/-core` is percentage-based, registered as **100 per card**. | +| `count` | per-card split count. Iluvatar sums it into the configured count resource; it is informational for the other supported mock families. | | `type` | device model string | -| `health` | must be `true` to be counted | +| `health` | HAMi uses this field when deciding whether a device is allocatable. Iluvatar mock totals also exclude entries where it is `false`. | | `index` | card index `0,1,2,...` (`0` may be omitted) | | `numa`, `mode` | optional | @@ -66,7 +83,14 @@ Annotation entry fields: ## Usage by vendor -To mock one card you always need the **three pieces**: the vendor **config block** (in the `hami-scheduler-device` ConfigMap, gives the resource names), the **count extended resource** (passes the health gate), and the **node-register annotation** (describes the fake cards). Replace `` below with your target node. Resource names follow the ConfigMap (HAMi defaults shown). See [Understanding the values](#understanding-the-values) for how the numbers are derived. +Every device needs its vendor **config block** (in the +`hami-scheduler-device` ConfigMap) and its **node-register annotation**. +NVIDIA, Ascend, and Hygon additionally need an external **count extended +resource** to pass their health gate. Iluvatar's count is generated by the mock. +Replace `` below with your target node. Resource names follow the +ConfigMap (HAMi defaults shown). See +[Understanding the values](#understanding-the-values) for how the numbers are +derived. ### NVIDIA (e.g. A100-80GB) @@ -127,6 +151,41 @@ kubectl get node -o json | jq '.status.allocatable|with_entries(select(.k # expect: hygon.com/dcumem=32768, hygon.com/dcucores=200 (2 cards x devcore 100) ``` +### Iluvatar GPU + +- config block: `iluvatars:` | annotation: + `hami.io/node--register` (**CSV**, not JSON) +- mock registers the configured count, vMem, and vCore resources; no separate + count-resource status patch is needed + +HAMi's default configuration contains these four entries: + +| `commonWord` | count | memory | core | +| :-- | :-- | :-- | :-- | +| `MR-V100` | `iluvatar.ai/MR-V100-vgpu` | `iluvatar.ai/MR-V100.vMem` | `iluvatar.ai/MR-V100.vCore` | +| `MR-V50` | `iluvatar.ai/MR-V50-vgpu` | `iluvatar.ai/MR-V50.vMem` | `iluvatar.ai/MR-V50.vCore` | +| `BI-V150` | `iluvatar.ai/BI-V150-vgpu` | `iluvatar.ai/BI-V150.vMem` | `iluvatar.ai/BI-V150.vCore` | +| `BI-V100` | `iluvatar.ai/BI-V100-vgpu` | `iluvatar.ai/BI-V100.vMem` | `iluvatar.ai/BI-V100.vCore` | + +The annotation key is derived from the configured `commonWord`. Each device is +encoded as either the legacy seven-field form +`id,count,devmem,devcore,type,numa,health:` or the nine-field form that appends +`index,mode:`. + +```bash +# Two MR-V100 cards. The mock itself registers all three resources. +kubectl annotate node \ + 'hami.io/node-MR-V100-register=GPU-MOCK-0,10,8192,100,MR-V100,0,true,0,hami-core:GPU-MOCK-1,10,8192,100,MR-V100,0,true,1,hami-core:' +# verify (~30s later) +kubectl get node -o json | jq '.status.allocatable|with_entries(select(.key|test("iluvatar.ai/MR-V100")))' +# expect: MR-V100-vgpu=20, MR-V100.vMem=64, MR-V100.vCore=200 +# vMem is floor((8192 + 8192) / 256) +``` + +Only entries whose `health` field is `true` contribute resources. All three +resource names in one Iluvatar config must use the same vendor namespace, and +CommonWords and full resource names must be unique across Iluvatar entries. + ## Ascend config compatibility (new vs legacy) The Ascend `vnpus` config has two layouts and the plugin accepts **both**: @@ -155,8 +214,11 @@ The new nested format is tried first; if that fails it falls back to the legacy | Nvidia GPU | `nvidia.com/gpumem`, `nvidia.com/gpumem-percentage`, `nvidia.com/gpucores` | | Hygon DCU | `hygon.com/dcumem`, `hygon.com/dcucores` (when `resourceCoreName` is set) | | Ascend | `huawei.com/Ascend{chip}-memory`, `huawei.com/Ascend{chip}-core` (when `resourceCoreName` is set **and** the node is in `hami-vnpu-core` mode) | +| Iluvatar | configured `iluvatar.ai/*-vgpu`, `iluvatar.ai/*.vMem`, and `iluvatar.ai/*.vCore` resources | -**Note:** If the counted memory is too large (e.g. > 120GB) it may display as 0. Set `memoryFactor` in the `hami-scheduler-device` ConfigMap (default 1). +**Note:** For device configs that expose `memoryFactor` (NVIDIA, Hygon, and +Ascend), use it when a directly counted memory value is too large for the +extended resource. Iluvatar uses its fixed 256-MiB vMem conversion instead. ## Build diff --git a/internal/pkg/api/device/iluvatar/device.go b/internal/pkg/api/device/iluvatar/device.go index 42695c3..748b524 100644 --- a/internal/pkg/api/device/iluvatar/device.go +++ b/internal/pkg/api/device/iluvatar/device.go @@ -16,10 +16,272 @@ limitations under the License. package iluvatar +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device" + "github.com/HAMi/mock-device-plugin/internal/pkg/mock" + "github.com/kubevirt/device-plugin-manager/pkg/dpm" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/klog/v2" +) + +// MemoryFactor matches HAMi's Iluvatar resource contract: one vMem unit is +// 256 MiB, while the device registration annotation stores memory in MiB. +const MemoryFactor = 256 + type IluvatarConfig struct { CommonWord string `yaml:"commonWord"` ChipName string `yaml:"chipName"` ResourceCountName string `yaml:"resourceCountName"` ResourceMemoryName string `yaml:"resourceMemoryName"` ResourceCoreName string `yaml:"resourceCoreName"` -} \ No newline at end of file +} + +type IluvatarDevices struct { + config IluvatarConfig + nodeRegisterAnno string +} + +// InitIluvatarDevices validates the complete Iluvatar configuration before it +// returns any devices. This keeps configuration failures from silently +// overwriting a device or resource registration. +func InitIluvatarDevices(configs []IluvatarConfig) ([]*IluvatarDevices, error) { + commonWords := make(map[string]struct{}, len(configs)) + resourceNames := make(map[string]struct{}, len(configs)*3) + devices := make([]*IluvatarDevices, 0, len(configs)) + + for idx, config := range configs { + if err := validateConfig(config, commonWords, resourceNames); err != nil { + return nil, fmt.Errorf("invalid iluvatar config at index %d: %w", idx, err) + } + + commonWords[config.CommonWord] = struct{}{} + resourceNames[config.ResourceCountName] = struct{}{} + resourceNames[config.ResourceMemoryName] = struct{}{} + resourceNames[config.ResourceCoreName] = struct{}{} + devices = append(devices, &IluvatarDevices{ + config: config, + nodeRegisterAnno: fmt.Sprintf("hami.io/node-%s-register", config.CommonWord), + }) + } + + return devices, nil +} + +func validateConfig(config IluvatarConfig, commonWords, resourceNames map[string]struct{}) error { + if strings.TrimSpace(config.CommonWord) == "" { + return errors.New("commonWord must not be empty") + } + if strings.Contains(config.CommonWord, "/") { + return fmt.Errorf("commonWord %q must not contain '/'", config.CommonWord) + } + if problems := validation.IsQualifiedName(config.CommonWord); len(problems) > 0 { + return fmt.Errorf("commonWord %q is invalid: %s", config.CommonWord, strings.Join(problems, "; ")) + } + annotationKey := fmt.Sprintf("hami.io/node-%s-register", config.CommonWord) + if problems := validation.IsQualifiedName(annotationKey); len(problems) > 0 { + return fmt.Errorf("commonWord %q produces invalid annotation key %q: %s", config.CommonWord, annotationKey, strings.Join(problems, "; ")) + } + if strings.TrimSpace(config.ChipName) == "" { + return errors.New("chipName must not be empty") + } + if _, exists := commonWords[config.CommonWord]; exists { + return fmt.Errorf("duplicate commonWord %q", config.CommonWord) + } + + resources := []struct { + field string + name string + }{ + {field: "resourceCountName", name: config.ResourceCountName}, + {field: "resourceMemoryName", name: config.ResourceMemoryName}, + {field: "resourceCoreName", name: config.ResourceCoreName}, + } + + vendor := "" + entryResources := make(map[string]struct{}, len(resources)) + for _, resource := range resources { + if problems := validation.IsQualifiedName(resource.name); len(problems) > 0 { + return fmt.Errorf("%s %q is not a valid extended-resource name: %s", resource.field, resource.name, strings.Join(problems, "; ")) + } + resourceVendor, resourceName, ok := strings.Cut(resource.name, "/") + if !ok || strings.TrimSpace(resourceVendor) == "" || strings.TrimSpace(resourceName) == "" { + return fmt.Errorf("%s %q must use the extended-resource form vendor/name", resource.field, resource.name) + } + if vendor == "" { + vendor = resourceVendor + } else if resourceVendor != vendor { + return fmt.Errorf("%s %q must use vendor namespace %q", resource.field, resource.name, vendor) + } + if _, exists := entryResources[resource.name]; exists { + return fmt.Errorf("duplicate resource name %q", resource.name) + } + if _, exists := resourceNames[resource.name]; exists { + return fmt.Errorf("duplicate resource name %q", resource.name) + } + entryResources[resource.name] = struct{}{} + } + + return nil +} + +func (dev *IluvatarDevices) CommonWord() string { + return dev.config.CommonWord +} + +func (dev *IluvatarDevices) GetNodeDevices(n *corev1.Node) ([]*device.DeviceInfo, error) { + devEncoded, ok := n.Annotations[dev.nodeRegisterAnno] + if !ok { + return []*device.DeviceInfo{}, errors.New("annos not found " + dev.nodeRegisterAnno) + } + + nodeDevices, err := decodeNodeDevices(devEncoded) + if err != nil { + klog.ErrorS(err, "failed to decode node devices", "node", n.Name, "device annotation", devEncoded) + return []*device.DeviceInfo{}, err + } + if len(nodeDevices) == 0 { + klog.InfoS("no iluvatar gpu device found", "node", n.Name, "device annotation", devEncoded) + return []*device.DeviceInfo{}, errors.New("no gpu found on node") + } + for idx, nodeDevice := range nodeDevices { + if nodeDevice.Count < 0 || nodeDevice.Devmem < 0 || nodeDevice.Devcore < 0 { + return []*device.DeviceInfo{}, fmt.Errorf( + "device %d has negative capacity: count=%d devmem=%d devcore=%d", + idx, nodeDevice.Count, nodeDevice.Devmem, nodeDevice.Devcore, + ) + } + nodeDevice.DeviceVendor = dev.CommonWord() + } + + return nodeDevices, nil +} + +// decodeNodeDevices follows HAMi's current legacy CSV annotation contract but +// fails closed on malformed fields. The repository's older shared decoder +// silently coerces parse errors to zero, which is unsafe for configured +// Iluvatar resources and would also turn a negative index into a large uint. +func decodeNodeDevices(encoded string) ([]*device.DeviceInfo, error) { + if !strings.Contains(encoded, device.OneContainerMultiDeviceSplitSymbol) { + return nil, errors.New("node annotation missing device separator") + } + + segments := strings.Split(encoded, device.OneContainerMultiDeviceSplitSymbol) + devices := make([]*device.DeviceInfo, 0, len(segments)) + deviceIDs := make(map[string]struct{}, len(segments)) + for _, segment := range segments { + if segment == "" { + continue + } + if !strings.Contains(segment, ",") { + return nil, fmt.Errorf("malformed node annotation segment: %q", segment) + } + fields := strings.Split(segment, ",") + if len(fields) != 7 && len(fields) != 9 { + return nil, fmt.Errorf("unexpected field count %d in node annotation", len(fields)) + } + if strings.TrimSpace(fields[0]) == "" { + return nil, errors.New("device ID must not be empty") + } + if _, exists := deviceIDs[fields[0]]; exists { + return nil, fmt.Errorf("duplicate device ID %q", fields[0]) + } + deviceIDs[fields[0]] = struct{}{} + + count, err := strconv.ParseInt(fields[1], 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid count field: %w", err) + } + memory, err := strconv.ParseInt(fields[2], 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid memory field: %w", err) + } + core, err := strconv.ParseInt(fields[3], 10, 32) + if err != nil { + return nil, fmt.Errorf("invalid core field: %w", err) + } + numa, err := strconv.Atoi(fields[5]) + if err != nil { + return nil, fmt.Errorf("invalid numa field: %w", err) + } + health, err := strconv.ParseBool(fields[6]) + if err != nil { + return nil, fmt.Errorf("invalid health field: %w", err) + } + + index := 0 + mode := "hami-core" + if len(fields) == 9 { + index, err = strconv.Atoi(fields[7]) + if err != nil { + return nil, fmt.Errorf("invalid index field: %w", err) + } + if index < 0 { + return nil, fmt.Errorf("index field must not be negative: %d", index) + } + mode = fields[8] + } + + devices = append(devices, &device.DeviceInfo{ + ID: fields[0], + Index: uint(index), + Count: int32(count), + Devmem: int32(memory), + Devcore: int32(core), + Type: fields[4], + Numa: numa, + Mode: mode, + Health: health, + }) + } + return devices, nil +} + +func (dev *IluvatarDevices) GetResource(n *corev1.Node) map[string]int { + countResourceName := device.GetResourceName(dev.config.ResourceCountName) + memoryResourceName := device.GetResourceName(dev.config.ResourceMemoryName) + coreResourceName := device.GetResourceName(dev.config.ResourceCoreName) + resourceMap := map[string]int{ + countResourceName: 0, + memoryResourceName: 0, + coreResourceName: 0, + } + + nodeDevices, err := dev.GetNodeDevices(n) + if err != nil { + klog.Infof("no device %s on this node", dev.CommonWord()) + return resourceMap + } + + rawMemory := 0 + for _, nodeDevice := range nodeDevices { + if !nodeDevice.Health { + continue + } + resourceMap[countResourceName] += int(nodeDevice.Count) + rawMemory += int(nodeDevice.Devmem) + resourceMap[coreResourceName] += int(nodeDevice.Devcore) + } + resourceMap[memoryResourceName] = rawMemory / MemoryFactor + + klog.InfoS("Add resources", + countResourceName, resourceMap[countResourceName], + memoryResourceName, resourceMap[memoryResourceName], + coreResourceName, resourceMap[coreResourceName], + ) + return resourceMap +} + +func (dev *IluvatarDevices) RunManager() { + lmock := mock.NewMockLister(device.GetVendorName(dev.config.ResourceCountName)) + go device.Register(lmock, dev) + mockManager := dpm.NewManager(lmock) + klog.Infof("Running mocking dp: %s", dev.CommonWord()) + mockManager.Run() +} diff --git a/internal/pkg/api/device/iluvatar/device_test.go b/internal/pkg/api/device/iluvatar/device_test.go new file mode 100644 index 0000000..6cf3aee --- /dev/null +++ b/internal/pkg/api/device/iluvatar/device_test.go @@ -0,0 +1,239 @@ +/* +Copyright 2025 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package iluvatar + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func testConfig(commonWord string) IluvatarConfig { + return IluvatarConfig{ + CommonWord: commonWord, + ChipName: commonWord, + ResourceCountName: "iluvatar.ai/" + commonWord + "-vgpu", + ResourceMemoryName: "iluvatar.ai/" + commonWord + ".vMem", + ResourceCoreName: "iluvatar.ai/" + commonWord + ".vCore", + } +} + +func mustInitDevice(t *testing.T, config IluvatarConfig) *IluvatarDevices { + t.Helper() + devices, err := InitIluvatarDevices([]IluvatarConfig{config}) + if err != nil { + t.Fatalf("InitIluvatarDevices() error = %v", err) + } + if len(devices) != 1 { + t.Fatalf("InitIluvatarDevices() returned %d devices, want 1", len(devices)) + } + return devices[0] +} + +func TestIluvatarDevicesGetNodeDevices(t *testing.T) { + dev := mustInitDevice(t, testConfig("MR-V100")) + + tests := []struct { + name string + annotation *string + wantErr bool + wantIndex uint + wantMode string + }{ + { + name: "seven fields", + annotation: stringPointer("GPU-0,10,8192,100,MR-V100,0,true:"), + wantIndex: 0, + wantMode: "hami-core", + }, + { + name: "nine fields", + annotation: stringPointer("GPU-0,10,8192,100,MR-V100,1,true,3,sriov:"), + wantIndex: 3, + wantMode: "sriov", + }, + {name: "missing annotation", wantErr: true}, + {name: "empty annotation", annotation: stringPointer(""), wantErr: true}, + {name: "empty device list", annotation: stringPointer(":"), wantErr: true}, + {name: "malformed device", annotation: stringPointer("GPU-0,10,8192:"), wantErr: true}, + {name: "malformed segment", annotation: stringPointer("junk:GPU-0,10,8192,100,MR-V100,0,true:"), wantErr: true}, + {name: "invalid count", annotation: stringPointer("GPU-0,oops,8192,100,MR-V100,0,true:"), wantErr: true}, + {name: "invalid memory", annotation: stringPointer("GPU-0,10,oops,100,MR-V100,0,true:"), wantErr: true}, + {name: "invalid core", annotation: stringPointer("GPU-0,10,8192,oops,MR-V100,0,true:"), wantErr: true}, + {name: "invalid numa", annotation: stringPointer("GPU-0,10,8192,100,MR-V100,oops,true:"), wantErr: true}, + {name: "invalid health", annotation: stringPointer("GPU-0,10,8192,100,MR-V100,0,oops:"), wantErr: true}, + {name: "invalid index", annotation: stringPointer("GPU-0,10,8192,100,MR-V100,0,true,oops,hami-core:"), wantErr: true}, + {name: "negative index", annotation: stringPointer("GPU-0,10,8192,100,MR-V100,0,true,-1,hami-core:"), wantErr: true}, + {name: "empty device ID", annotation: stringPointer(",10,8192,100,MR-V100,0,true:"), wantErr: true}, + {name: "duplicate device ID", annotation: stringPointer("GPU-0,10,8192,100,MR-V100,0,true:GPU-0,10,8192,100,MR-V100,0,true:"), wantErr: true}, + {name: "negative count", annotation: stringPointer("GPU-0,-1,8192,100,MR-V100,0,true:"), wantErr: true}, + {name: "negative memory", annotation: stringPointer("GPU-0,10,-1,100,MR-V100,0,true:"), wantErr: true}, + {name: "negative core", annotation: stringPointer("GPU-0,10,8192,-1,MR-V100,0,true:"), wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + annotations := map[string]string{} + if tt.annotation != nil { + annotations[dev.nodeRegisterAnno] = *tt.annotation + } + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-1", Annotations: annotations}} + + got, err := dev.GetNodeDevices(node) + if (err != nil) != tt.wantErr { + t.Fatalf("GetNodeDevices() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + if len(got) != 0 { + t.Fatalf("GetNodeDevices() returned %d devices on error", len(got)) + } + return + } + if len(got) != 1 { + t.Fatalf("GetNodeDevices() returned %d devices, want 1", len(got)) + } + if got[0].ID != "GPU-0" || got[0].Count != 10 || got[0].Devmem != 8192 || got[0].Devcore != 100 { + t.Fatalf("GetNodeDevices() decoded unexpected device: %+v", got[0]) + } + if got[0].Index != tt.wantIndex || got[0].Mode != tt.wantMode { + t.Fatalf("GetNodeDevices() index/mode = %d/%q, want %d/%q", got[0].Index, got[0].Mode, tt.wantIndex, tt.wantMode) + } + if got[0].DeviceVendor != "MR-V100" || !got[0].Health { + t.Fatalf("GetNodeDevices() vendor/health = %q/%v", got[0].DeviceVendor, got[0].Health) + } + }) + } +} + +func TestIluvatarDevicesGetResource(t *testing.T) { + config := testConfig("MR-V100") + dev := mustInitDevice(t, config) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Annotations: map[string]string{ + dev.nodeRegisterAnno: strings.Join([]string{ + "GPU-0,2,255,40,MR-V100,0,true,0,hami-core", + "GPU-1,3,257,60,MR-V100,0,true,1,hami-core", + "GPU-2,99,8192,100,MR-V100,0,false,2,hami-core", + }, ":") + ":", + }, + }} + + got := dev.GetResource(node) + if got["MR-V100-vgpu"] != 5 { + t.Errorf("count = %d, want 5", got["MR-V100-vgpu"]) + } + // The division applies after summing: floor((255 + 257) / 256) = 2. + if got["MR-V100.vMem"] != 2 { + t.Errorf("memory = %d, want 2", got["MR-V100.vMem"]) + } + if got["MR-V100.vCore"] != 100 { + t.Errorf("core = %d, want 100", got["MR-V100.vCore"]) + } +} + +func TestIluvatarDevicesGetResourceWithoutUsableAnnotation(t *testing.T) { + dev := mustInitDevice(t, testConfig("MR-V100")) + tests := []struct { + name string + annotations map[string]string + }{ + {name: "missing annotation", annotations: map[string]string{}}, + {name: "malformed annotation", annotations: map[string]string{dev.nodeRegisterAnno: "malformed"}}, + {name: "empty annotation", annotations: map[string]string{dev.nodeRegisterAnno: ":"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := dev.GetResource(&corev1.Node{ObjectMeta: metav1.ObjectMeta{Annotations: tt.annotations}}) + if len(got) != 3 || got["MR-V100-vgpu"] != 0 || got["MR-V100.vMem"] != 0 || got["MR-V100.vCore"] != 0 { + t.Fatalf("GetResource() = %v, want three zero-valued resources", got) + } + }) + } +} + +func TestInitIluvatarDevices(t *testing.T) { + validConfigs := []IluvatarConfig{ + testConfig("MR-V100"), + testConfig("MR-V50"), + testConfig("BI-V150"), + testConfig("BI-V100"), + } + devices, err := InitIluvatarDevices(validConfigs) + if err != nil { + t.Fatalf("InitIluvatarDevices(valid) error = %v", err) + } + if len(devices) != 4 { + t.Fatalf("InitIluvatarDevices(valid) returned %d devices, want 4", len(devices)) + } + for idx, dev := range devices { + if dev.CommonWord() != validConfigs[idx].CommonWord { + t.Errorf("device %d CommonWord() = %q, want %q", idx, dev.CommonWord(), validConfigs[idx].CommonWord) + } + wantAnnotation := "hami.io/node-" + validConfigs[idx].CommonWord + "-register" + if dev.nodeRegisterAnno != wantAnnotation { + t.Errorf("device %d annotation = %q, want %q", idx, dev.nodeRegisterAnno, wantAnnotation) + } + } + + if empty, err := InitIluvatarDevices(nil); err != nil || len(empty) != 0 { + t.Fatalf("InitIluvatarDevices(nil) = %v, %v; want empty, nil", empty, err) + } +} + +func TestInitIluvatarDevicesRejectsInvalidConfig(t *testing.T) { + valid := testConfig("MR-V100") + tests := []struct { + name string + configs []IluvatarConfig + }{ + {name: "missing commonWord", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.CommonWord = "" })}}, + {name: "invalid commonWord", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.CommonWord = "bad/word" })}}, + {name: "annotation key too long", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.CommonWord = strings.Repeat("a", 63) })}}, + {name: "missing chipName", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.ChipName = "" })}}, + {name: "resource without vendor", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.ResourceCountName = "vgpu" })}}, + {name: "invalid resource name", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.ResourceCountName = "iluvatar.ai/not valid" })}}, + {name: "mixed vendor namespaces", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.ResourceCoreName = "other.example/vcore" })}}, + {name: "duplicate resource within config", configs: []IluvatarConfig{changedConfig(valid, func(c *IluvatarConfig) { c.ResourceCoreName = c.ResourceMemoryName })}}, + {name: "duplicate commonWord", configs: []IluvatarConfig{valid, changedConfig(testConfig("MR-V50"), func(c *IluvatarConfig) { c.CommonWord = valid.CommonWord })}}, + {name: "duplicate resource across configs", configs: []IluvatarConfig{valid, changedConfig(testConfig("MR-V50"), func(c *IluvatarConfig) { c.ResourceCoreName = valid.ResourceCoreName })}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + devices, err := InitIluvatarDevices(tt.configs) + if err == nil { + t.Fatalf("InitIluvatarDevices() = %v, nil; want validation error", devices) + } + if devices != nil { + t.Fatalf("InitIluvatarDevices() returned partial devices on error: %v", devices) + } + }) + } +} + +func stringPointer(value string) *string { + return &value +} + +func changedConfig(config IluvatarConfig, change func(*IluvatarConfig)) IluvatarConfig { + change(&config) + return config +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 9cbd2a4..a2c6faa 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -18,6 +18,7 @@ package config import ( "flag" + "fmt" "os" "gopkg.in/yaml.v2" @@ -71,39 +72,102 @@ func LoadConfig(path string) (*Config, error) { } func InitDevicesWithConfig(config *Config) error { - device.DevicesMap = make(map[string]device.Devices) + iluvatarDevices, err := iluvatar.InitIluvatarDevices(config.IluvatarConfig) + if err != nil { + return err + } + if err := validateIluvatarCommonWordCollisions(config, iluvatarDevices); err != nil { + return err + } + if err := validateIluvatarResourceCollisions(config); err != nil { + return err + } + + devicesMap := make(map[string]device.Devices) /*amdDevice := amd.InitAMDDevice(config.AMDGPUConfig) if amdDevice != nil { - device.DevicesMap[amdDevice.CommonWord()] = amdDevice + devicesMap[amdDevice.CommonWord()] = amdDevice }*/ for _, dev := range ascend.InitDevices(config.VNPUs) { commonWord := dev.CommonWord() - device.DevicesMap[commonWord] = dev + devicesMap[commonWord] = dev klog.Infof("Ascend device %s initialized", commonWord) } /*awsNeuronDevice := awsneuron.InitAWSNeuronDevice(config.AWSNeuronConfig) if awsNeuronDevice != nil { - device.DevicesMap[awsNeuronDevice.CommonWord()] = awsNeuronDevice + devicesMap[awsNeuronDevice.CommonWord()] = awsNeuronDevice } cambriconDevice := cambricon.InitMLUDevice(config.CambriconConfig) if cambriconDevice != nil { - device.DevicesMap[cambriconDevice.CommonWord()] = cambriconDevice + devicesMap[cambriconDevice.CommonWord()] = cambriconDevice } enflameDevice := enflame.InitEnflameVGCUDevice(config.EnflameConfig) if enflameDevice != nil { - device.DevicesMap[enflameDevice.CommonWord()] = enflameDevice + devicesMap[enflameDevice.CommonWord()] = enflameDevice } kunlunDevice := kunlun.InitKunlunVDevice(config.KunlunConfig) if kunlunDevice != nil { - device.DevicesMap[kunlunDevice.CommonWord()] = kunlunDevice + devicesMap[kunlunDevice.CommonWord()] = kunlunDevice }*/ hygonDevice := hygon.InitDCUDevice(config.HygonConfig) if hygonDevice != nil { - device.DevicesMap[hygonDevice.CommonWord()] = hygonDevice + devicesMap[hygonDevice.CommonWord()] = hygonDevice } nvidiaDevice := nvidia.InitNvidiaDevice(config.NvidiaConfig) if nvidiaDevice != nil { - device.DevicesMap[nvidiaDevice.CommonWord()] = nvidiaDevice + devicesMap[nvidiaDevice.CommonWord()] = nvidiaDevice + } + for _, dev := range iluvatarDevices { + devicesMap[dev.CommonWord()] = dev + klog.Infof("Iluvatar device %s initialized", dev.CommonWord()) + } + device.DevicesMap = devicesMap + return nil +} + +func validateIluvatarCommonWordCollisions(config *Config, iluvatarDevices []*iluvatar.IluvatarDevices) error { + reserved := map[string]string{ + hygon.HygonDCUCommonWord: "Hygon", + nvidia.NvidiaGPUDevice: "NVIDIA", + } + for _, vnpuConfig := range config.VNPUs.Configs { + reserved[vnpuConfig.CommonWord] = "Ascend " + vnpuConfig.CommonWord + } + for _, dev := range iluvatarDevices { + if owner, exists := reserved[dev.CommonWord()]; exists { + return fmt.Errorf("iluvatar commonWord %q conflicts with %s", dev.CommonWord(), owner) + } + } + return nil +} + +func validateIluvatarResourceCollisions(config *Config) error { + reserved := make(map[string]string) + add := func(owner, name string) { + if name != "" { + reserved[name] = owner + } + } + + add("NVIDIA", config.NvidiaConfig.ResourceCountName) + add("NVIDIA", config.NvidiaConfig.ResourceMemoryName) + add("NVIDIA", config.NvidiaConfig.ResourceCoreName) + add("NVIDIA", config.NvidiaConfig.ResourceMemoryPercentageName) + add("Hygon", config.HygonConfig.ResourceCountName) + add("Hygon", config.HygonConfig.ResourceMemoryName) + add("Hygon", config.HygonConfig.ResourceCoreName) + for _, vnpuConfig := range config.VNPUs.Configs { + add("Ascend "+vnpuConfig.CommonWord, vnpuConfig.ResourceName) + add("Ascend "+vnpuConfig.CommonWord, vnpuConfig.ResourceMemoryName) + add("Ascend "+vnpuConfig.CommonWord, vnpuConfig.ResourceCoreName) + } + + for idx, iluvatarConfig := range config.IluvatarConfig { + for _, name := range []string{iluvatarConfig.ResourceCountName, iluvatarConfig.ResourceMemoryName, iluvatarConfig.ResourceCoreName} { + if owner, exists := reserved[name]; exists { + return fmt.Errorf("iluvatar config at index %d reuses resource name %q from %s", idx, name, owner) + } + } } return nil } diff --git a/internal/pkg/config/iluvatar_test.go b/internal/pkg/config/iluvatar_test.go new file mode 100644 index 0000000..c5cb23c --- /dev/null +++ b/internal/pkg/config/iluvatar_test.go @@ -0,0 +1,140 @@ +/* +Copyright 2025 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "testing" + + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device" + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device/hygon" + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device/iluvatar" + "github.com/HAMi/mock-device-plugin/internal/pkg/api/device/nvidia" +) + +func iluvatarConfig(commonWord string) iluvatar.IluvatarConfig { + return iluvatar.IluvatarConfig{ + CommonWord: commonWord, + ChipName: commonWord, + ResourceCountName: "iluvatar.ai/" + commonWord + "-vgpu", + ResourceMemoryName: "iluvatar.ai/" + commonWord + ".vMem", + ResourceCoreName: "iluvatar.ai/" + commonWord + ".vCore", + } +} + +func TestInitDevicesWithConfigRegistersIluvatarDevices(t *testing.T) { + config := &Config{IluvatarConfig: []iluvatar.IluvatarConfig{ + iluvatarConfig("MR-V100"), + iluvatarConfig("MR-V50"), + iluvatarConfig("BI-V150"), + iluvatarConfig("BI-V100"), + }} + + if err := InitDevicesWithConfig(config); err != nil { + t.Fatalf("InitDevicesWithConfig() error = %v", err) + } + for _, commonWord := range []string{"MR-V100", "MR-V50", "BI-V150", "BI-V100"} { + dev, exists := device.DevicesMap[commonWord] + if !exists { + t.Errorf("device %q was not registered", commonWord) + continue + } + if dev.CommonWord() != commonWord { + t.Errorf("device %q CommonWord() = %q", commonWord, dev.CommonWord()) + } + } +} + +func TestInitDevicesWithConfigRejectsIluvatarConstructorCollision(t *testing.T) { + if err := InitDevicesWithConfig(&Config{}); err != nil { + t.Fatalf("InitDevicesWithConfig() setup error = %v", err) + } + duplicate := iluvatarConfig("MR-V100") + config := &Config{IluvatarConfig: []iluvatar.IluvatarConfig{duplicate, duplicate}} + + if err := InitDevicesWithConfig(config); err == nil { + t.Fatal("InitDevicesWithConfig() error = nil, want duplicate-config error") + } + if _, exists := device.DevicesMap["MR-V100"]; exists { + t.Fatal("InitDevicesWithConfig() partially registered Iluvatar devices on constructor error") + } +} + +func TestInitDevicesWithConfigRejectsExistingCommonWordCollision(t *testing.T) { + initialHygon := hygon.HygonConfig{ + ResourceCountName: "old.example/dcu", + ResourceMemoryName: "old.example/dcumem", + ResourceCoreName: "old.example/dcucores", + MemoryFactor: 1, + } + if err := InitDevicesWithConfig(&Config{HygonConfig: initialHygon}); err != nil { + t.Fatalf("InitDevicesWithConfig() setup error = %v", err) + } + previousNvidia := device.DevicesMap[nvidia.NvidiaGPUDevice] + previousLen := len(device.DevicesMap) + config := &Config{ + HygonConfig: hygon.HygonConfig{ + ResourceCountName: "new.example/dcu", + ResourceMemoryName: "new.example/dcumem", + ResourceCoreName: "new.example/dcucores", + MemoryFactor: 2, + }, + IluvatarConfig: []iluvatar.IluvatarConfig{iluvatarConfig("NVIDIA")}, + } + + if err := InitDevicesWithConfig(config); err == nil { + t.Fatal("InitDevicesWithConfig() error = nil, want CommonWord collision error") + } + if len(device.DevicesMap) != previousLen || device.DevicesMap[nvidia.NvidiaGPUDevice] != previousNvidia { + t.Fatalf("failed initialization changed DevicesMap: %#v", device.DevicesMap) + } + if hygon.HygonResourceCount != initialHygon.ResourceCountName || + hygon.HygonResourceMemory != initialHygon.ResourceMemoryName || + hygon.HygonResourceCores != initialHygon.ResourceCoreName || + hygon.MemoryFactor != initialHygon.MemoryFactor { + t.Fatalf("failed initialization changed Hygon globals: count=%q memory=%q core=%q factor=%d", + hygon.HygonResourceCount, hygon.HygonResourceMemory, hygon.HygonResourceCores, hygon.MemoryFactor) + } +} + +func TestInitDevicesWithConfigRejectsIluvatarResourceCollisionAtomically(t *testing.T) { + if err := InitDevicesWithConfig(&Config{}); err != nil { + t.Fatalf("initial InitDevicesWithConfig() error = %v", err) + } + previousNvidia := device.DevicesMap[nvidia.NvidiaGPUDevice] + previousLen := len(device.DevicesMap) + + iluvatarConfig := iluvatarConfig("MR-V100") + iluvatarConfig.ResourceMemoryName = "nvidia.com/gpumem" + iluvatarConfig.ResourceCountName = "nvidia.com/iluvatar-vgpu" + iluvatarConfig.ResourceCoreName = "nvidia.com/iluvatar-vcore" + config := &Config{ + NvidiaConfig: nvidia.NvidiaConfig{ResourceMemoryName: "nvidia.com/gpumem"}, + IluvatarConfig: []iluvatar.IluvatarConfig{ + iluvatarConfig, + }, + } + + if err := InitDevicesWithConfig(config); err == nil { + t.Fatal("InitDevicesWithConfig() error = nil, want resource collision error") + } + if len(device.DevicesMap) != previousLen || device.DevicesMap[nvidia.NvidiaGPUDevice] != previousNvidia { + t.Fatalf("failed initialization changed DevicesMap: %#v", device.DevicesMap) + } + if _, exists := device.DevicesMap[iluvatarConfig.CommonWord]; exists { + t.Fatalf("failed initialization registered Iluvatar device %q", iluvatarConfig.CommonWord) + } +}