diff --git a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml index 561d9fc3c..ccfd7930a 100644 --- a/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml +++ b/helm/bundles/cortex-nova/templates/pipelines_kvm.yaml @@ -557,4 +557,40 @@ spec: VM is allocated get a higher weight, encouraging placement on pre-reserved failover capacity. For non-evacuation requests, this weigher has no effect. +--- +apiVersion: cortex.cloud/v1alpha1 +kind: Pipeline +metadata: + name: kvm-report-capacity +spec: + schedulingDomain: nova + description: | + This pipeline is used by the Liquid capacity reporter to determine the + theoretical maximum capacity of each flavor group per availability zone, + as if all hosts were completely empty. It ignores current VM allocations + and all reservation blockings so that only raw hardware capacity is + considered. + type: filter-weigher + createDecisions: false + # Fetch all placement candidates, ignoring nova's preselection. + ignorePreselection: true + filters: + - name: filter_correct_az + description: | + Restricts host candidates to the requested availability zone. + - name: filter_has_enough_capacity + description: | + Filters hosts that cannot fit the flavor based on raw hardware capacity. + VM allocations and all reservation types are ignored to represent an + empty datacenter scenario. + params: + - {key: ignoreAllocations, boolValue: true} + - {key: ignoredReservationTypes, stringListValue: ["CommittedResourceReservation", "FailoverReservation"]} + - name: filter_has_requested_traits + description: | + Ensures hosts have the hardware traits required by the flavor. + - name: filter_status_conditions + description: | + Excludes hosts that are not ready or are disabled. + weighers: [] {{- end }} diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index d32316e47..ce9839d32 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -146,6 +146,10 @@ cortex-scheduling-controllers: "*": "kvm-general-purpose-load-balancing" # Catch-all fallback # Default pipeline for CR reservations when no CommittedResourceFlavorGroupPipelines entry matches committedResourcePipelineDefault: "kvm-general-purpose-load-balancing" + # Pipeline used for currently-available capacity check (respects VM allocations and reservations) + reportCapacityCurrentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled" + # Pipeline used for total theoretical capacity check (ignores VM allocations and reservations) + reportCapacityTotalPipeline: "kvm-report-capacity" # How often to re-verify active reservations # 5m = 300000000000 nanoseconds committedResourceRequeueIntervalActive: 300000000000 diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index d26c7c940..26382ad4f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -26,6 +26,10 @@ type FilterHasEnoughCapacityOpts struct { // When a reservation type is in this list, its capacity is not blocked. // Default: empty (all reservation types are considered) IgnoredReservationTypes []v1alpha1.ReservationType `json:"ignoredReservationTypes,omitempty"` + + // IgnoreAllocations skips subtracting current VM allocations from host capacity. + // When true, only raw hardware capacity is considered (empty datacenter scenario). + IgnoreAllocations bool `json:"ignoreAllocations,omitempty"` } func (FilterHasEnoughCapacityOpts) Validate() error { return nil } @@ -71,18 +75,20 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa freeResourcesByHost[hv.Name] = hv.Status.EffectiveCapacity } - // Subtract allocated resources. - for resourceName, allocated := range hv.Status.Allocation { - free, ok := freeResourcesByHost[hv.Name][resourceName] - if !ok { - traceLog.Error( - "hypervisor with allocation for unknown resource", - "host", hv.Name, "resource", resourceName, - ) - continue + // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). + if !s.Options.IgnoreAllocations { + for resourceName, allocated := range hv.Status.Allocation { + free, ok := freeResourcesByHost[hv.Name][resourceName] + if !ok { + traceLog.Error( + "hypervisor with allocation for unknown resource", + "host", hv.Name, "resource", resourceName, + ) + continue + } + free.Sub(allocated) + freeResourcesByHost[hv.Name][resourceName] = free } - free.Sub(allocated) - freeResourcesByHost[hv.Name][resourceName] = free } } diff --git a/internal/scheduling/reservations/commitments/api.go b/internal/scheduling/reservations/commitments/api.go index 06fb97be1..0d81db2aa 100644 --- a/internal/scheduling/reservations/commitments/api.go +++ b/internal/scheduling/reservations/commitments/api.go @@ -12,6 +12,7 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/nova" "github.com/go-logr/logr" "github.com/prometheus/client_golang/prometheus" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -78,3 +79,5 @@ func (api *HTTPAPI) handleProjectEndpoint(w http.ResponseWriter, r *http.Request http.Error(w, "Not found", http.StatusNotFound) } } + +var commitmentApiLog = ctrl.Log.WithName("commitment_api") diff --git a/internal/scheduling/reservations/commitments/api_report_capacity.go b/internal/scheduling/reservations/commitments/api_report_capacity.go index 09fc55168..9d084b211 100644 --- a/internal/scheduling/reservations/commitments/api_report_capacity.go +++ b/internal/scheduling/reservations/commitments/api_report_capacity.go @@ -11,7 +11,6 @@ import ( "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" "github.com/google/uuid" - "github.com/sapcc/go-api-declarations/liquid" ) // handles POST /commitments/v1/report-capacity requests from Limes: @@ -50,16 +49,9 @@ func (api *HTTPAPI) HandleReportCapacity(w http.ResponseWriter, r *http.Request) logger.V(1).Info("processing report capacity request") - // Parse request body (may be empty or contain ServiceCapacityRequest) - var req liquid.ServiceCapacityRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - // Empty body is acceptable for capacity reports - req = liquid.ServiceCapacityRequest{} - } - // Calculate capacity - calculator := NewCapacityCalculator(api.client) - report, err := calculator.CalculateCapacity(ctx, req) + calculator := NewCapacityCalculator(api.client, api.config) + report, err := calculator.CalculateCapacity(ctx) if err != nil { logger.Error(err, "failed to calculate capacity") statusCode = http.StatusInternalServerError diff --git a/internal/scheduling/reservations/commitments/api_report_capacity_test.go b/internal/scheduling/reservations/commitments/api_report_capacity_test.go index b151382cf..c9957f16e 100644 --- a/internal/scheduling/reservations/commitments/api_report_capacity_test.go +++ b/internal/scheduling/reservations/commitments/api_report_capacity_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "slices" "strings" "testing" @@ -18,7 +17,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" + novaapi "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) func TestHandleReportCapacity(t *testing.T) { @@ -135,11 +136,8 @@ func TestCapacityCalculator(t *testing.T) { WithScheme(scheme). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"az-one", "az-two"}, - } - _, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + _, err := calculator.CalculateCapacity(context.Background()) if err == nil { t.Fatal("Expected error when flavor groups knowledge doesn't exist, got nil") } @@ -157,11 +155,8 @@ func TestCapacityCalculator(t *testing.T) { WithObjects(emptyKnowledge). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"az-one", "az-two"}, - } - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -175,18 +170,19 @@ func TestCapacityCalculator(t *testing.T) { } }) - t.Run("CalculateCapacity returns perAZ entries for all AZs from request", func(t *testing.T) { + t.Run("CalculateCapacity returns perAZ entries for all AZs from host details", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") + hostDetails := createTestHostDetailsKnowledge(t, map[string]string{ + "host-1": "qa-de-1a", + "host-2": "qa-de-1b", + }) fakeClient := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(flavorGroupKnowledge). + WithObjects(flavorGroupKnowledge, hostDetails). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"qa-de-1a", "qa-de-1b", "qa-de-1d"}, - } - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -195,22 +191,32 @@ func TestCapacityCalculator(t *testing.T) { t.Fatalf("Expected 3 resources (_ram, _cores, _instances), got %d", len(report.Resources)) } - // Verify all resources have exactly the requested AZs - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_ram"], req.AllAZs) - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_cores"], req.AllAZs) - verifyPerAZMatchesRequest(t, report.Resources["hw_version_test-group_instances"], req.AllAZs) + // Verify all resources have entries for the AZs from host details + expectedAZs := []liquid.AvailabilityZone{"qa-de-1a", "qa-de-1b"} + for _, resName := range []string{"hw_version_test-group_ram", "hw_version_test-group_cores", "hw_version_test-group_instances"} { + res := report.Resources[liquid.ResourceName(resName)] + if res == nil { + t.Errorf("resource %s not found", resName) + continue + } + for _, az := range expectedAZs { + if _, ok := res.PerAZ[az]; !ok { + t.Errorf("%s: missing entry for AZ %s", resName, az) + } + } + } }) - t.Run("CalculateCapacity with empty AllAZs returns empty perAZ maps", func(t *testing.T) { + t.Run("CalculateCapacity with no host details returns empty perAZ maps", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") + // No host details knowledge - no AZs can be derived. fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(flavorGroupKnowledge). Build() - calculator := NewCapacityCalculator(fakeClient) - req := liquid.ServiceCapacityRequest{AllAZs: []liquid.AvailabilityZone{}} - report, err := calculator.CalculateCapacity(context.Background(), req) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } @@ -226,64 +232,37 @@ func TestCapacityCalculator(t *testing.T) { } }) - t.Run("CalculateCapacity responds to different AZ sets correctly", func(t *testing.T) { + t.Run("CalculateCapacity produces perAZ entries matching host details AZs", func(t *testing.T) { flavorGroupKnowledge := createTestFlavorGroupKnowledge(t, "test-group") + hostDetails := createTestHostDetailsKnowledge(t, map[string]string{ + "host-a": "eu-de-1a", + "host-b": "eu-de-1b", + }) fakeClient := fake.NewClientBuilder(). WithScheme(scheme). - WithObjects(flavorGroupKnowledge). + WithObjects(flavorGroupKnowledge, hostDetails). Build() - calculator := NewCapacityCalculator(fakeClient) - - req1 := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"eu-de-1a", "eu-de-1b"}, - } - report1, err := calculator.CalculateCapacity(context.Background(), req1) - if err != nil { - t.Fatalf("Expected no error, got: %v", err) - } - - req2 := liquid.ServiceCapacityRequest{ - AllAZs: []liquid.AvailabilityZone{"us-west-1a", "us-west-1b", "us-west-1c", "us-west-1d"}, - } - report2, err := calculator.CalculateCapacity(context.Background(), req2) + calculator := NewCapacityCalculator(fakeClient, DefaultConfig()) + report, err := calculator.CalculateCapacity(context.Background()) if err != nil { t.Fatalf("Expected no error, got: %v", err) } - // Verify reports have exactly the requested AZs - for _, res := range report1.Resources { - verifyPerAZMatchesRequest(t, res, req1.AllAZs) - } - for _, res := range report2.Resources { - verifyPerAZMatchesRequest(t, res, req2.AllAZs) + // Verify resources contain exactly the AZs from host details + for resName, res := range report.Resources { + if len(res.PerAZ) != 2 { + t.Errorf("%s: expected 2 AZs, got %d", resName, len(res.PerAZ)) + } + for _, az := range []liquid.AvailabilityZone{"eu-de-1a", "eu-de-1b"} { + if _, ok := res.PerAZ[az]; !ok { + t.Errorf("%s: missing entry for AZ %s", resName, az) + } + } } }) } -// verifyPerAZMatchesRequest checks that perAZ entries match exactly the requested AZs. -// This follows the same semantics as nova liquid: the response must contain -// entries for all AZs in AllAZs, no more and no less. -func verifyPerAZMatchesRequest(t *testing.T, res *liquid.ResourceCapacityReport, requestedAZs []liquid.AvailabilityZone) { - t.Helper() - if res == nil { - t.Error("resource is nil") - return - } - if len(res.PerAZ) != len(requestedAZs) { - t.Errorf("expected %d AZs, got %d", len(requestedAZs), len(res.PerAZ)) - } - for _, az := range requestedAZs { - if _, ok := res.PerAZ[az]; !ok { - t.Errorf("missing entry for requested AZ %s", az) - } - } - for az := range res.PerAZ { - if !slices.Contains(requestedAZs, az) { - t.Errorf("unexpected AZ %s in response (not in request)", az) - } - } -} // createEmptyFlavorGroupKnowledge creates an empty flavor groups Knowledge CRD func createEmptyFlavorGroupKnowledge() *v1alpha1.Knowledge { @@ -378,3 +357,298 @@ func createTestFlavorGroupKnowledge(t *testing.T, groupName string) *v1alpha1.Kn }, } } + +func TestCapacityCalculatorWithScheduler(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + const ( + flavorGroup = "test-group" + az = "az-a" + flavorMemMB = uint64(32768) + flavorVCPUs = uint64(8) + ) + + flavorGroupKnowledge := createTestFlavorGroupKnowledgeWithSmallest(t, flavorGroup, flavorMemMB, flavorVCPUs) + + t.Run("computes capacity and usage via two scheduler calls", func(t *testing.T) { + // kvm-report-capacity returns 5 hosts (total capacity). + // kvm-general-purpose-load-balancing-all-filters-enabled returns 3 hosts (currently available). + // usage = 5 - 3 = 2. + server := newPipelineMockSchedulerServer(t, map[string][]string{ + "kvm-report-capacity": {"h1", "h2", "h3", "h4", "h5"}, + "kvm-general-purpose-load-balancing-all-filters-enabled": {"h1", "h2", "h3"}, + }) + defer server.Close() + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + currentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + groupData, ok := groups[flavorGroup] + if !ok { + t.Fatalf("flavor group %q not found", flavorGroup) + } + + capacity, usage, err := calculator.calculateInstanceCapacity(context.Background(), flavorGroup, groupData, az) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if capacity != 5 { + t.Errorf("expected capacity = 5, got %d", capacity) + } + if usage != 2 { + t.Errorf("expected usage = 2, got %d", usage) + } + }) + + t.Run("usage is zero when total equals currently available", func(t *testing.T) { + server := newPipelineMockSchedulerServer(t, map[string][]string{ + "kvm-report-capacity": {"h1", "h2"}, + "kvm-general-purpose-load-balancing-all-filters-enabled": {"h1", "h2"}, + }) + defer server.Close() + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + currentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + _, usage, err := calculator.calculateInstanceCapacity(context.Background(), flavorGroup, groups[flavorGroup], az) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if usage != 0 { + t.Errorf("expected usage = 0, got %d", usage) + } + }) + + t.Run("usage is clamped to zero when currently available exceeds total", func(t *testing.T) { + // Pathological: currently-available call returns more hosts than total capacity call. + server := newPipelineMockSchedulerServer(t, map[string][]string{ + "kvm-report-capacity": {"h1"}, + "kvm-general-purpose-load-balancing-all-filters-enabled": {"h1", "h2", "h3"}, + }) + defer server.Close() + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + currentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + _, usage, err := calculator.calculateInstanceCapacity(context.Background(), flavorGroup, groups[flavorGroup], az) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if usage != 0 { + t.Errorf("expected usage = 0 (clamped), got %d", usage) + } + }) + + t.Run("scheduler failure returns error", func(t *testing.T) { + failServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "internal error", http.StatusInternalServerError) + })) + defer failServer.Close() + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(failServer.URL), + currentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + totalPipeline: "kvm-report-capacity", + } + + knowledge := &reservations.FlavorGroupKnowledgeClient{Client: fakeClient} + groups, err := knowledge.GetAllFlavorGroups(context.Background(), nil) + if err != nil { + t.Fatalf("failed to get flavor groups: %v", err) + } + _, _, err = calculator.calculateInstanceCapacity(context.Background(), flavorGroup, groups[flavorGroup], az) + if err == nil { + t.Fatal("expected error on scheduler failure, got nil") + } + }) + + t.Run("multiple AZs are reported independently", func(t *testing.T) { + twoAZHostDetails := createTestHostDetailsKnowledge(t, map[string]string{ + "host-1": "az-a", + "host-2": "az-b", + }) + // Both calls always return 3 hosts regardless of AZ (pipeline-routing mock). + server := newPipelineMockSchedulerServer(t, map[string][]string{ + "kvm-report-capacity": {"h1", "h2", "h3"}, + "kvm-general-purpose-load-balancing-all-filters-enabled": {"h1"}, + }) + defer server.Close() + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(flavorGroupKnowledge, twoAZHostDetails). + Build() + + calculator := &CapacityCalculator{ + client: fakeClient, + schedulerClient: reservations.NewSchedulerClient(server.URL), + currentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + totalPipeline: "kvm-report-capacity", + } + + report, err := calculator.CalculateCapacity(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + res := report.Resources[liquid.ResourceName(ResourceNameRAM(flavorGroup))] + if len(res.PerAZ) != 2 { + t.Errorf("expected 2 AZs, got %d", len(res.PerAZ)) + } + if _, ok := res.PerAZ[liquid.AvailabilityZone("az-a")]; !ok { + t.Error("expected az-a in report") + } + if _, ok := res.PerAZ[liquid.AvailabilityZone("az-b")]; !ok { + t.Error("expected az-b in report") + } + }) +} + +// newPipelineMockSchedulerServer starts a test HTTP server that returns different +// host lists depending on the pipeline name in the request body. +func newPipelineMockSchedulerServer(t *testing.T, hostsByPipeline map[string][]string) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req novaapi.ExternalSchedulerRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "bad request", http.StatusBadRequest) + return + } + hosts := hostsByPipeline[req.Pipeline] + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(novaapi.ExternalSchedulerResponse{Hosts: hosts}); err != nil { + t.Errorf("mock scheduler: encode error: %v", err) + } + })) +} + +// createTestFlavorGroupKnowledgeWithSmallest creates a Knowledge CRD where smallestFlavor +// is explicitly set so the capacity calculator uses the correct memory unit. +func createTestFlavorGroupKnowledgeWithSmallest(t *testing.T, groupName string, memMB, vcpus uint64) *v1alpha1.Knowledge { + t.Helper() + + features := []map[string]interface{}{ + { + "name": groupName, + "flavors": []map[string]interface{}{ + { + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + }, + "smallestFlavor": map[string]interface{}{ + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + "largestFlavor": map[string]interface{}{ + "name": "test_flavor", + "vcpus": vcpus, + "memoryMB": memMB, + "diskGB": 50, + }, + }, + } + + raw, err := v1alpha1.BoxFeatureList(features) + if err != nil { + t.Fatal(err) + } + + return &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: "flavor-groups"}, + Spec: v1alpha1.KnowledgeSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + Extractor: v1alpha1.KnowledgeExtractorSpec{Name: "flavor_groups"}, + }, + Status: v1alpha1.KnowledgeStatus{ + Conditions: []v1.Condition{{Type: v1alpha1.KnowledgeConditionReady, Status: "True"}}, + Raw: raw, + }, + } +} + +// createTestHostDetailsKnowledge creates a Knowledge CRD with host→AZ mappings. +func createTestHostDetailsKnowledge(t *testing.T, hostToAZ map[string]string) *v1alpha1.Knowledge { + t.Helper() + + features := make([]map[string]interface{}, 0, len(hostToAZ)) + for host, az := range hostToAZ { + features = append(features, map[string]interface{}{ + "ComputeHost": host, + "AvailabilityZone": az, + }) + } + + raw, err := v1alpha1.BoxFeatureList(features) + if err != nil { + t.Fatal(err) + } + + return &v1alpha1.Knowledge{ + ObjectMeta: v1.ObjectMeta{Name: "host-details"}, + Spec: v1alpha1.KnowledgeSpec{ + SchedulingDomain: v1alpha1.SchedulingDomainNova, + Extractor: v1alpha1.KnowledgeExtractorSpec{Name: "sap_host_details_extractor"}, + }, + Status: v1alpha1.KnowledgeStatus{ + Conditions: []v1.Condition{{Type: v1alpha1.KnowledgeConditionReady, Status: "True"}}, + Raw: raw, + }, + } +} diff --git a/internal/scheduling/reservations/commitments/capacity.go b/internal/scheduling/reservations/commitments/capacity.go index 8cd3a7159..067e066a7 100644 --- a/internal/scheduling/reservations/commitments/capacity.go +++ b/internal/scheduling/reservations/commitments/capacity.go @@ -6,28 +6,40 @@ package commitments import ( "context" "fmt" + "sort" + "time" - "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" - "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" . "github.com/majewsky/gg/option" "github.com/sapcc/go-api-declarations/liquid" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/knowledge/extractor/plugins/compute" + "github.com/cobaltcore-dev/cortex/internal/scheduling/reservations" ) // CapacityCalculator computes capacity reports for Limes LIQUID API. type CapacityCalculator struct { - client client.Client + client client.Client + schedulerClient *reservations.SchedulerClient + currentPipeline string + totalPipeline string } -func NewCapacityCalculator(client client.Client) *CapacityCalculator { - return &CapacityCalculator{client: client} +func NewCapacityCalculator(client client.Client, config Config) *CapacityCalculator { + return &CapacityCalculator{ + client: client, + schedulerClient: reservations.NewSchedulerClient(config.SchedulerURL), + currentPipeline: config.ReportCapacityCurrentPipeline, + totalPipeline: config.ReportCapacityTotalPipeline, + } } // CalculateCapacity computes per-AZ capacity for all flavor groups. // For each flavor group, three resources are reported: _ram, _cores, _instances. // All flavor groups are included, not just those with fixed RAM/core ratio. -// The request provides the list of all AZs from Limes that must be included in the report. -func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.ServiceCapacityRequest) (liquid.ServiceCapacityReport, error) { +// AZs are derived from HostDetails Knowledge CRDs. +func (c *CapacityCalculator) CalculateCapacity(ctx context.Context) (liquid.ServiceCapacityReport, error) { // Get all flavor groups from Knowledge CRDs knowledge := &reservations.FlavorGroupKnowledgeClient{Client: c.client} flavorGroups, err := knowledge.GetAllFlavorGroups(ctx, nil) @@ -41,6 +53,12 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S infoVersion = knowledgeCRD.Status.LastContentChange.Unix() } + // Get availability zones from host details + azs, err := c.getAvailabilityZones(ctx) + if err != nil { + return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to get availability zones: %w", err) + } + // Build capacity report for all flavor groups report := liquid.ServiceCapacityReport{ InfoVersion: infoVersion, @@ -48,10 +66,11 @@ func (c *CapacityCalculator) CalculateCapacity(ctx context.Context, req liquid.S } for groupName, groupData := range flavorGroups { - // All flavor groups are included in capacity reporting (not just those with fixed ratio). - - // Calculate per-AZ capacity (placeholder: capacity=0 for all resources) - azCapacity := c.calculateAZCapacity(groupName, groupData, req.AllAZs) + // Calculate per-AZ capacity using scheduler + azCapacity, err := c.calculateAZCapacity(ctx, groupName, groupData, azs) + if err != nil { + return liquid.ServiceCapacityReport{}, fmt.Errorf("failed to calculate capacity for %s: %w", groupName, err) + } // === 1. RAM Resource === ramResourceName := liquid.ResourceName(ResourceNameRAM(groupName)) @@ -96,30 +115,136 @@ func (c *CapacityCalculator) copyAZCapacity( return result } +// calculateAZCapacity computes capacity per AZ for a flavor group via scheduler calls. +// On scheduler failure for an AZ, that AZ still gets an entry with capacity=0. func (c *CapacityCalculator) calculateAZCapacity( - _ string, // groupName - reserved for future use - _ compute.FlavorGroupFeature, // groupData - reserved for future use - allAZs []liquid.AvailabilityZone, // list of all AZs from Limes request -) map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport { + ctx context.Context, + groupName string, + groupData compute.FlavorGroupFeature, + azs []string, +) (map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport, error) { - // Create report entry for each AZ with placeholder capacity=0. - // - // NOTE: When implementing real capacity calculation here, you MUST also update - // the copying logic in CalculateCapacity() for _cores and _instances resources. - // Those resources use different units (vCPUs and VM count) than _ram (memory multiples), - // so the capacity values cannot be simply copied - they require unit conversion: - // - _cores capacity = RAM capacity / ramCoreRatio - // - _instances capacity = needs its own derivation logic - // - // TODO: Calculate actual capacity from Reservation CRDs or host resources - // TODO: Calculate actual usage from VM allocations result := make(map[liquid.AvailabilityZone]*liquid.AZResourceCapacityReport) - for _, az := range allAZs { - result[az] = &liquid.AZResourceCapacityReport{ - Capacity: 0, // Placeholder: capacity=0 until actual calculation is implemented - Usage: Some[uint64](0), // Placeholder: usage=0 until actual calculation is implemented + for _, az := range azs { + capacity, usage, err := c.calculateInstanceCapacity(ctx, groupName, groupData, az) + if err != nil { + // On failure, report az with capacity=0 rather than aborting entirely. + result[liquid.AvailabilityZone(az)] = &liquid.AZResourceCapacityReport{ + Capacity: 0, + Usage: Some[uint64](0), + } + continue + } + result[liquid.AvailabilityZone(az)] = &liquid.AZResourceCapacityReport{ + Capacity: capacity, + Usage: Some[uint64](usage), } } + return result, nil +} - return result +// calculateInstanceCapacity returns the total capacity and current usage for a flavor group in an AZ. +// Capacity is expressed in multiples of the smallest flavor's memory. +// Total capacity is derived directly from Hypervisor CRDs (as if everything were empty). +// Currently available is derived from the scheduler (respecting current VM and reservation state). +// Usage = totalCapacity - currentlyAvailable. +func (c *CapacityCalculator) calculateInstanceCapacity( + ctx context.Context, + groupName string, + groupData compute.FlavorGroupFeature, + az string, +) (capacity, usage uint64, err error) { + + smallestFlavor := groupData.SmallestFlavor + + // Request 1: currently available — how many instances can be placed right now. + currentResp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ + InstanceUUID: fmt.Sprintf("capacity-current-%s-%s-%d", groupName, az, time.Now().UnixNano()), + ProjectID: "cortex-capacity-check", + FlavorName: smallestFlavor.Name, + MemoryMB: smallestFlavor.MemoryMB, + VCPUs: smallestFlavor.VCPUs, + FlavorExtraSpecs: map[string]string{"hw_version": groupName}, + AvailabilityZone: az, + Pipeline: c.currentPipeline, + }) + if err != nil { + return 0, 0, fmt.Errorf("failed to get current available capacity: %w", err) + } + currentlyAvailable := uint64(len(currentResp.Hosts)) + + // Request 2: total capacity — hosts eligible if everything were empty. + // Uses a dedicated pipeline that ignores VM allocations and all reservations. + totalResp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ + InstanceUUID: fmt.Sprintf("capacity-total-%s-%s-%d", groupName, az, time.Now().UnixNano()), + ProjectID: "cortex-capacity-check", + FlavorName: smallestFlavor.Name, + MemoryMB: smallestFlavor.MemoryMB, + VCPUs: smallestFlavor.VCPUs, + FlavorExtraSpecs: map[string]string{"hw_version": groupName}, + AvailabilityZone: az, + Pipeline: c.totalPipeline, + }) + if err != nil { + return 0, 0, fmt.Errorf("failed to get total capacity: %w", err) + } + totalCapacity := uint64(len(totalResp.Hosts)) + + var usageValue uint64 + if totalCapacity >= currentlyAvailable { + usageValue = totalCapacity - currentlyAvailable + } + + return totalCapacity, usageValue, nil +} + +// getHostAZMap returns a map from compute host name to availability zone. +func (c *CapacityCalculator) getHostAZMap(ctx context.Context) (map[string]string, error) { + var knowledgeList v1alpha1.KnowledgeList + if err := c.client.List(ctx, &knowledgeList); err != nil { + return nil, fmt.Errorf("failed to list Knowledge CRDs: %w", err) + } + + type hostAZEntry struct { + ComputeHost string `json:"ComputeHost"` + AvailabilityZone string `json:"AvailabilityZone"` + } + + hostAZMap := make(map[string]string) + for _, knowledge := range knowledgeList.Items { + if knowledge.Spec.Extractor.Name != "sap_host_details_extractor" { + continue + } + features, err := v1alpha1.UnboxFeatureList[hostAZEntry](knowledge.Status.Raw) + if err != nil { + continue + } + for _, feature := range features { + if feature.ComputeHost != "" && feature.AvailabilityZone != "" { + hostAZMap[feature.ComputeHost] = feature.AvailabilityZone + } + } + } + + return hostAZMap, nil +} + +func (c *CapacityCalculator) getAvailabilityZones(ctx context.Context) ([]string, error) { + hostAZMap, err := c.getHostAZMap(ctx) + if err != nil { + return nil, err + } + + azSet := make(map[string]struct{}) + for _, az := range hostAZMap { + azSet[az] = struct{}{} + } + + azs := make([]string, 0, len(azSet)) + for az := range azSet { + azs = append(azs, az) + } + sort.Strings(azs) + + return azs, nil } diff --git a/internal/scheduling/reservations/commitments/config.go b/internal/scheduling/reservations/commitments/config.go index 7a6c9005f..4579e41d3 100644 --- a/internal/scheduling/reservations/commitments/config.go +++ b/internal/scheduling/reservations/commitments/config.go @@ -30,6 +30,14 @@ type Config struct { // Secret ref to the database credentials for querying VM state. DatabaseSecretRef *corev1.SecretReference `json:"databaseSecretRef,omitempty"` + // ReportCapacityCurrentPipeline is the pipeline used to determine currently available capacity + // (respects VM allocations and reservations). + ReportCapacityCurrentPipeline string `json:"reportCapacityCurrentPipeline"` + + // ReportCapacityTotalPipeline is the pipeline used to determine total theoretical capacity + // (ignores VM allocations and reservations). + ReportCapacityTotalPipeline string `json:"reportCapacityTotalPipeline"` + // FlavorGroupPipelines maps flavor group names to pipeline names. // Example: {"2152": "kvm-hana-bin-packing", "2101": "kvm-general-purpose-load-balancing", "*": "kvm-general-purpose-load-balancing"} // Used to select different scheduling pipelines based on flavor group characteristics. @@ -90,6 +98,8 @@ func DefaultConfig() Config { RequeueIntervalRetry: 1 * time.Minute, PipelineDefault: "kvm-general-purpose-load-balancing", SchedulerURL: "http://localhost:8080/scheduler/nova/external", + ReportCapacityCurrentPipeline: "kvm-general-purpose-load-balancing-all-filters-enabled", + ReportCapacityTotalPipeline: "kvm-report-capacity", ChangeAPIWatchReservationsTimeout: 10 * time.Second, ChangeAPIWatchReservationsPollInterval: 500 * time.Millisecond, EnableChangeCommitmentsAPI: true,