diff --git a/api/scheduling/options.go b/api/scheduling/options.go index 1d5991cc4..bab01a6e6 100644 --- a/api/scheduling/options.go +++ b/api/scheduling/options.go @@ -35,6 +35,10 @@ type Options struct { // committed resource reservation slot. Set for non-VM-placement runs (capacity checks, // failover scheduling, CR slot scheduling) that must not modify reservation allocations. SkipCommittedResourceTracking bool `json:"skip_committed_resource_tracking,omitempty"` + // SkipPlacementContextFilters skips filters that are only meaningful for actual VM + // placement triggered by a user request (e.g. instance group affinity). See the + // filters that check this option for the full list. + SkipPlacementContextFilters bool `json:"skip_placement_context_filters,omitempty"` } // Validate checks for mutually exclusive or inconsistent option combinations. diff --git a/helm/bundles/cortex-nova/values.yaml b/helm/bundles/cortex-nova/values.yaml index d7ef28094..034428fd7 100644 --- a/helm/bundles/cortex-nova/values.yaml +++ b/helm/bundles/cortex-nova/values.yaml @@ -151,9 +151,9 @@ cortex-scheduling-controllers: # that use committed resources. Requires also enabling of CR controllers and tasks committedResourceTracking: false # Pipeline used for the empty-state capacity probe (ignores allocations and reservations). - capacityTotalPipeline: "kvm-report-capacity" + capacityTotalPipeline: "kvm-general-purpose-load-balancing" # Pipeline used for the current-state capacity probe (considers current VM allocations). - capacityPlaceablePipeline: "kvm-general-purpose-load-balancing-no-history" + capacityPlaceablePipeline: "kvm-general-purpose-load-balancing" # How often the capacity controller re-runs its scheduler probes. capacityReconcileInterval: 5m # If true, the external scheduler API will limit the list of hosts in its @@ -163,11 +163,12 @@ cortex-scheduling-controllers: # Set to 0 or negative to disable shuffling. evacuationShuffleK: 3 committedResourceReservationController: - # Maps flavor group IDs to pipeline names; "*" acts as catch-all fallback + # Pipeline selection for CR reservation scheduling. The catch-all default covers + # general-purpose flavors. For HANA flavor groups, add an explicit entry, e.g.: + # "my-hana-group": "kvm-hana-bin-packing" flavorGroupPipelines: - "*": "kvm-general-purpose-load-balancing-no-history" # Catch-all fallback - # Fallback pipeline when no flavorGroupPipelines entry matches - pipelineDefault: "kvm-general-purpose-load-balancing-no-history" + "*": "kvm-general-purpose-load-balancing" + pipelineDefault: "kvm-general-purpose-load-balancing" # How often to re-verify active Reservation CRDs (healthy state) requeueIntervalActive: "5m" # Back-off interval when knowledge is unavailable diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go index 157a80521..4010822ea 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata.go @@ -21,6 +21,9 @@ type FilterAggregateMetadata struct { // the "filter_tenant_id" metadata key set. func (s *FilterAggregateMetadata) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } hvs := &hv1.HypervisorList{} if err := s.Client.List(context.Background(), hvs); err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go index d1ff9cd2d..d293f69c2 100644 --- a/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_aggregate_metadata_test.go @@ -9,6 +9,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -404,3 +405,40 @@ func TestFilterAggregateMetadata_IndexRegistration(t *testing.T) { t.Errorf("expected factory to return *FilterAggregateMetadata, got %T", filter) } } + +func TestFilterAggregateMetadata_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host1 is in an aggregate restricting to project-x; request is project-y → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{ + Aggregates: []hv1.Aggregate{{ + Name: "restricted", + Metadata: map[string]string{"filter_tenant_id": "project-x"}, + }}, + }, + }, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ProjectID: "project-y"}, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + step := &FilterAggregateMetadata{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go index a0a486f3d..8e3eb511b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects.go @@ -21,6 +21,9 @@ type FilterAllowedProjectsStep struct { // Note that hosts without specified projects are still accessible. func (s *FilterAllowedProjectsStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } if request.Spec.Data.ProjectID == "" { traceLog.Info("no project ID in request, skipping filter") return result, nil diff --git a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go index 53a4ac958..1a5ff923f 100644 --- a/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_allowed_projects_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -323,3 +324,35 @@ func TestFilterAllowedProjectsStep_Run(t *testing.T) { }) } } + +func TestFilterAllowedProjectsStep_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 restricts to project-x; request is project-y → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Spec: hv1.HypervisorSpec{AllowedProjects: []string{"project-x"}}, + }, + } + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ProjectID: "project-y"}, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + step := &FilterAllowedProjectsStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer.go b/internal/scheduling/nova/plugins/filters/filter_external_customer.go index bcbf74716..b9f4e7778 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer.go @@ -35,6 +35,9 @@ type FilterExternalCustomerStep struct { // that are not intended for external customers. func (s *FilterExternalCustomerStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } // Skip for failover reservation scheduling — domain restrictions don't apply // since failover reservations are not tied to a specific customer domain. diff --git a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go index 97c9d6925..7730012e1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_external_customer_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -492,3 +493,38 @@ func TestFilterExternalCustomerStepOpts_Validate(t *testing.T) { }) } } + +func TestFilterExternalCustomerStep_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // Domain matches external prefix; host1 lacks the exclusive trait → host1 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}}, + }, + } + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{"domain_name": "iaas-customer"}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + step := &FilterExternalCustomerStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterExternalCustomerStepOpts{CustomerDomainNamePrefixes: []string{"iaas-"}} + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go index 8940c0d86..347fd1390 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go @@ -66,6 +66,10 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa opts := request.GetOptions() result := s.IncludeAllHostsFromRequest(request) + // Merge call-time options with static step config. + ignoreAllocations := s.Options.IgnoreAllocations || opts.AssumeEmptyHosts + ignoredReservationTypes := slices.Concat(s.Options.IgnoredReservationTypes, opts.IgnoredReservationTypes) + // This map holds the free resources per host. freeResourcesByHost := make(map[string]map[hv1.ResourceName]resource.Quantity) @@ -87,7 +91,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } // Subtract allocated resources (skip when ignoring allocations for empty-datacenter capacity queries). - if !s.Options.IgnoreAllocations { + if !ignoreAllocations { for resourceName, allocated := range hv.Status.Allocation { free, ok := freeResourcesByHost[hv.Name][resourceName] if !ok { @@ -110,7 +114,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa } for _, reservation := range reservations.Items { // Check if this reservation type should be ignored — applies regardless of ready state. - if slices.Contains(s.Options.IgnoredReservationTypes, reservation.Spec.Type) { + if slices.Contains(ignoredReservationTypes, reservation.Spec.Type) { traceLog.Debug("ignoring reservation type", "type", reservation.Spec.Type, "reservation", reservation.Name) continue } @@ -209,7 +213,7 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa // Oversize spec-only: if a pending VM is larger than the remaining slot, block its full size. // // FailoverReservations: block = Spec.Resources (always fully blocked). - resourcesToBlock := resv.UnusedReservationCapacity(&reservation, s.Options.IgnoreAllocations) + resourcesToBlock := resv.UnusedReservationCapacity(&reservation, ignoreAllocations) // Block the calculated resources on each host for host := range hostsToBlock { diff --git a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go index 8b55556f8..b1b659e59 100644 --- a/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go @@ -248,7 +248,7 @@ func parseMemoryToMB(memory string) uint64 { return uint64(bytes / (1024 * 1024)) //nolint:gosec // test code } -func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { //nolint:unparam // vcpus varies in real usage +func newNovaRequest(instanceUUID, projectID, flavorName, flavorGroup string, vcpus int, memory string, evacuation bool, hosts []string) api.ExternalSchedulerRequest { return newNovaRequestWithIntent(instanceUUID, projectID, flavorName, flavorGroup, vcpus, memory, "", evacuation, hosts) } @@ -961,6 +961,41 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes(t *testing.T) { } } +func TestFilterHasEnoughCapacity_AssumeEmptyHosts(t *testing.T) { + scheme := buildTestScheme(t) + + // host1: 8 CPU total, 6 CPU allocated to running VMs → 2 free; request needs 4 → fails normally. + // With AssumeEmptyHosts: allocations ignored → 8 free → passes. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "8", "6", "32Gi", "0"), + } + request := newNovaRequest("vm", "proj", "m1.large", "gp", 4, "1Gi", false, []string{"host1"}) + + objects := make([]client.Object, 0, len(hypervisors)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterHasEnoughCapacityOpts{} + + // Without AssumeEmptyHosts: host1 filtered (only 2 CPU free). + resultWithout, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWithout.Activations, []string{}, []string{"host1"}) + + // With AssumeEmptyHosts via call-time options: allocations ignored → host1 passes. + request.Options = scheduling.Options{AssumeEmptyHosts: true} + resultWith, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + assertActivations(t, resultWith.Activations, []string{"host1"}, []string{}) +} + func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) { scheme := buildTestScheme(t) @@ -986,7 +1021,7 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) step := &FilterHasEnoughCapacity{} step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() - // Ignore CR reservations via pipeline-level opts (call-time opts.IgnoredReservationTypes removed in favour of YAML params). + // Ignore CR reservations via YAML step opts (static, configured per pipeline step). step.Options = FilterHasEnoughCapacityOpts{ LockReserved: true, IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, @@ -999,6 +1034,49 @@ func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTime(t *testing.T) assertActivations(t, result.Activations, []string{"host1"}, []string{"host2"}) } +func TestFilterHasEnoughCapacity_IgnoredReservationTypes_CallTimeMerge(t *testing.T) { + scheme := buildTestScheme(t) + + // Same two-host setup: CR on host1, Failover on host2. + // Each blocks 4 CPU, leaving 4 free; request needs 8 CPU so both hosts fail without ignoring. + // Verify that call-time opts.IgnoredReservationTypes is merged with (not instead of) YAML opts. + hypervisors := []*hv1.Hypervisor{ + newHypervisor("host1", "16", "8", "32Gi", "16Gi"), + newHypervisor("host2", "16", "8", "32Gi", "16Gi"), + } + reservations := []*v1alpha1.Reservation{ + newCommittedReservation("cr-res", "host1", "project-X", "m1.large", "gp-1", "4", "8Gi", nil, nil), + newFailoverReservation("failover-res", "host2", "4", "8Gi", map[string]string{"other-vm": "host3"}), + } + request := newNovaRequest("instance-123", "project-A", "m1.large", "gp-1", 8, "16Gi", false, []string{"host1", "host2"}) + + objects := make([]client.Object, 0, len(hypervisors)+len(reservations)) + for _, h := range hypervisors { + objects = append(objects, h.DeepCopy()) + } + for _, r := range reservations { + objects = append(objects, r.DeepCopy()) + } + + step := &FilterHasEnoughCapacity{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + // YAML opts ignore CR; call-time opts ignore Failover — both must be respected. + step.Options = FilterHasEnoughCapacityOpts{ + LockReserved: true, + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeCommittedResource}, + } + request.Options = scheduling.Options{ + IgnoredReservationTypes: []v1alpha1.ReservationType{v1alpha1.ReservationTypeFailover}, + } + + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + // Both CR (host1) and Failover (host2) reservations ignored → both hosts pass. + assertActivations(t, result.Activations, []string{"host1", "host2"}, []string{}) +} + func TestFilterHasEnoughCapacity_ReserveForCommittedResourceIntent(t *testing.T) { scheme := buildTestScheme(t) diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go index 326864b9d..e96e2a1a2 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity.go @@ -22,6 +22,9 @@ func (s *FilterInstanceGroupAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go index 7321747e3..a9f2cf9bb 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_affinity_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" ) func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { @@ -352,3 +353,21 @@ func TestFilterInstanceGroupAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAffinityStep_SkipPlacementContextFilters(t *testing.T) { + // Affinity group on host1 only — host2 would normally be filtered. + request := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, + } + step := &FilterInstanceGroupAffinityStep{} + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go index 0dee29d9e..418ee7f3b 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity.go @@ -25,6 +25,9 @@ func (s *FilterInstanceGroupAntiAffinityStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } ig := request.Spec.Data.InstanceGroup if ig == nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go index 931265b9b..d81aa590e 100644 --- a/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_instance_group_anti_affinity_test.go @@ -8,6 +8,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -547,3 +548,37 @@ func TestFilterInstanceGroupAntiAffinityStep_Run(t *testing.T) { }) } } + +func TestFilterInstanceGroupAntiAffinityStep_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 has the group member vm; max_server_per_host=1 → host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: v1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: v1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Instances: []hv1.Instance{{ID: "vm-existing"}}}, + }, + } + request := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + request.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + Rules: map[string]any{"max_server_per_host": 1}, + }, + } + step := &FilterInstanceGroupAntiAffinityStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go index a19238721..10e1a5e15 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable.go @@ -54,6 +54,9 @@ func (s *FilterLiveMigratableStep) Run( ) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } intent, err := request.GetIntent() if err != nil { diff --git a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go index c5651b025..3762f3af1 100644 --- a/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_live_migratable_test.go @@ -9,6 +9,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -779,3 +780,47 @@ func TestFilterLiveMigratableStep_Run_ClientError(t *testing.T) { t.Errorf("expected error when client fails, got none") } } + +func TestFilterLiveMigratableStep_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // host2 has a different CPU arch → incompatible for live migration → would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "source-host"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}}, + }, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}}, + }, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "aarch64"}}, + }, + } + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{ + "_nova_check_type": "live_migrate", + "source_host": "source-host", + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + step := &FilterLiveMigratableStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go index 4e139f8d8..9becb84ec 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement.go @@ -74,6 +74,9 @@ type FilterQuotaEnforcement struct { func (s *FilterQuotaEnforcement) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } mode := "shadow" if s.Options.Enforce { diff --git a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go index 32ea16ad4..934b9c515 100644 --- a/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_quota_enforcement_test.go @@ -11,6 +11,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/api/v1alpha1" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" @@ -1087,3 +1088,37 @@ func TestQuotaEnforcementMetrics_RecordDecision_NilWarns(t *testing.T) { msg, got, buf.String()) } } + +func TestFilterQuotaEnforcement_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add v1alpha1 to scheme: %v", err) + } + // Enforce mode: quota fully consumed → all hosts would normally be filtered. + objects := []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-no-quota-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-no-quota", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + }, + } + request := makeQuotaEnforcementRequest("project-no-quota", "az-1", "gp", 1, 1, nil) + step := &FilterQuotaEnforcement{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + step.Options = FilterQuotaEnforcementOpts{Enforce: true} + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != len(request.Hosts) { + t.Errorf("expected all %d hosts to pass, got %d", len(request.Hosts), len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go index 8922ab8c4..b2e9774f7 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination.go @@ -102,6 +102,9 @@ func (s *FilterRequestedDestinationStep) processRequestedHost( // host filtering. func (s *FilterRequestedDestinationStep) Run(traceLog *slog.Logger, request api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) { result := s.IncludeAllHostsFromRequest(request) + if request.GetOptions().SkipPlacementContextFilters { + return result, nil + } rd := request.Spec.Data.RequestedDestination if rd == nil { traceLog.Info("no requested_destination in request, skipping filter") diff --git a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go index 5a752160e..008dae061 100644 --- a/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go +++ b/internal/scheduling/nova/plugins/filters/filter_requested_destination_test.go @@ -9,6 +9,7 @@ import ( "testing" api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -782,3 +783,36 @@ func TestFilterRequestedDestinationStep_Run_ClientError(t *testing.T) { t.Errorf("expected error when client fails, got none") } } + +func TestFilterRequestedDestinationStep_SkipPlacementContextFilters(t *testing.T) { + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + // RequestedDestination forces host1 — host2 would normally be filtered. + objects := []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + } + request := api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ + Data: api.NovaRequestedDestination{Host: "host1"}, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + } + step := &FilterRequestedDestinationStep{} + step.Client = fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() + + request.Options = scheduling.Options{SkipPlacementContextFilters: true} + result, err := step.Run(slog.Default(), request) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result.Activations) != 2 { + t.Errorf("expected both hosts to pass, got %d", len(result.Activations)) + } +} diff --git a/internal/scheduling/nova/plugins/filters/filter_skip_placement_context_test.go b/internal/scheduling/nova/plugins/filters/filter_skip_placement_context_test.go new file mode 100644 index 000000000..e383c6c76 --- /dev/null +++ b/internal/scheduling/nova/plugins/filters/filter_skip_placement_context_test.go @@ -0,0 +1,299 @@ +// Copyright SAP SE +// SPDX-License-Identifier: Apache-2.0 + +package filters + +// Contract test for SkipPlacementContextFilters: every filter that claims to respect this +// option must pass all hosts when it is set, even when the request would normally filter some. +// Add a row to skipCases whenever a new filter is wired to SkipPlacementContextFilters. + +import ( + "log/slog" + "testing" + + api "github.com/cobaltcore-dev/cortex/api/external/nova" + "github.com/cobaltcore-dev/cortex/api/scheduling" + "github.com/cobaltcore-dev/cortex/api/v1alpha1" + "github.com/cobaltcore-dev/cortex/internal/scheduling/lib" + hv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type skipCase struct { + name string + request api.ExternalSchedulerRequest + objects []client.Object + newStep func(client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } +} + +var skipCases = []skipCase{ + { + name: "filter_instance_group_affinity", + // Affinity group on host1 only — host2 would be filtered without the flag. + request: func() api.ExternalSchedulerRequest { + r := newNovaRequest("vm", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + r.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{Policy: "affinity", Hosts: []string{"host1"}}, + } + return r + }(), + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterInstanceGroupAffinityStep{} + s.Client = c + return s + }, + }, + { + name: "filter_instance_group_anti_affinity", + // host2 already has the group member vm; max_server_per_host=1 → host2 filtered. + request: func() api.ExternalSchedulerRequest { + r := newNovaRequest("vm-new", "proj", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + r.Spec.Data.InstanceGroup = &api.NovaObject[api.NovaInstanceGroup]{ + Data: api.NovaInstanceGroup{ + Policy: "anti-affinity", + Members: []string{"vm-existing"}, + Rules: map[string]any{"max_server_per_host": 1}, + }, + } + return r + }(), + objects: []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Instances: []hv1.Instance{{ID: "vm-existing"}}}, + }, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterInstanceGroupAntiAffinityStep{} + s.Client = c + return s + }, + }, + { + name: "filter_aggregate_metadata", + // host1 is in an aggregate restricting to project-x; request is project-y → host1 filtered. + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ProjectID: "project-y"}, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + }, + objects: []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{ + Aggregates: []hv1.Aggregate{{ + Name: "restricted", + Metadata: map[string]string{"filter_tenant_id": "project-x"}, + }}, + }, + }, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterAggregateMetadata{} + s.Client = c + return s + }, + }, + { + name: "filter_live_migratable", + // host2 has a different CPU arch than the source → incompatible for live migration → filtered. + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{ + "_nova_check_type": "live_migrate", + "source_host": "source-host", + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + }, + objects: []client.Object{ + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "source-host"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}}, + }, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host1"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "x86_64"}}, + }, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Capabilities: hv1.Capabilities{HostCpuArch: "aarch64"}}, + }, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterLiveMigratableStep{} + s.Client = c + return s + }, + }, + { + name: "filter_requested_destination", + // RequestedDestination forces host1 — host2 would be filtered. + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + RequestedDestination: &api.NovaObject[api.NovaRequestedDestination]{ + Data: api.NovaRequestedDestination{Host: "host1"}, + }, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + }, + objects: []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host2"}}, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterRequestedDestinationStep{} + s.Client = c + return s + }, + }, + { + name: "filter_quota_enforcement", + // Enforce mode: quota fully consumed (PaygUsage == Quota) → all hosts filtered. + request: func() api.ExternalSchedulerRequest { + r := newNovaRequest("vm", "project-no-quota", "m1.small", "gp", 1, "1Gi", false, []string{"host1", "host2"}) + r.Spec.Data.AvailabilityZone = "az-1" + return r + }(), + objects: []client.Object{ + &v1alpha1.ProjectQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "quota-project-no-quota-az-1"}, + Spec: v1alpha1.ProjectQuotaSpec{ + ProjectID: "project-no-quota", + AvailabilityZone: "az-1", + Quota: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + Status: v1alpha1.ProjectQuotaStatus{ + PaygUsage: map[string]int64{"hw_version_gp_ram": 10, "hw_version_gp_cores": 10, "hw_version_gp_instances": 1}, + }, + }, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterQuotaEnforcement{} + s.Client = c + s.Options = FilterQuotaEnforcementOpts{Enforce: true} + return s + }, + }, + { + name: "filter_allowed_projects", + // host2 restricts to project-x; request is project-y → host2 filtered. + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ProjectID: "project-y"}, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + }, + objects: []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host2"}, + Spec: hv1.HypervisorSpec{AllowedProjects: []string{"project-x"}}, + }, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterAllowedProjectsStep{} + s.Client = c + return s + }, + }, + { + name: "filter_external_customer", + // Domain matches external prefix; host1 lacks the exclusive trait → host1 filtered. + request: api.ExternalSchedulerRequest{ + Spec: api.NovaObject[api.NovaSpec]{ + Data: api.NovaSpec{ + SchedulerHints: map[string]any{"domain_name": "iaas-customer"}, + }, + }, + Hosts: []api.ExternalSchedulerHost{{ComputeHost: "host1"}, {ComputeHost: "host2"}}, + }, + objects: []client.Object{ + &hv1.Hypervisor{ObjectMeta: metav1.ObjectMeta{Name: "host1"}}, + &hv1.Hypervisor{ + ObjectMeta: metav1.ObjectMeta{Name: "host2"}, + Status: hv1.HypervisorStatus{Traits: []string{"CUSTOM_EXTERNAL_CUSTOMER_EXCLUSIVE"}}, + }, + }, + newStep: func(c client.Client) interface { + Run(*slog.Logger, api.ExternalSchedulerRequest) (*lib.FilterWeigherPipelineStepResult, error) + } { + s := &FilterExternalCustomerStep{} + s.Client = c + s.Options = FilterExternalCustomerStepOpts{CustomerDomainNamePrefixes: []string{"iaas-"}} + return s + }, + }, +} + +func TestSkipPlacementContextFilters(t *testing.T) { + scheme := buildSkipTestScheme(t) + + for _, tc := range skipCases { + t.Run(tc.name, func(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tc.objects...).Build() + step := tc.newStep(c) + + // Without the flag: verify the scenario actually filters something (proves setup is valid). + withoutFlag := tc.request + withoutFlag.Options = scheduling.Options{} + resultWithout, err := step.Run(slog.Default(), withoutFlag) + if err != nil { + t.Fatalf("Run without flag: unexpected error: %v", err) + } + if len(resultWithout.Activations) == len(tc.request.Hosts) { + t.Fatalf("setup invalid: expected at least one host to be filtered without the flag, but all %d passed", len(tc.request.Hosts)) + } + + // With the flag: all hosts must pass. + withFlag := tc.request + withFlag.Options = scheduling.Options{SkipPlacementContextFilters: true} + resultWith, err := step.Run(slog.Default(), withFlag) + if err != nil { + t.Fatalf("Run with SkipPlacementContextFilters=true: unexpected error: %v", err) + } + if len(resultWith.Activations) != len(tc.request.Hosts) { + t.Errorf("expected all %d hosts to pass with SkipPlacementContextFilters=true, got %d", + len(tc.request.Hosts), len(resultWith.Activations)) + } + }) + } +} + +func buildSkipTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + if err := hv1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add hv1 to scheme: %v", err) + } + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add v1alpha1 to scheme: %v", err) + } + return scheme +} diff --git a/internal/scheduling/reservations/capacity/config.go b/internal/scheduling/reservations/capacity/config.go index 264a0b59d..d4834c53e 100644 --- a/internal/scheduling/reservations/capacity/config.go +++ b/internal/scheduling/reservations/capacity/config.go @@ -46,8 +46,8 @@ func (c *Config) ApplyDefaults() { func DefaultConfig() Config { return Config{ ReconcileInterval: metav1.Duration{Duration: 5 * time.Minute}, - TotalPipeline: "kvm-report-capacity", - PlaceablePipeline: "kvm-general-purpose-load-balancing-no-history", + TotalPipeline: "kvm-general-purpose-load-balancing", + PlaceablePipeline: "kvm-general-purpose-load-balancing", SchedulerURL: "http://localhost:8080/scheduler/nova/external", } } diff --git a/internal/scheduling/reservations/capacity/controller.go b/internal/scheduling/reservations/capacity/controller.go index d49cada59..a28cefe2d 100644 --- a/internal/scheduling/reservations/capacity/controller.go +++ b/internal/scheduling/reservations/capacity/controller.go @@ -291,9 +291,21 @@ func (c *Controller) probeScheduler( eligibleHosts = append(eligibleHosts, schedulerapi.ExternalSchedulerHost{ComputeHost: name}) } + // Total probe ignores all reservation blocks (raw hardware capacity). + // Placeable probe counts reservations as capacity blocks. + var ignoredReservationTypes []v1alpha1.ReservationType + if ignoreAllocations { + ignoredReservationTypes = []v1alpha1.ReservationType{ + v1alpha1.ReservationTypeCommittedResource, + v1alpha1.ReservationTypeFailover, + } + } + resp, err := c.schedulerClient.ScheduleReservation(ctx, reservations.ScheduleReservationRequest{ - InstanceUUID: "capacity-" + flavor.Name, - ProjectID: "cortex-capacity-probe", + InstanceUUID: "capacity-" + flavor.Name, + // Empty project ID so filter_allowed_projects passes all hosts — the capacity probe + // must see the full host set regardless of per-project restrictions. + ProjectID: "", FlavorName: flavor.Name, MemoryMB: flavor.MemoryMB, VCPUs: flavor.VCPUs, @@ -301,7 +313,15 @@ func (c *Controller) probeScheduler( AvailabilityZone: az, Pipeline: pipeline, EligibleHosts: eligibleHosts, - }, scheduling.Options{SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + }, scheduling.Options{ + ReadOnly: true, + AssumeEmptyHosts: ignoreAllocations, + IgnoredReservationTypes: ignoredReservationTypes, + SkipPlacementContextFilters: true, + SkipHistory: true, + SkipInflight: true, + SkipCommittedResourceTracking: true, + }) if err != nil { return 0, 0, fmt.Errorf("scheduler call failed (pipeline=%s): %w", pipeline, err) } diff --git a/internal/scheduling/reservations/capacity/controller_test.go b/internal/scheduling/reservations/capacity/controller_test.go index 8e25ff644..8dd0b67bd 100644 --- a/internal/scheduling/reservations/capacity/controller_test.go +++ b/internal/scheduling/reservations/capacity/controller_test.go @@ -692,3 +692,32 @@ func TestProbeScheduler_SubtractsReservationBlocksWhenNotIgnored(t *testing.T) { t.Errorf("placeable capacity = %d, want 1 (3 slots − 1 alloc − 1 reservation)", placeableCap) } } + +func TestProbeScheduler_SetsSkipPlacementContextFilters(t *testing.T) { + scheme := newTestScheme(t) + hv := newHypervisor("host-1", "az-a", 4096*1024*1024) + + var capturedReq schedulerapi.ExternalSchedulerRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + json.NewEncoder(w).Encode(schedulerapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}}) //nolint:errcheck + })) + defer srv.Close() + + c := NewController(fake.NewClientBuilder().WithScheme(scheme).Build(), Config{SchedulerURL: srv.URL}) + hvByName := map[string]hv1.Hypervisor{"host-1": *hv} + flavor := compute.FlavorInGroup{Name: "test-flavor", MemoryMB: 4096} + + if _, _, err := c.probeScheduler(context.Background(), flavor, "az-a", "test-pipeline", hvByName, true, nil); err != nil { + t.Fatalf("probeScheduler failed: %v", err) + } + if !capturedReq.Options.SkipPlacementContextFilters { + t.Error("capacity probe must set SkipPlacementContextFilters=true to see all hosts regardless of project restrictions") + } + if capturedReq.Spec.Data.ProjectID != "" { + t.Errorf("capacity probe must send empty ProjectID, got %q", capturedReq.Spec.Data.ProjectID) + } +} diff --git a/internal/scheduling/reservations/commitments/integration_test.go b/internal/scheduling/reservations/commitments/integration_test.go index 8c51a8162..d48560d52 100644 --- a/internal/scheduling/reservations/commitments/integration_test.go +++ b/internal/scheduling/reservations/commitments/integration_test.go @@ -1101,3 +1101,37 @@ func TestCRLifecycle(t *testing.T) { } }) } + +func TestCRScheduling_DoesNotSetSkipPlacementContextFilters(t *testing.T) { + // CR slot scheduling has a real project ID and must run tenant-context filters + // (filter_allowed_projects, filter_aggregate_metadata, etc.). Verify SkipPlacementContextFilters + // is never set so those filters are not bypassed. + var capturedReq schedulerdelegationapi.ExternalSchedulerRequest + var schedulerCalled bool + captureScheduler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + schedulerCalled = true + if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp := &schedulerdelegationapi.ExternalSchedulerResponse{Hosts: []string{"host-1"}} + json.NewEncoder(w).Encode(resp) //nolint:errcheck + }) + + env := newIntgEnv(t, []client.Object{newTestFlavorKnowledge(), intgHypervisor("host-1")}, captureScheduler) + defer env.close() + + cr := intgCR("test-cr", "commit-uuid-1", v1alpha1.CommitmentStatusConfirmed) + if err := env.k8sClient.Create(context.Background(), cr); err != nil { + t.Fatalf("create CR: %v", err) + } + env.reconcileCR(t, cr.Name) + env.reconcileChildReservations(t, cr.Name) + + if !schedulerCalled { + t.Fatal("scheduler was never called — test did not exercise the scheduling path") + } + if capturedReq.Options.SkipPlacementContextFilters { + t.Error("CR slot scheduling must not set SkipPlacementContextFilters — tenant-context filters must run") + } +} diff --git a/internal/scheduling/reservations/commitments/reservation_controller.go b/internal/scheduling/reservations/commitments/reservation_controller.go index db24c0ed5..30d9e0c2a 100644 --- a/internal/scheduling/reservations/commitments/reservation_controller.go +++ b/internal/scheduling/reservations/commitments/reservation_controller.go @@ -296,6 +296,9 @@ func (r *CommitmentReservationController) Reconcile(ctx context.Context, req ctr SkipHistory: true, SkipInflight: false, // TODO pessimistic blocking needed, will be addressed in follow up ticket SkipCommittedResourceTracking: true, // CR slot scheduling, not a VM placement + // CR slot scheduling has a real project ID and must respect per-project host + // restrictions (allowed projects, aggregate metadata, external customer, etc.). + SkipPlacementContextFilters: false, } scheduleResp, err := r.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduleOpts) diff --git a/internal/scheduling/reservations/failover/integration_test.go b/internal/scheduling/reservations/failover/integration_test.go index a145ec676..b7ad34d5f 100644 --- a/internal/scheduling/reservations/failover/integration_test.go +++ b/internal/scheduling/reservations/failover/integration_test.go @@ -435,6 +435,22 @@ func TestIntegration(t *testing.T) { ExpectedMinRes: 1, // Both HANA VMs can share reservation on host3 UseTraitsFilter: true, }, + { + Name: "HANA VM uses kvm-hana-bin-packing pipeline (trait:CUSTOM_HANA_EXCLUSIVE_HOST=required)", + Hypervisors: []*hv1.Hypervisor{ + newHypervisor("host1", 16, 32, 4, 8, []hv1.Instance{{ID: "vm-hana-exclusive-1", Name: "vm-hana-exclusive-1", Active: true}}, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + newHypervisor("host2", 16, 32, 0, 0, nil, []string{"CUSTOM_HANA_EXCLUSIVE_HOST"}), + }, + VMs: []reservations.VM{ + newVMWithExtraSpecs("vm-hana-exclusive-1", "m1.hana", "project-A", "host1", 8192, 4, + map[string]string{"trait:CUSTOM_HANA_EXCLUSIVE_HOST": "required"}), + }, + FlavorRequirements: map[string]int{"m1.hana": 1}, + ExpectedMinRes: 1, + ExpectedMaxRes: 1, + VerifyVMReservation: []string{"vm-hana-exclusive-1"}, + UseTraitsFilter: true, + }, } for _, tc := range testCases { @@ -1153,7 +1169,7 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1181,6 +1197,17 @@ func newIntegrationTestEnv(t *testing.T, vms []reservations.VM, hypervisors []*h }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { @@ -1343,7 +1370,7 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, { ObjectMeta: metav1.ObjectMeta{ - Name: PipelineReuseFailoverReservation, + Name: PipelineNewFailoverReservation, }, Spec: v1alpha1.PipelineSpec{ Type: v1alpha1.PipelineTypeFilterWeigher, @@ -1371,6 +1398,18 @@ func newIntegrationTestEnvWithTraitsFilter(t *testing.T, vms []reservations.VM, }, }, } + pipelines = append(pipelines, v1alpha1.Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "kvm-hana-bin-packing"}, + Spec: v1alpha1.PipelineSpec{ + Type: v1alpha1.PipelineTypeFilterWeigher, + Filters: []v1alpha1.FilterSpec{ + {Name: "filter_has_enough_capacity"}, + {Name: "filter_has_requested_traits"}, + {Name: "filter_correct_az"}, + }, + Weighers: []v1alpha1.WeigherSpec{{Name: "kvm_failover_evacuation"}}, + }, + }) ctx := context.Background() for _, pipeline := range pipelines { diff --git a/internal/scheduling/reservations/failover/reservation_scheduling.go b/internal/scheduling/reservations/failover/reservation_scheduling.go index 5f5903de9..3150162de 100644 --- a/internal/scheduling/reservations/failover/reservation_scheduling.go +++ b/internal/scheduling/reservations/failover/reservation_scheduling.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "sort" + "strings" api "github.com/cobaltcore-dev/cortex/api/external/nova" "github.com/cobaltcore-dev/cortex/api/scheduling" @@ -17,21 +18,20 @@ import ( // Pipeline names for failover reservation scheduling const ( - // PipelineReuseFailoverReservation is used to check if a VM can reuse an existing reservation. - // It validates host compatibility without checking capacity (since reservation already has capacity). - PipelineReuseFailoverReservation = "kvm-valid-host-reuse-failover-reservation" - // PipelineNewFailoverReservation is used to find a host for creating a new reservation. - // It validates host compatibility AND checks capacity. // Uses the general-purpose pipeline; LockReservations and SkipHistory are set via Options. PipelineNewFailoverReservation = "kvm-general-purpose-load-balancing" - - // PipelineAcknowledgeFailoverReservation is used to validate that a failover reservation - // is still valid for all its allocated VMs. It sends an evacuation-style scheduling request - // for each VM with only the reservation's host as the eligible target. - PipelineAcknowledgeFailoverReservation = "kvm-acknowledge-failover-reservation" ) +// inferFailoverPipeline returns the standard pipeline for a failover scheduling call based on +// the VM's flavor extra specs — the same HANA vs general-purpose split used by Nova placement. +func inferFailoverPipeline(extraSpecs map[string]string) string { + if strings.ToLower(extraSpecs["trait:CUSTOM_HANA_EXCLUSIVE_HOST"]) == "required" { + return "kvm-hana-bin-packing" + } + return "kvm-general-purpose-load-balancing" +} + func (c *FailoverReservationController) queryHypervisorsFromScheduler(ctx context.Context, vm reservations.VM, allHypervisors []string, pipeline string, resSpec resolvedReservationSpec, opts scheduling.Options) ([]string, error) { logger := LoggerFromContext(ctx) @@ -123,7 +123,7 @@ func (c *FailoverReservationController) tryReuseExistingReservation( logger := LoggerFromContext(ctx) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineReuseFailoverReservation, resSpec, scheduling.Options{ReadOnly: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, inferFailoverPipeline(vm.FlavorExtraSpecs), resSpec, scheduling.Options{ReadOnly: true, SkipPlacementContextFilters: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) if err != nil { logger.Error(err, "failed to get potential hypervisors for VM", "vmUUID", vm.UUID) return nil @@ -213,7 +213,7 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( VCPUs: vcpus, EligibleHosts: []api.ExternalSchedulerHost{{ComputeHost: reservationHost}}, IgnoreHosts: []string{vm.CurrentHypervisor}, - Pipeline: PipelineAcknowledgeFailoverReservation, + Pipeline: inferFailoverPipeline(flavorExtraSpecs), AvailabilityZone: vm.AvailabilityZone, SchedulerHints: map[string]any{"_nova_check_type": string(api.EvacuateIntent)}, } @@ -222,9 +222,9 @@ func (c *FailoverReservationController) validateVMViaSchedulerEvacuation( "vmUUID", vm.UUID, "reservationHost", reservationHost, "vmCurrentHost", vm.CurrentHypervisor, - "pipeline", PipelineAcknowledgeFailoverReservation) + "pipeline", scheduleReq.Pipeline) - resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduling.Options{ReadOnly: true, LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + resp, err := c.SchedulerClient.ScheduleReservation(ctx, scheduleReq, scheduling.Options{ReadOnly: true, LockReservations: true, SkipPlacementContextFilters: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) if err != nil { logger.Error(err, "failed to validate VM for reservation host", "vmUUID", vm.UUID, "reservationHost", reservationHost) return false, fmt.Errorf("failed to validate VM for reservation host: %w", err) @@ -266,7 +266,7 @@ func (c *FailoverReservationController) scheduleAndBuildNewFailoverReservation( // Get potential hypervisors from scheduler using the reservation spec resources // (which may be sized to the LargestFlavor from the flavor group) - validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, scheduling.Options{LockReservations: true, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) + validHypervisors, err := c.queryHypervisorsFromScheduler(ctx, vm, allHypervisors, PipelineNewFailoverReservation, resSpec, scheduling.Options{LockReservations: true, SkipPlacementContextFilters: false, SkipHistory: true, SkipInflight: true, SkipCommittedResourceTracking: true}) if err != nil { return nil, fmt.Errorf("failed to get potential hypervisors for VM: %w", err) }