Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions api/external/cinder/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package api
import (
"log/slog"

"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
)

Expand All @@ -30,8 +31,11 @@ type ExternalSchedulerRequest struct {
Weights map[string]float64 `json:"weights"`
// The name of the pipeline to execute.
Pipeline string `json:"pipeline"`
// Options configure the pipeline behavior for this scheduling call.
Options scheduling.Options `json:"options,omitempty"`
}

func (r ExternalSchedulerRequest) GetOptions() scheduling.Options { return r.Options }
func (r ExternalSchedulerRequest) GetHosts() []string {
hosts := make([]string, len(r.Hosts))
for i, host := range r.Hosts {
Expand Down
4 changes: 4 additions & 0 deletions api/external/ironcore/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@ import (
"log/slog"

ironcorev1alpha1 "github.com/cobaltcore-dev/cortex/api/external/ironcore/v1alpha1"
"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
)

type MachinePipelineRequest struct {
// The available machine pools.
Pools []ironcorev1alpha1.MachinePool `json:"pools"`
// Options configure the pipeline behavior for this scheduling call.
Options scheduling.Options `json:"options,omitempty"`
}

func (r MachinePipelineRequest) GetOptions() scheduling.Options { return r.Options }
func (r MachinePipelineRequest) GetHosts() []string {
hosts := make([]string, len(r.Pools))
for i, host := range r.Pools {
Expand Down
4 changes: 4 additions & 0 deletions api/external/manila/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package api
import (
"log/slog"

"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
)

Expand All @@ -30,8 +31,11 @@ type ExternalSchedulerRequest struct {
Weights map[string]float64 `json:"weights"`
// The name of the pipeline to execute.
Pipeline string `json:"pipeline"`
// Options configure the pipeline behavior for this scheduling call.
Options scheduling.Options `json:"options,omitempty"`
}

func (r ExternalSchedulerRequest) GetOptions() scheduling.Options { return r.Options }
func (r ExternalSchedulerRequest) GetHosts() []string {
hosts := make([]string, len(r.Hosts))
for i, host := range r.Hosts {
Expand Down
7 changes: 7 additions & 0 deletions api/external/nova/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"log/slog"
"strings"

"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/api/v1alpha1"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
)
Expand Down Expand Up @@ -37,8 +38,14 @@ type ExternalSchedulerRequest struct {

// The name of the pipeline to execute.
Pipeline string `json:"pipeline"`

// Options configure the pipeline behavior for this scheduling call.
// Set by the caller (CR controller, failover controller, Nova).
// Nova does not set these; Cortex fills in config-derived defaults server-side.
Options scheduling.Options `json:"options,omitempty"`
}

func (r ExternalSchedulerRequest) GetOptions() scheduling.Options { return r.Options }
func (r ExternalSchedulerRequest) GetHosts() []string {
hosts := make([]string, len(r.Hosts))
for i, host := range r.Hosts {
Expand Down
4 changes: 4 additions & 0 deletions api/external/pods/messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package pods
import (
"log/slog"

"github.com/cobaltcore-dev/cortex/api/scheduling"
"github.com/cobaltcore-dev/cortex/internal/scheduling/lib"
corev1 "k8s.io/api/core/v1"
)
Expand All @@ -15,8 +16,11 @@ type PodPipelineRequest struct {
Nodes []corev1.Node `json:"nodes"`
// The pod to be scheduled.
Pod corev1.Pod `json:"pod"`
// Options configure the pipeline behavior for this scheduling call.
Options scheduling.Options `json:"options,omitempty"`
}

func (r PodPipelineRequest) GetOptions() scheduling.Options { return r.Options }
func (r PodPipelineRequest) GetHosts() []string {
hosts := make([]string, len(r.Nodes))
for i, host := range r.Nodes {
Expand Down
43 changes: 43 additions & 0 deletions api/scheduling/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright SAP SE
// SPDX-License-Identifier: Apache-2.0

package scheduling

import (
"errors"

"github.com/cobaltcore-dev/cortex/api/v1alpha1"
)

// Options configure the behavior of a single pipeline run at call time.
// These are distinct from per-step YAML options (FilterWeigherPipelineStepOpts),
// which are static and set when the pipeline is initialized.
type Options struct {
// ReadOnly means the pipeline run does not modify shared scheduling state (reservations,
// history, inflight records). Concurrent read-only runs are safe under a shared read lock.
ReadOnly bool `json:"read_only,omitempty"`
// LockReservations prevents reservation unlocking, i.e. considering those as unavailable resources.
LockReservations bool `json:"lock_reservations,omitempty"`
// AssumeEmptyHosts ignores running instances on hosts, considering them as empty.
AssumeEmptyHosts bool `json:"assume_empty_hosts,omitempty"`
// IgnoredReservationTypes lists reservation types whose reserved capacity the capacity filter does not block.
IgnoredReservationTypes []v1alpha1.ReservationType `json:"ignored_reservation_types,omitempty"`
// MaxCandidates limits the number of hosts returned after weighing. 0 means no limit.
MaxCandidates int `json:"max_candidates,omitempty"`

// SkipHistory skips recording the placement decision in placement history.
SkipHistory bool `json:"skip_history,omitempty"`
// SkipInflight skips creating pessimistic blocking reservations for returned candidates.
SkipInflight bool `json:"skip_inflight,omitempty"`
}

// Validate checks for mutually exclusive or inconsistent option combinations.
func (o Options) Validate() error {
if o.ReadOnly && !o.SkipHistory {
return errors.New("read-only runs must not write scheduling history: set SkipHistory=true")
}
if o.ReadOnly && !o.SkipInflight {
return errors.New("read-only runs cannot create inflight reservations")
}
return nil
}
31 changes: 31 additions & 0 deletions api/scheduling/options_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright SAP SE
// SPDX-License-Identifier: Apache-2.0

package scheduling

import "testing"

func TestOptions_Validate(t *testing.T) {
tests := []struct {
name string
opts Options
wantErr bool
}{
{"zero value is valid", Options{}, false},
{"read-only run, skipping history and inflight", Options{ReadOnly: true, SkipHistory: true, SkipInflight: true}, false},
{"ReadOnly without SkipHistory is invalid", Options{ReadOnly: true}, true},
{"ReadOnly without SkipInflight is invalid", Options{ReadOnly: true, SkipHistory: true}, true},
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.opts.Validate()
if tt.wantErr && err == nil {
t.Error("expected error, got nil")
}
if !tt.wantErr && err != nil {
t.Errorf("expected no error, got %v", err)
}
})
}
}
5 changes: 0 additions & 5 deletions api/v1alpha1/pipeline_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,6 @@ type PipelineSpec struct {
// +kubebuilder:validation:Optional
Description string `json:"description,omitempty"`

// If this pipeline should create history objects.
// When this is false, the pipeline will still process requests.
// +kubebuilder:default=false
CreateHistory bool `json:"createHistory,omitempty"`

// If this pipeline should ignore host preselection and gather all
// available placement candidates before applying filters, instead of
// relying on a pre-filtered set and weights.
Expand Down
2 changes: 2 additions & 0 deletions docs/apis.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ Pipelines bundle scheduling steps together. Filters are mandatory, while weigher

The state of the pipeline is propagated automatically through the states of its steps. Check the pipeline state object to determine if the pipeline can currently be executed or not.

Pipeline behavior has two configuration layers: static per-step params defined in the Pipeline CRD YAML (thresholds, weights, traits), and call-time `Options` set by the controller invoking the pipeline (e.g. whether to record history, lock reservations, or skip VM allocation accounting).

### Decisions

```bash
Expand Down
8 changes: 6 additions & 2 deletions docs/reservations/failover-reservations.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,20 +144,24 @@ We use three different scheduler pipelines for failover reservations, each servi

**Why:** When reusing a reservation, capacity is already reserved on the target host. We only need to verify that the VM is compatible with the host (traits, capabilities, AZ, etc.) without checking if there's enough free capacity.

### `kvm-new-failover-reservation`
Options: `ReadOnly: true, SkipHistory: true` — pure compatibility check, no state mutations.

### `kvm-general-purpose-load-balancing` (new reservation)
**Used when:** Creating a new failover reservation.

**Why:** When creating a new reservation, we need to find a host that:
1. Is compatible with the VM (traits, capabilities, AZ, etc.)
2. Has enough free capacity to accommodate the VM if it needs to evacuate

This is the most restrictive pipeline since we're actually reserving new capacity.
Options: `LockReservations: true, SkipHistory: true` — capacity check must see true remaining capacity with all reservation slots locked.

### `kvm-acknowledge-failover-reservation`
**Used when:** Validating that an existing reservation is still valid (watch-based reconciliation).

**Why:** Periodically we need to verify that a VM could still evacuate to its reserved host. This sends an evacuation-style scheduling request with only the reservation's host as the eligible target. If the scheduler rejects it, the reservation is no longer valid and should be deleted so the periodic controller can create a new one on a valid host.

Options: `ReadOnly: true, SkipHistory: true` — validation only, no state mutations.

## Data Model

### VM Struct
Expand Down
1 change: 0 additions & 1 deletion helm/bundles/cortex-ironcore/templates/pipelines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ spec:
description: |
This pipeline is used to schedule ironcore machines onto machinepools.
type: filter-weigher
createHistory: false
filters: []
weighers:
- name: noop
Expand Down
2 changes: 0 additions & 2 deletions helm/bundles/cortex-nova/templates/pipelines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ spec:

Specifically, this pipeline is used for general purpose workloads.
type: filter-weigher
createHistory: false
filters: []
weighers:
- name: vmware_binpack
Expand Down Expand Up @@ -73,6 +72,5 @@ spec:

Specifically, this pipeline is used for HANA workloads.
type: filter-weigher
createHistory: false
filters: []
weighers: []
Loading