ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
483 lines
14 KiB
Go
483 lines
14 KiB
Go
// Package placement decides which site a request should run on.
|
|
//
|
|
// Plan is a pure function of (sites, request). It performs no I/O and reaches
|
|
// no cloud, which is what makes the decision testable against synthetic
|
|
// multi-site fixtures and what makes a dry-run endpoint possible at all.
|
|
//
|
|
// Two phases, in this order:
|
|
//
|
|
// filter - eliminate the impossible, recording why for each site
|
|
// score - rank the survivors
|
|
//
|
|
// The recording matters as much as the ranking. A portal that answers "no
|
|
// capacity" is useless; one that answers "site-a has the InfiniBand but has
|
|
// not confirmed the link, and site-b has the cores but no GPU" tells an
|
|
// operator exactly what to fix.
|
|
package placement
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/model"
|
|
"github.com/ilkermanap/hpcod/internal/spec"
|
|
)
|
|
|
|
// Weights tune the score. Exposed so they can be adjusted per deployment
|
|
// without editing the algorithm.
|
|
type Weights struct {
|
|
CapabilityFit float64
|
|
FreeCapacity float64
|
|
Packing float64
|
|
Fragmentation float64
|
|
}
|
|
|
|
func DefaultWeights() Weights {
|
|
return Weights{
|
|
// Dominant on purpose. Scarce hardware must be kept for requests that
|
|
// need it: putting a plain CPU job on the only InfiniBand site is a
|
|
// scheduling failure even though it fits.
|
|
CapabilityFit: 3.0,
|
|
FreeCapacity: 1.0,
|
|
Packing: 0.8,
|
|
Fragmentation: 1.2,
|
|
}
|
|
}
|
|
|
|
// PlacedDevice is one piece of hardware assigned to a VM.
|
|
//
|
|
// It carries the capability and not just the address, because the decision
|
|
// outlives the inventory it was made from: a stored row reading
|
|
// "0000:01:00.0" cannot afterwards be read as "this cluster holds a GPU",
|
|
// and by the time anyone asks, the card may have been moved or removed.
|
|
type PlacedDevice struct {
|
|
NativeID string `json:"native_id"`
|
|
Capability model.Capability `json:"capability"`
|
|
Vendor string `json:"vendor,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
}
|
|
|
|
func (d PlacedDevice) String() string {
|
|
if d.Model != "" {
|
|
return fmt.Sprintf("%s %s (%s)", d.Capability, d.NativeID, d.Model)
|
|
}
|
|
return fmt.Sprintf("%s %s", d.Capability, d.NativeID)
|
|
}
|
|
|
|
// VMPlacement is one tenant VM pinned to a hypervisor node.
|
|
type VMPlacement struct {
|
|
Role string `json:"role"` // "head" or "compute"
|
|
Index int `json:"index"`
|
|
HostNode string `json:"host_node"`
|
|
Cores int `json:"cores"`
|
|
MemoryGB int `json:"memory_gb"`
|
|
Devices []PlacedDevice `json:"devices,omitempty"`
|
|
}
|
|
|
|
// Candidate is one site's evaluation.
|
|
type Candidate struct {
|
|
SiteID string `json:"site_id"`
|
|
Feasible bool `json:"feasible"`
|
|
Reasons []string `json:"reasons,omitempty"` // why not, if infeasible
|
|
Score float64 `json:"score,omitempty"`
|
|
Breakdown []string `json:"breakdown,omitempty"`
|
|
Placement []VMPlacement `json:"placement,omitempty"`
|
|
HostsUsed int `json:"hosts_used,omitempty"`
|
|
}
|
|
|
|
// Decision is the answer.
|
|
type Decision struct {
|
|
Feasible bool `json:"feasible"`
|
|
Chosen *Candidate `json:"chosen,omitempty"`
|
|
Candidates []Candidate `json:"candidates"`
|
|
Summary string `json:"summary"`
|
|
}
|
|
|
|
// Plan evaluates every site and picks the best feasible one.
|
|
func Plan(sites []model.Site, req spec.Request, w Weights) Decision {
|
|
d := Decision{Candidates: make([]Candidate, 0, len(sites))}
|
|
|
|
for _, s := range sites {
|
|
d.Candidates = append(d.Candidates, evaluate(s, req, w))
|
|
}
|
|
sort.Slice(d.Candidates, func(i, j int) bool {
|
|
a, b := d.Candidates[i], d.Candidates[j]
|
|
if a.Feasible != b.Feasible {
|
|
return a.Feasible
|
|
}
|
|
if a.Feasible {
|
|
return a.Score > b.Score
|
|
}
|
|
return a.SiteID < b.SiteID
|
|
})
|
|
|
|
if len(d.Candidates) > 0 && d.Candidates[0].Feasible {
|
|
d.Feasible = true
|
|
d.Chosen = &d.Candidates[0]
|
|
d.Summary = fmt.Sprintf("%s: %d compute + 1 head across %d host(s), score %.2f",
|
|
d.Chosen.SiteID, req.Compute.Count, d.Chosen.HostsUsed, d.Chosen.Score)
|
|
return d
|
|
}
|
|
|
|
d.Summary = "no site can satisfy this request"
|
|
return d
|
|
}
|
|
|
|
// hostBudget tracks what remains on one hypervisor as we pack onto it.
|
|
type hostBudget struct {
|
|
name string
|
|
cores int
|
|
memGB int
|
|
devices []model.Device // unallocated, schedulable only
|
|
usedByUs bool
|
|
}
|
|
|
|
func evaluate(s model.Site, req spec.Request, w Weights) Candidate {
|
|
c := Candidate{SiteID: s.ID}
|
|
|
|
if !s.Reachable {
|
|
c.Reasons = append(c.Reasons, "site is unreachable"+detail(s.Error))
|
|
return c
|
|
}
|
|
if !s.Healthy {
|
|
c.Reasons = append(c.Reasons, "site is not healthy (not quorate)")
|
|
return c
|
|
}
|
|
|
|
budgets := budgetsFor(s)
|
|
if len(budgets) == 0 {
|
|
c.Reasons = append(c.Reasons, "no online hypervisor nodes")
|
|
return c
|
|
}
|
|
|
|
// Capability check first: it produces the most actionable message, and a
|
|
// site missing the hardware can never be fixed by freeing capacity.
|
|
if reasons := checkCapabilities(s, req); len(reasons) > 0 {
|
|
c.Reasons = append(c.Reasons, reasons...)
|
|
return c
|
|
}
|
|
|
|
place, ok, why := pack(budgets, req, s.HeadNode)
|
|
if !ok {
|
|
c.Reasons = append(c.Reasons, why...)
|
|
return c
|
|
}
|
|
|
|
c.Feasible = true
|
|
c.Placement = place
|
|
hosts := map[string]bool{}
|
|
for _, p := range place {
|
|
hosts[p.HostNode] = true
|
|
}
|
|
c.HostsUsed = len(hosts)
|
|
c.Score, c.Breakdown = score(s, req, c, w)
|
|
return c
|
|
}
|
|
|
|
func budgetsFor(s model.Site) []*hostBudget {
|
|
var out []*hostBudget
|
|
for _, n := range s.Nodes {
|
|
if !n.Online {
|
|
continue
|
|
}
|
|
b := &hostBudget{
|
|
name: n.Name,
|
|
cores: n.FreeCores(),
|
|
memGB: int(n.FreeMemBytes() / (1 << 30)),
|
|
}
|
|
for _, d := range n.Devices {
|
|
if d.AllocatedTo == "" && d.Passthrough && d.Capability.Trusted(d.Confidence) {
|
|
b.devices = append(b.devices, d)
|
|
}
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
// Fullest-first packing: consume the tightest hosts before the roomy ones,
|
|
// so large future requests still have a whole node to land on.
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if out[i].cores != out[j].cores {
|
|
return out[i].cores < out[j].cores
|
|
}
|
|
return out[i].name < out[j].name
|
|
})
|
|
return out
|
|
}
|
|
|
|
// checkCapabilities reports, in operator terms, why a site's hardware cannot
|
|
// satisfy the request. It deliberately distinguishes "absent" from "present
|
|
// but unconfirmed", because those need completely different responses.
|
|
func checkCapabilities(s model.Site, req spec.Request) []string {
|
|
var reasons []string
|
|
|
|
trusted := map[model.Capability]int{}
|
|
untrusted := map[model.Capability]int{}
|
|
blocked := map[model.Capability]int{}
|
|
for _, n := range s.Nodes {
|
|
if !n.Online {
|
|
continue
|
|
}
|
|
for _, d := range n.Devices {
|
|
if d.AllocatedTo != "" {
|
|
continue
|
|
}
|
|
switch {
|
|
case !d.Capability.Trusted(d.Confidence):
|
|
untrusted[d.Capability]++
|
|
case !d.Passthrough:
|
|
blocked[d.Capability]++
|
|
default:
|
|
trusted[d.Capability]++
|
|
}
|
|
}
|
|
}
|
|
|
|
for _, want := range req.Compute.Require {
|
|
need := want.Count * req.Compute.Count
|
|
have := trusted[want.Capability]
|
|
if have >= need {
|
|
continue
|
|
}
|
|
switch {
|
|
case untrusted[want.Capability] > 0:
|
|
reasons = append(reasons, fmt.Sprintf(
|
|
"%s: %d present but unconfirmed - the fabric has not been verified by a node agent, so it is not schedulable (need %d, confirmed %d)",
|
|
want.Capability, untrusted[want.Capability], need, have))
|
|
case blocked[want.Capability] > 0:
|
|
reasons = append(reasons, fmt.Sprintf(
|
|
"%s: %d present but not passthrough-safe (shared IOMMU group); need %d, usable %d",
|
|
want.Capability, blocked[want.Capability], need, have))
|
|
case have == 0:
|
|
reasons = append(reasons, fmt.Sprintf("%s: none present (need %d)", want.Capability, need))
|
|
default:
|
|
reasons = append(reasons, fmt.Sprintf("%s: need %d, only %d usable", want.Capability, need, have))
|
|
}
|
|
}
|
|
return reasons
|
|
}
|
|
|
|
// pack assigns the head node and every compute node to a hypervisor.
|
|
//
|
|
// headNode, when set, is the only host the head may use: the template is
|
|
// there, and a clone cannot cross nodes without shared storage.
|
|
func pack(budgets []*hostBudget, req spec.Request, headNode string) ([]VMPlacement, bool, []string) {
|
|
var out []VMPlacement
|
|
|
|
// Head first: it needs no devices, so placing it early avoids stranding a
|
|
// device-bearing host on a VM that did not need one.
|
|
headMem := req.Head.MemoryGB
|
|
placedHead := false
|
|
for _, b := range budgets {
|
|
if headNode != "" && b.name != headNode {
|
|
continue
|
|
}
|
|
if b.cores >= req.Head.Cores && b.memGB >= headMem {
|
|
b.cores -= req.Head.Cores
|
|
b.memGB -= headMem
|
|
b.usedByUs = true
|
|
out = append(out, VMPlacement{
|
|
Role: "head", Index: 0, HostNode: b.name,
|
|
Cores: req.Head.Cores, MemoryGB: headMem,
|
|
})
|
|
placedHead = true
|
|
break
|
|
}
|
|
}
|
|
if !placedHead {
|
|
if headNode != "" {
|
|
return nil, false, []string{fmt.Sprintf(
|
|
"%s, which holds the head template, cannot fit the head node (%d cores, %d GiB) or is offline",
|
|
headNode, req.Head.Cores, headMem)}
|
|
}
|
|
return nil, false, []string{fmt.Sprintf(
|
|
"no host can fit the head node (%d cores, %d GiB)", req.Head.Cores, headMem)}
|
|
}
|
|
|
|
computeMem := req.MemoryPerComputeGB()
|
|
for i := 1; i <= req.Compute.Count; i++ {
|
|
b, devs := fitCompute(budgets, req, computeMem)
|
|
if b == nil {
|
|
return nil, false, []string{fmt.Sprintf(
|
|
"placed %d of %d compute nodes; no host has room for another (%d cores, %d GiB incl. %d GiB rootfs%s)",
|
|
i-1, req.Compute.Count, req.Compute.Cores, computeMem,
|
|
spec.RootfsReserveGB, devSuffix(req))}
|
|
}
|
|
b.cores -= req.Compute.Cores
|
|
b.memGB -= computeMem
|
|
b.usedByUs = true
|
|
|
|
var placed []PlacedDevice
|
|
for _, d := range devs {
|
|
placed = append(placed, PlacedDevice{
|
|
NativeID: d.NativeID, Capability: d.Capability,
|
|
Vendor: d.Vendor, Model: d.Model,
|
|
})
|
|
b.devices = removeDevice(b.devices, d.NativeID)
|
|
}
|
|
out = append(out, VMPlacement{
|
|
Role: "compute", Index: i, HostNode: b.name,
|
|
Cores: req.Compute.Cores, MemoryGB: computeMem, Devices: placed,
|
|
})
|
|
}
|
|
return out, true, nil
|
|
}
|
|
|
|
// fitCompute finds a host with room and the required devices. Hosts already in
|
|
// use by this tenant are preferred, which keeps a cluster on as few hypervisors
|
|
// as possible and leaves whole hosts free for later requests.
|
|
func fitCompute(budgets []*hostBudget, req spec.Request, memGB int) (*hostBudget, []model.Device) {
|
|
try := func(onlyUsed bool) (*hostBudget, []model.Device) {
|
|
for _, b := range budgets {
|
|
if onlyUsed != b.usedByUs {
|
|
continue
|
|
}
|
|
if b.cores < req.Compute.Cores || b.memGB < memGB {
|
|
continue
|
|
}
|
|
devs, ok := takeDevices(b.devices, req.Compute.Require)
|
|
if !ok {
|
|
continue
|
|
}
|
|
return b, devs
|
|
}
|
|
return nil, nil
|
|
}
|
|
if b, d := try(true); b != nil {
|
|
return b, d
|
|
}
|
|
return try(false)
|
|
}
|
|
|
|
func takeDevices(pool []model.Device, want []spec.Constraint) ([]model.Device, bool) {
|
|
var taken []model.Device
|
|
used := map[string]bool{}
|
|
for _, c := range want {
|
|
found := 0
|
|
for _, d := range pool {
|
|
if found == c.Count {
|
|
break
|
|
}
|
|
if used[d.NativeID] || d.Capability != c.Capability {
|
|
continue
|
|
}
|
|
if c.Vendor != "" && !strings.EqualFold(c.Vendor, d.Vendor) {
|
|
continue
|
|
}
|
|
used[d.NativeID] = true
|
|
taken = append(taken, d)
|
|
found++
|
|
}
|
|
if found < c.Count {
|
|
return nil, false
|
|
}
|
|
}
|
|
return taken, true
|
|
}
|
|
|
|
func removeDevice(pool []model.Device, id string) []model.Device {
|
|
out := pool[:0]
|
|
for _, d := range pool {
|
|
if d.NativeID != id {
|
|
out = append(out, d)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// score ranks a feasible candidate. Higher is better.
|
|
func score(s model.Site, req spec.Request, c Candidate, w Weights) (float64, []string) {
|
|
var notes []string
|
|
|
|
// capability_fit: penalise consuming a site whose scarce capabilities the
|
|
// request does not want.
|
|
wanted := map[model.Capability]bool{}
|
|
for _, r := range req.Compute.Require {
|
|
wanted[r.Capability] = true
|
|
}
|
|
for _, p := range req.Compute.Prefer {
|
|
wanted[p.Capability] = true
|
|
}
|
|
waste := 0
|
|
for cap, n := range s.Capabilities() {
|
|
if !wanted[cap] {
|
|
waste += n
|
|
}
|
|
}
|
|
fit := 1.0 / (1.0 + float64(waste))
|
|
notes = append(notes, fmt.Sprintf("capability_fit %.2f (%d unrequested scarce device(s) here)", fit, waste))
|
|
|
|
// prefer: soft capabilities that are present.
|
|
bonus := 0.0
|
|
for _, p := range req.Compute.Prefer {
|
|
if s.Capabilities()[p.Capability] > 0 {
|
|
bonus += 0.25
|
|
notes = append(notes, fmt.Sprintf("prefer %s satisfied (+0.25)", p.Capability))
|
|
}
|
|
}
|
|
|
|
// free_capacity_ratio: prefer emptier sites.
|
|
var totalCores, freeCores int
|
|
for _, n := range s.Nodes {
|
|
if n.Online {
|
|
totalCores += n.Cores
|
|
freeCores += n.FreeCores()
|
|
}
|
|
}
|
|
ratio := 0.0
|
|
if totalCores > 0 {
|
|
ratio = float64(freeCores) / float64(totalCores)
|
|
}
|
|
notes = append(notes, fmt.Sprintf("free_capacity %.2f (%d/%d cores)", ratio, freeCores, totalCores))
|
|
|
|
// packing: fewer hypervisors per tenant is better.
|
|
packing := 1.0 / float64(max(1, c.HostsUsed))
|
|
notes = append(notes, fmt.Sprintf("packing %.2f (%d host(s))", packing, c.HostsUsed))
|
|
|
|
// fragmentation: penalise leaving small unusable remainders.
|
|
frag := 0.0
|
|
online := 0
|
|
for _, n := range s.Nodes {
|
|
if n.Online {
|
|
online++
|
|
}
|
|
}
|
|
if online > 0 && req.Compute.Cores > 0 {
|
|
leftover := freeCores - (req.Compute.Cores*req.Compute.Count + req.Head.Cores)
|
|
if leftover > 0 && leftover < req.Compute.Cores {
|
|
frag = 1.0
|
|
notes = append(notes, fmt.Sprintf("fragmentation penalty (%d cores left, too few for another node)", leftover))
|
|
}
|
|
}
|
|
|
|
total := w.CapabilityFit*fit + w.FreeCapacity*ratio + w.Packing*packing - w.Fragmentation*frag + bonus
|
|
return total, notes
|
|
}
|
|
|
|
func detail(s string) string {
|
|
if s == "" {
|
|
return ""
|
|
}
|
|
if len(s) > 120 {
|
|
s = s[:120] + "..."
|
|
}
|
|
return ": " + s
|
|
}
|
|
|
|
func devSuffix(req spec.Request) string {
|
|
if len(req.Compute.Require) == 0 {
|
|
return ""
|
|
}
|
|
var parts []string
|
|
for _, c := range req.Compute.Require {
|
|
parts = append(parts, c.String())
|
|
}
|
|
return ", " + strings.Join(parts, " + ")
|
|
}
|
|
|
|
func max(a, b int) int {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|