Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

483 lines
14 KiB
Go

// Package placement decides which site a request should run on.
//
// Plan is a pure function of (sites, request). It performs no I/O and reaches
// no cloud, which is what makes the decision testable against synthetic
// multi-site fixtures and what makes a dry-run endpoint possible at all.
//
// Two phases, in this order:
//
// filter - eliminate the impossible, recording why for each site
// score - rank the survivors
//
// The recording matters as much as the ranking. A portal that answers "no
// capacity" is useless; one that answers "site-a has the InfiniBand but has
// not confirmed the link, and site-b has the cores but no GPU" tells an
// operator exactly what to fix.
package placement
import (
"fmt"
"sort"
"strings"
"github.com/ilkermanap/hpcod/internal/model"
"github.com/ilkermanap/hpcod/internal/spec"
)
// Weights tune the score. Exposed so they can be adjusted per deployment
// without editing the algorithm.
type Weights struct {
CapabilityFit float64
FreeCapacity float64
Packing float64
Fragmentation float64
}
func DefaultWeights() Weights {
return Weights{
// Dominant on purpose. Scarce hardware must be kept for requests that
// need it: putting a plain CPU job on the only InfiniBand site is a
// scheduling failure even though it fits.
CapabilityFit: 3.0,
FreeCapacity: 1.0,
Packing: 0.8,
Fragmentation: 1.2,
}
}
// PlacedDevice is one piece of hardware assigned to a VM.
//
// It carries the capability and not just the address, because the decision
// outlives the inventory it was made from: a stored row reading
// "0000:01:00.0" cannot afterwards be read as "this cluster holds a GPU",
// and by the time anyone asks, the card may have been moved or removed.
type PlacedDevice struct {
NativeID string `json:"native_id"`
Capability model.Capability `json:"capability"`
Vendor string `json:"vendor,omitempty"`
Model string `json:"model,omitempty"`
}
func (d PlacedDevice) String() string {
if d.Model != "" {
return fmt.Sprintf("%s %s (%s)", d.Capability, d.NativeID, d.Model)
}
return fmt.Sprintf("%s %s", d.Capability, d.NativeID)
}
// VMPlacement is one tenant VM pinned to a hypervisor node.
type VMPlacement struct {
Role string `json:"role"` // "head" or "compute"
Index int `json:"index"`
HostNode string `json:"host_node"`
Cores int `json:"cores"`
MemoryGB int `json:"memory_gb"`
Devices []PlacedDevice `json:"devices,omitempty"`
}
// Candidate is one site's evaluation.
type Candidate struct {
SiteID string `json:"site_id"`
Feasible bool `json:"feasible"`
Reasons []string `json:"reasons,omitempty"` // why not, if infeasible
Score float64 `json:"score,omitempty"`
Breakdown []string `json:"breakdown,omitempty"`
Placement []VMPlacement `json:"placement,omitempty"`
HostsUsed int `json:"hosts_used,omitempty"`
}
// Decision is the answer.
type Decision struct {
Feasible bool `json:"feasible"`
Chosen *Candidate `json:"chosen,omitempty"`
Candidates []Candidate `json:"candidates"`
Summary string `json:"summary"`
}
// Plan evaluates every site and picks the best feasible one.
func Plan(sites []model.Site, req spec.Request, w Weights) Decision {
d := Decision{Candidates: make([]Candidate, 0, len(sites))}
for _, s := range sites {
d.Candidates = append(d.Candidates, evaluate(s, req, w))
}
sort.Slice(d.Candidates, func(i, j int) bool {
a, b := d.Candidates[i], d.Candidates[j]
if a.Feasible != b.Feasible {
return a.Feasible
}
if a.Feasible {
return a.Score > b.Score
}
return a.SiteID < b.SiteID
})
if len(d.Candidates) > 0 && d.Candidates[0].Feasible {
d.Feasible = true
d.Chosen = &d.Candidates[0]
d.Summary = fmt.Sprintf("%s: %d compute + 1 head across %d host(s), score %.2f",
d.Chosen.SiteID, req.Compute.Count, d.Chosen.HostsUsed, d.Chosen.Score)
return d
}
d.Summary = "no site can satisfy this request"
return d
}
// hostBudget tracks what remains on one hypervisor as we pack onto it.
type hostBudget struct {
name string
cores int
memGB int
devices []model.Device // unallocated, schedulable only
usedByUs bool
}
func evaluate(s model.Site, req spec.Request, w Weights) Candidate {
c := Candidate{SiteID: s.ID}
if !s.Reachable {
c.Reasons = append(c.Reasons, "site is unreachable"+detail(s.Error))
return c
}
if !s.Healthy {
c.Reasons = append(c.Reasons, "site is not healthy (not quorate)")
return c
}
budgets := budgetsFor(s)
if len(budgets) == 0 {
c.Reasons = append(c.Reasons, "no online hypervisor nodes")
return c
}
// Capability check first: it produces the most actionable message, and a
// site missing the hardware can never be fixed by freeing capacity.
if reasons := checkCapabilities(s, req); len(reasons) > 0 {
c.Reasons = append(c.Reasons, reasons...)
return c
}
place, ok, why := pack(budgets, req, s.HeadNode)
if !ok {
c.Reasons = append(c.Reasons, why...)
return c
}
c.Feasible = true
c.Placement = place
hosts := map[string]bool{}
for _, p := range place {
hosts[p.HostNode] = true
}
c.HostsUsed = len(hosts)
c.Score, c.Breakdown = score(s, req, c, w)
return c
}
func budgetsFor(s model.Site) []*hostBudget {
var out []*hostBudget
for _, n := range s.Nodes {
if !n.Online {
continue
}
b := &hostBudget{
name: n.Name,
cores: n.FreeCores(),
memGB: int(n.FreeMemBytes() / (1 << 30)),
}
for _, d := range n.Devices {
if d.AllocatedTo == "" && d.Passthrough && d.Capability.Trusted(d.Confidence) {
b.devices = append(b.devices, d)
}
}
out = append(out, b)
}
// Fullest-first packing: consume the tightest hosts before the roomy ones,
// so large future requests still have a whole node to land on.
sort.Slice(out, func(i, j int) bool {
if out[i].cores != out[j].cores {
return out[i].cores < out[j].cores
}
return out[i].name < out[j].name
})
return out
}
// checkCapabilities reports, in operator terms, why a site's hardware cannot
// satisfy the request. It deliberately distinguishes "absent" from "present
// but unconfirmed", because those need completely different responses.
func checkCapabilities(s model.Site, req spec.Request) []string {
var reasons []string
trusted := map[model.Capability]int{}
untrusted := map[model.Capability]int{}
blocked := map[model.Capability]int{}
for _, n := range s.Nodes {
if !n.Online {
continue
}
for _, d := range n.Devices {
if d.AllocatedTo != "" {
continue
}
switch {
case !d.Capability.Trusted(d.Confidence):
untrusted[d.Capability]++
case !d.Passthrough:
blocked[d.Capability]++
default:
trusted[d.Capability]++
}
}
}
for _, want := range req.Compute.Require {
need := want.Count * req.Compute.Count
have := trusted[want.Capability]
if have >= need {
continue
}
switch {
case untrusted[want.Capability] > 0:
reasons = append(reasons, fmt.Sprintf(
"%s: %d present but unconfirmed - the fabric has not been verified by a node agent, so it is not schedulable (need %d, confirmed %d)",
want.Capability, untrusted[want.Capability], need, have))
case blocked[want.Capability] > 0:
reasons = append(reasons, fmt.Sprintf(
"%s: %d present but not passthrough-safe (shared IOMMU group); need %d, usable %d",
want.Capability, blocked[want.Capability], need, have))
case have == 0:
reasons = append(reasons, fmt.Sprintf("%s: none present (need %d)", want.Capability, need))
default:
reasons = append(reasons, fmt.Sprintf("%s: need %d, only %d usable", want.Capability, need, have))
}
}
return reasons
}
// pack assigns the head node and every compute node to a hypervisor.
//
// headNode, when set, is the only host the head may use: the template is
// there, and a clone cannot cross nodes without shared storage.
func pack(budgets []*hostBudget, req spec.Request, headNode string) ([]VMPlacement, bool, []string) {
var out []VMPlacement
// Head first: it needs no devices, so placing it early avoids stranding a
// device-bearing host on a VM that did not need one.
headMem := req.Head.MemoryGB
placedHead := false
for _, b := range budgets {
if headNode != "" && b.name != headNode {
continue
}
if b.cores >= req.Head.Cores && b.memGB >= headMem {
b.cores -= req.Head.Cores
b.memGB -= headMem
b.usedByUs = true
out = append(out, VMPlacement{
Role: "head", Index: 0, HostNode: b.name,
Cores: req.Head.Cores, MemoryGB: headMem,
})
placedHead = true
break
}
}
if !placedHead {
if headNode != "" {
return nil, false, []string{fmt.Sprintf(
"%s, which holds the head template, cannot fit the head node (%d cores, %d GiB) or is offline",
headNode, req.Head.Cores, headMem)}
}
return nil, false, []string{fmt.Sprintf(
"no host can fit the head node (%d cores, %d GiB)", req.Head.Cores, headMem)}
}
computeMem := req.MemoryPerComputeGB()
for i := 1; i <= req.Compute.Count; i++ {
b, devs := fitCompute(budgets, req, computeMem)
if b == nil {
return nil, false, []string{fmt.Sprintf(
"placed %d of %d compute nodes; no host has room for another (%d cores, %d GiB incl. %d GiB rootfs%s)",
i-1, req.Compute.Count, req.Compute.Cores, computeMem,
spec.RootfsReserveGB, devSuffix(req))}
}
b.cores -= req.Compute.Cores
b.memGB -= computeMem
b.usedByUs = true
var placed []PlacedDevice
for _, d := range devs {
placed = append(placed, PlacedDevice{
NativeID: d.NativeID, Capability: d.Capability,
Vendor: d.Vendor, Model: d.Model,
})
b.devices = removeDevice(b.devices, d.NativeID)
}
out = append(out, VMPlacement{
Role: "compute", Index: i, HostNode: b.name,
Cores: req.Compute.Cores, MemoryGB: computeMem, Devices: placed,
})
}
return out, true, nil
}
// fitCompute finds a host with room and the required devices. Hosts already in
// use by this tenant are preferred, which keeps a cluster on as few hypervisors
// as possible and leaves whole hosts free for later requests.
func fitCompute(budgets []*hostBudget, req spec.Request, memGB int) (*hostBudget, []model.Device) {
try := func(onlyUsed bool) (*hostBudget, []model.Device) {
for _, b := range budgets {
if onlyUsed != b.usedByUs {
continue
}
if b.cores < req.Compute.Cores || b.memGB < memGB {
continue
}
devs, ok := takeDevices(b.devices, req.Compute.Require)
if !ok {
continue
}
return b, devs
}
return nil, nil
}
if b, d := try(true); b != nil {
return b, d
}
return try(false)
}
func takeDevices(pool []model.Device, want []spec.Constraint) ([]model.Device, bool) {
var taken []model.Device
used := map[string]bool{}
for _, c := range want {
found := 0
for _, d := range pool {
if found == c.Count {
break
}
if used[d.NativeID] || d.Capability != c.Capability {
continue
}
if c.Vendor != "" && !strings.EqualFold(c.Vendor, d.Vendor) {
continue
}
used[d.NativeID] = true
taken = append(taken, d)
found++
}
if found < c.Count {
return nil, false
}
}
return taken, true
}
func removeDevice(pool []model.Device, id string) []model.Device {
out := pool[:0]
for _, d := range pool {
if d.NativeID != id {
out = append(out, d)
}
}
return out
}
// score ranks a feasible candidate. Higher is better.
func score(s model.Site, req spec.Request, c Candidate, w Weights) (float64, []string) {
var notes []string
// capability_fit: penalise consuming a site whose scarce capabilities the
// request does not want.
wanted := map[model.Capability]bool{}
for _, r := range req.Compute.Require {
wanted[r.Capability] = true
}
for _, p := range req.Compute.Prefer {
wanted[p.Capability] = true
}
waste := 0
for cap, n := range s.Capabilities() {
if !wanted[cap] {
waste += n
}
}
fit := 1.0 / (1.0 + float64(waste))
notes = append(notes, fmt.Sprintf("capability_fit %.2f (%d unrequested scarce device(s) here)", fit, waste))
// prefer: soft capabilities that are present.
bonus := 0.0
for _, p := range req.Compute.Prefer {
if s.Capabilities()[p.Capability] > 0 {
bonus += 0.25
notes = append(notes, fmt.Sprintf("prefer %s satisfied (+0.25)", p.Capability))
}
}
// free_capacity_ratio: prefer emptier sites.
var totalCores, freeCores int
for _, n := range s.Nodes {
if n.Online {
totalCores += n.Cores
freeCores += n.FreeCores()
}
}
ratio := 0.0
if totalCores > 0 {
ratio = float64(freeCores) / float64(totalCores)
}
notes = append(notes, fmt.Sprintf("free_capacity %.2f (%d/%d cores)", ratio, freeCores, totalCores))
// packing: fewer hypervisors per tenant is better.
packing := 1.0 / float64(max(1, c.HostsUsed))
notes = append(notes, fmt.Sprintf("packing %.2f (%d host(s))", packing, c.HostsUsed))
// fragmentation: penalise leaving small unusable remainders.
frag := 0.0
online := 0
for _, n := range s.Nodes {
if n.Online {
online++
}
}
if online > 0 && req.Compute.Cores > 0 {
leftover := freeCores - (req.Compute.Cores*req.Compute.Count + req.Head.Cores)
if leftover > 0 && leftover < req.Compute.Cores {
frag = 1.0
notes = append(notes, fmt.Sprintf("fragmentation penalty (%d cores left, too few for another node)", leftover))
}
}
total := w.CapabilityFit*fit + w.FreeCapacity*ratio + w.Packing*packing - w.Fragmentation*frag + bonus
return total, notes
}
func detail(s string) string {
if s == "" {
return ""
}
if len(s) > 120 {
s = s[:120] + "..."
}
return ": " + s
}
func devSuffix(req spec.Request) string {
if len(req.Compute.Require) == 0 {
return ""
}
var parts []string
for _, c := range req.Compute.Require {
parts = append(parts, c.String())
}
return ", " + strings.Join(parts, " + ")
}
func max(a, b int) int {
if a > b {
return a
}
return b
}