Files
ilkerandClaude Opus 5.5 090719fc74
ci / derle (push) Successful in 30s
hpcod: clusters across the nodes of a Proxmox cluster
Discovery already saw every node of a Proxmox cluster and placement could
put a VM on any of them, but the template, the snippets and a simple SDN
zone only worked on PVE_NODE, so a cluster could land where it could not
be built.

- Setup asks which nodes clusters may use (PVE_NODES; a multi-select in
  the TUI when Proxmox has more than one). Discovery offers only those.
- Heads are placed on PVE_NODE only (head_node), where the template is, so
  no shared storage is needed; diskless compute nodes spread.
- PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone
  vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade
  only while no cluster has a vnet in it. Guest fabric NICs take the
  bridge MTU through virtio (mtu = 1).
- setup keeps nodes and head_node of its own site in sites.yaml, leaving
  the rest of the file as the operator wrote it.
- verify checks the nodes are online and the zone matches.

Run on the simulated Proxmox with the zone forced to vxlan: setup remade
the zone, a two-node selftest passed, both compute NICs came up at 1450,
and a DF ping of 1422 bytes passed between them while 1423 did not. Two
real Proxmox nodes have not been tested.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 19:51:35 +01:00

252 lines
8.6 KiB
Go

package placement
import (
"strings"
"testing"
"github.com/ilkermanap/hpcod/internal/model"
"github.com/ilkermanap/hpcod/internal/spec"
)
func gpu(id string, trusted, passthrough bool) model.Device {
conf := model.ConfClass
if !trusted {
conf = model.ConfClass // GPUs are trusted at class level; use passthrough to block
}
return model.Device{
NativeID: id, Vendor: "nvidia", Capability: model.CapGPU,
Confidence: conf, Passthrough: passthrough,
}
}
func ib(id string, conf model.Confidence) model.Device {
return model.Device{
NativeID: id, Vendor: "mellanox", Capability: model.CapInfiniBand,
Confidence: conf, Passthrough: true,
}
}
func node(name string, cores, memGB int, devs ...model.Device) model.Node {
return model.Node{
Name: name, Online: true, Cores: cores,
MemBytes: int64(memGB) << 30, Devices: devs,
}
}
func site(id string, nodes ...model.Node) model.Site {
return model.Site{ID: id, Kind: "proxmox", Reachable: true, Healthy: true, Nodes: nodes}
}
func req(count, cores, memGB int, require ...spec.Constraint) spec.Request {
return spec.Request{
Name: "t", Head: spec.NodeSpec{Cores: 4, MemoryGB: 16},
Compute: spec.ComputeSpec{Count: count, Cores: cores, MemoryGB: memGB, Require: require},
}
}
// The core promise of the design: scarce hardware is kept for requests that
// need it. A plain CPU job must not land on the only InfiniBand+GPU site while
// a plain site is available.
func TestScarceHardwareIsProtected(t *testing.T) {
scarce := site("gpu-ib",
node("n1", 64, 512, gpu("0000:01:00.0", true, true), ib("0000:07:00.0", model.ConfAgent)))
plain := site("plain", node("n1", 64, 512))
d := Plan([]model.Site{scarce, plain}, req(4, 8, 32), DefaultWeights())
if !d.Feasible {
t.Fatalf("expected a placement, got: %s", d.Summary)
}
if d.Chosen.SiteID != "plain" {
t.Errorf("placed a plain CPU job on %q; scarce hardware should have been protected", d.Chosen.SiteID)
}
}
// ...and conversely, a request that needs the hardware gets it.
func TestRequestThatNeedsHardwareGetsIt(t *testing.T) {
scarce := site("gpu-ib",
node("n1", 64, 512,
gpu("0000:01:00.0", true, true), gpu("0000:02:00.0", true, true),
ib("0000:07:00.0", model.ConfAgent)))
plain := site("plain", node("n1", 64, 512))
r := req(2, 8, 32, spec.Constraint{Capability: model.CapGPU, Count: 1})
d := Plan([]model.Site{scarce, plain}, r, DefaultWeights())
if !d.Feasible || d.Chosen.SiteID != "gpu-ib" {
t.Fatalf("want gpu-ib, got feasible=%v site=%v (%s)", d.Feasible, d.Chosen, d.Summary)
}
for _, p := range d.Chosen.Placement {
if p.Role == "compute" && len(p.Devices) != 1 {
t.Errorf("compute node %d got %d devices, want 1", p.Index, len(p.Devices))
}
}
}
// An unconfirmed fabric must be refused, and the refusal must say so clearly
// rather than reporting a generic shortage.
func TestUnconfirmedFabricIsRefusedWithAUsefulReason(t *testing.T) {
s := site("maybe-ib", node("n1", 64, 512, ib("0000:07:00.0", model.ConfMap)))
r := req(1, 8, 32, spec.Constraint{Capability: model.CapInfiniBand, Count: 1})
d := Plan([]model.Site{s}, r, DefaultWeights())
if d.Feasible {
t.Fatal("scheduled onto a fabric no agent has confirmed")
}
joined := strings.Join(d.Candidates[0].Reasons, " ")
if !strings.Contains(joined, "unconfirmed") {
t.Errorf("reason should name the real problem, got: %s", joined)
}
}
// A device in a shared IOMMU group exists but cannot be handed to a guest.
func TestNonPassthroughDeviceIsNotOffered(t *testing.T) {
s := site("shared-iommu", node("n1", 64, 512, gpu("0000:01:00.0", true, false)))
r := req(1, 8, 32, spec.Constraint{Capability: model.CapGPU, Count: 1})
d := Plan([]model.Site{s}, r, DefaultWeights())
if d.Feasible {
t.Fatal("offered a GPU that cannot be passed through")
}
if !strings.Contains(strings.Join(d.Candidates[0].Reasons, " "), "passthrough") {
t.Errorf("reason should mention passthrough, got: %v", d.Candidates[0].Reasons)
}
}
// Every compute node carries a rootfs in RAM, so the allocation must exceed
// the requested usable memory. A host with exactly count*memory free is not
// actually big enough.
func TestRootfsReserveIsAccountedFor(t *testing.T) {
// 4 compute x 32 GiB usable = 128, plus 4x2 rootfs = 136, plus head 16 = 152.
tight := site("tight", node("n1", 64, 150))
roomy := site("roomy", node("n1", 64, 160))
d := Plan([]model.Site{tight, roomy}, req(4, 8, 32), DefaultWeights())
if !d.Feasible {
t.Fatalf("expected roomy to work: %s", d.Summary)
}
if d.Chosen.SiteID != "roomy" {
t.Errorf("chose %q; the tight site cannot hold the rootfs reserve", d.Chosen.SiteID)
}
for _, p := range d.Chosen.Placement {
if p.Role == "compute" && p.MemoryGB != 32+spec.RootfsReserveGB {
t.Errorf("compute allocated %d GiB, want %d", p.MemoryGB, 32+spec.RootfsReserveGB)
}
}
}
func TestUnreachableAndUnhealthySitesAreExcluded(t *testing.T) {
down := model.Site{ID: "down", Reachable: false, Error: "connection refused"}
split := model.Site{ID: "split", Reachable: true, Healthy: false,
Nodes: []model.Node{node("n1", 64, 512)}}
good := site("good", node("n1", 64, 512))
d := Plan([]model.Site{down, split, good}, req(1, 8, 32), DefaultWeights())
if !d.Feasible || d.Chosen.SiteID != "good" {
t.Fatalf("want good, got %v", d.Summary)
}
for _, c := range d.Candidates {
if c.SiteID == "down" && !strings.Contains(strings.Join(c.Reasons, " "), "unreachable") {
t.Error("unreachable site should say so")
}
if c.SiteID == "split" && !strings.Contains(strings.Join(c.Reasons, " "), "quorate") {
t.Error("non-quorate site should say so")
}
}
}
// Packing should prefer hosts already in use, so a tenant occupies as few
// hypervisors as possible and whole hosts stay free for later requests.
func TestPackingPrefersFewerHosts(t *testing.T) {
s := site("multi",
node("n1", 64, 512),
node("n2", 64, 512),
node("n3", 64, 512))
d := Plan([]model.Site{s}, req(4, 8, 32), DefaultWeights())
if !d.Feasible {
t.Fatal(d.Summary)
}
if d.Chosen.HostsUsed > 2 {
t.Errorf("spread 4 small nodes over %d hosts; expected tighter packing", d.Chosen.HostsUsed)
}
}
func TestInfeasibleReportsPerSiteReasons(t *testing.T) {
small := site("small", node("n1", 4, 16))
nogpu := site("nogpu", node("n1", 64, 512))
r := req(8, 16, 64, spec.Constraint{Capability: model.CapGPU, Count: 2})
d := Plan([]model.Site{small, nogpu}, r, DefaultWeights())
if d.Feasible {
t.Fatal("should not be feasible")
}
if len(d.Candidates) != 2 {
t.Fatalf("want an explanation per site, got %d", len(d.Candidates))
}
for _, c := range d.Candidates {
if len(c.Reasons) == 0 {
t.Errorf("%s: refused with no reason", c.SiteID)
}
}
}
func TestPreferIsSoft(t *testing.T) {
withNVMe := site("nvme", node("n1", 64, 512, model.Device{
NativeID: "0000:03:00.0", Capability: model.CapNVMe,
Confidence: model.ConfClass, Passthrough: true,
}))
without := site("plain", node("n1", 64, 512))
r := req(2, 8, 32)
r.Compute.Prefer = []spec.Constraint{{Capability: model.CapNVMe, Count: 1}}
d := Plan([]model.Site{withNVMe, without}, r, DefaultWeights())
if !d.Feasible {
t.Fatal(d.Summary)
}
if d.Chosen.SiteID != "nvme" {
t.Errorf("prefer should have tipped the choice to nvme, got %s", d.Chosen.SiteID)
}
// but it must remain satisfiable without it
d2 := Plan([]model.Site{without}, r, DefaultWeights())
if !d2.Feasible {
t.Error("prefer must not act as a hard requirement")
}
}
// The head is cloned from a template that lives on one node, so it may only
// go there; compute nodes are diskless and go wherever there is room.
func TestHeadStaysOnTheTemplateNode(t *testing.T) {
// Fullest-first packing would put the head on pve2, the tighter node.
// pve1 has room for the head and one compute node, not two.
s := site("lab", node("pve1", 16, 256), node("pve2", 12, 256))
s.HeadNode = "pve1"
d := Plan([]model.Site{s}, req(2, 8, 32), DefaultWeights())
if !d.Feasible {
t.Fatalf("expected a placement, got: %s", d.Summary)
}
hosts := map[string]bool{}
for _, p := range d.Chosen.Placement {
hosts[p.HostNode] = true
if p.Role == "head" && p.HostNode != "pve1" {
t.Errorf("head on %s, want pve1, where the template is", p.HostNode)
}
}
if !hosts["pve2"] {
t.Errorf("nothing on pve2: compute nodes should still use the other node (%v)", hosts)
}
}
func TestHeadNodeWithoutRoomIsAClearRefusal(t *testing.T) {
s := site("lab", node("pve1", 2, 4), node("pve2", 64, 256))
s.HeadNode = "pve1"
d := Plan([]model.Site{s}, req(1, 8, 32), DefaultWeights())
if d.Feasible {
t.Fatal("placed a head on pve2, which has no template")
}
if !strings.Contains(d.Summary+strings.Join(d.Candidates[0].Reasons, ";"), "head template") {
t.Errorf("reason does not say why: %s %v", d.Summary, d.Candidates[0].Reasons)
}
}