ci / derle (push) Successful in 30s
Discovery already saw every node of a Proxmox cluster and placement could put a VM on any of them, but the template, the snippets and a simple SDN zone only worked on PVE_NODE, so a cluster could land where it could not be built. - Setup asks which nodes clusters may use (PVE_NODES; a multi-select in the TUI when Proxmox has more than one). Discovery offers only those. - Heads are placed on PVE_NODE only (head_node), where the template is, so no shared storage is needed; diskless compute nodes spread. - PVE_SDN_ZONE_TYPE (auto|simple|vxlan): more than one node makes the zone vxlan, every node a peer, MTU 1450. A zone of the wrong type is remade only while no cluster has a vnet in it. Guest fabric NICs take the bridge MTU through virtio (mtu = 1). - setup keeps nodes and head_node of its own site in sites.yaml, leaving the rest of the file as the operator wrote it. - verify checks the nodes are online and the zone matches. Run on the simulated Proxmox with the zone forced to vxlan: setup remade the zone, a two-node selftest passed, both compute NICs came up at 1450, and a DF ping of 1422 bytes passed between them while 1423 did not. Two real Proxmox nodes have not been tested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
252 lines
8.6 KiB
Go
252 lines
8.6 KiB
Go
package placement
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/model"
|
|
"github.com/ilkermanap/hpcod/internal/spec"
|
|
)
|
|
|
|
func gpu(id string, trusted, passthrough bool) model.Device {
|
|
conf := model.ConfClass
|
|
if !trusted {
|
|
conf = model.ConfClass // GPUs are trusted at class level; use passthrough to block
|
|
}
|
|
return model.Device{
|
|
NativeID: id, Vendor: "nvidia", Capability: model.CapGPU,
|
|
Confidence: conf, Passthrough: passthrough,
|
|
}
|
|
}
|
|
|
|
func ib(id string, conf model.Confidence) model.Device {
|
|
return model.Device{
|
|
NativeID: id, Vendor: "mellanox", Capability: model.CapInfiniBand,
|
|
Confidence: conf, Passthrough: true,
|
|
}
|
|
}
|
|
|
|
func node(name string, cores, memGB int, devs ...model.Device) model.Node {
|
|
return model.Node{
|
|
Name: name, Online: true, Cores: cores,
|
|
MemBytes: int64(memGB) << 30, Devices: devs,
|
|
}
|
|
}
|
|
|
|
func site(id string, nodes ...model.Node) model.Site {
|
|
return model.Site{ID: id, Kind: "proxmox", Reachable: true, Healthy: true, Nodes: nodes}
|
|
}
|
|
|
|
func req(count, cores, memGB int, require ...spec.Constraint) spec.Request {
|
|
return spec.Request{
|
|
Name: "t", Head: spec.NodeSpec{Cores: 4, MemoryGB: 16},
|
|
Compute: spec.ComputeSpec{Count: count, Cores: cores, MemoryGB: memGB, Require: require},
|
|
}
|
|
}
|
|
|
|
// The core promise of the design: scarce hardware is kept for requests that
|
|
// need it. A plain CPU job must not land on the only InfiniBand+GPU site while
|
|
// a plain site is available.
|
|
func TestScarceHardwareIsProtected(t *testing.T) {
|
|
scarce := site("gpu-ib",
|
|
node("n1", 64, 512, gpu("0000:01:00.0", true, true), ib("0000:07:00.0", model.ConfAgent)))
|
|
plain := site("plain", node("n1", 64, 512))
|
|
|
|
d := Plan([]model.Site{scarce, plain}, req(4, 8, 32), DefaultWeights())
|
|
if !d.Feasible {
|
|
t.Fatalf("expected a placement, got: %s", d.Summary)
|
|
}
|
|
if d.Chosen.SiteID != "plain" {
|
|
t.Errorf("placed a plain CPU job on %q; scarce hardware should have been protected", d.Chosen.SiteID)
|
|
}
|
|
}
|
|
|
|
// ...and conversely, a request that needs the hardware gets it.
|
|
func TestRequestThatNeedsHardwareGetsIt(t *testing.T) {
|
|
scarce := site("gpu-ib",
|
|
node("n1", 64, 512,
|
|
gpu("0000:01:00.0", true, true), gpu("0000:02:00.0", true, true),
|
|
ib("0000:07:00.0", model.ConfAgent)))
|
|
plain := site("plain", node("n1", 64, 512))
|
|
|
|
r := req(2, 8, 32, spec.Constraint{Capability: model.CapGPU, Count: 1})
|
|
d := Plan([]model.Site{scarce, plain}, r, DefaultWeights())
|
|
if !d.Feasible || d.Chosen.SiteID != "gpu-ib" {
|
|
t.Fatalf("want gpu-ib, got feasible=%v site=%v (%s)", d.Feasible, d.Chosen, d.Summary)
|
|
}
|
|
for _, p := range d.Chosen.Placement {
|
|
if p.Role == "compute" && len(p.Devices) != 1 {
|
|
t.Errorf("compute node %d got %d devices, want 1", p.Index, len(p.Devices))
|
|
}
|
|
}
|
|
}
|
|
|
|
// An unconfirmed fabric must be refused, and the refusal must say so clearly
|
|
// rather than reporting a generic shortage.
|
|
func TestUnconfirmedFabricIsRefusedWithAUsefulReason(t *testing.T) {
|
|
s := site("maybe-ib", node("n1", 64, 512, ib("0000:07:00.0", model.ConfMap)))
|
|
|
|
r := req(1, 8, 32, spec.Constraint{Capability: model.CapInfiniBand, Count: 1})
|
|
d := Plan([]model.Site{s}, r, DefaultWeights())
|
|
|
|
if d.Feasible {
|
|
t.Fatal("scheduled onto a fabric no agent has confirmed")
|
|
}
|
|
joined := strings.Join(d.Candidates[0].Reasons, " ")
|
|
if !strings.Contains(joined, "unconfirmed") {
|
|
t.Errorf("reason should name the real problem, got: %s", joined)
|
|
}
|
|
}
|
|
|
|
// A device in a shared IOMMU group exists but cannot be handed to a guest.
|
|
func TestNonPassthroughDeviceIsNotOffered(t *testing.T) {
|
|
s := site("shared-iommu", node("n1", 64, 512, gpu("0000:01:00.0", true, false)))
|
|
r := req(1, 8, 32, spec.Constraint{Capability: model.CapGPU, Count: 1})
|
|
|
|
d := Plan([]model.Site{s}, r, DefaultWeights())
|
|
if d.Feasible {
|
|
t.Fatal("offered a GPU that cannot be passed through")
|
|
}
|
|
if !strings.Contains(strings.Join(d.Candidates[0].Reasons, " "), "passthrough") {
|
|
t.Errorf("reason should mention passthrough, got: %v", d.Candidates[0].Reasons)
|
|
}
|
|
}
|
|
|
|
// Every compute node carries a rootfs in RAM, so the allocation must exceed
|
|
// the requested usable memory. A host with exactly count*memory free is not
|
|
// actually big enough.
|
|
func TestRootfsReserveIsAccountedFor(t *testing.T) {
|
|
// 4 compute x 32 GiB usable = 128, plus 4x2 rootfs = 136, plus head 16 = 152.
|
|
tight := site("tight", node("n1", 64, 150))
|
|
roomy := site("roomy", node("n1", 64, 160))
|
|
|
|
d := Plan([]model.Site{tight, roomy}, req(4, 8, 32), DefaultWeights())
|
|
if !d.Feasible {
|
|
t.Fatalf("expected roomy to work: %s", d.Summary)
|
|
}
|
|
if d.Chosen.SiteID != "roomy" {
|
|
t.Errorf("chose %q; the tight site cannot hold the rootfs reserve", d.Chosen.SiteID)
|
|
}
|
|
for _, p := range d.Chosen.Placement {
|
|
if p.Role == "compute" && p.MemoryGB != 32+spec.RootfsReserveGB {
|
|
t.Errorf("compute allocated %d GiB, want %d", p.MemoryGB, 32+spec.RootfsReserveGB)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestUnreachableAndUnhealthySitesAreExcluded(t *testing.T) {
|
|
down := model.Site{ID: "down", Reachable: false, Error: "connection refused"}
|
|
split := model.Site{ID: "split", Reachable: true, Healthy: false,
|
|
Nodes: []model.Node{node("n1", 64, 512)}}
|
|
good := site("good", node("n1", 64, 512))
|
|
|
|
d := Plan([]model.Site{down, split, good}, req(1, 8, 32), DefaultWeights())
|
|
if !d.Feasible || d.Chosen.SiteID != "good" {
|
|
t.Fatalf("want good, got %v", d.Summary)
|
|
}
|
|
for _, c := range d.Candidates {
|
|
if c.SiteID == "down" && !strings.Contains(strings.Join(c.Reasons, " "), "unreachable") {
|
|
t.Error("unreachable site should say so")
|
|
}
|
|
if c.SiteID == "split" && !strings.Contains(strings.Join(c.Reasons, " "), "quorate") {
|
|
t.Error("non-quorate site should say so")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Packing should prefer hosts already in use, so a tenant occupies as few
|
|
// hypervisors as possible and whole hosts stay free for later requests.
|
|
func TestPackingPrefersFewerHosts(t *testing.T) {
|
|
s := site("multi",
|
|
node("n1", 64, 512),
|
|
node("n2", 64, 512),
|
|
node("n3", 64, 512))
|
|
|
|
d := Plan([]model.Site{s}, req(4, 8, 32), DefaultWeights())
|
|
if !d.Feasible {
|
|
t.Fatal(d.Summary)
|
|
}
|
|
if d.Chosen.HostsUsed > 2 {
|
|
t.Errorf("spread 4 small nodes over %d hosts; expected tighter packing", d.Chosen.HostsUsed)
|
|
}
|
|
}
|
|
|
|
func TestInfeasibleReportsPerSiteReasons(t *testing.T) {
|
|
small := site("small", node("n1", 4, 16))
|
|
nogpu := site("nogpu", node("n1", 64, 512))
|
|
|
|
r := req(8, 16, 64, spec.Constraint{Capability: model.CapGPU, Count: 2})
|
|
d := Plan([]model.Site{small, nogpu}, r, DefaultWeights())
|
|
|
|
if d.Feasible {
|
|
t.Fatal("should not be feasible")
|
|
}
|
|
if len(d.Candidates) != 2 {
|
|
t.Fatalf("want an explanation per site, got %d", len(d.Candidates))
|
|
}
|
|
for _, c := range d.Candidates {
|
|
if len(c.Reasons) == 0 {
|
|
t.Errorf("%s: refused with no reason", c.SiteID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPreferIsSoft(t *testing.T) {
|
|
withNVMe := site("nvme", node("n1", 64, 512, model.Device{
|
|
NativeID: "0000:03:00.0", Capability: model.CapNVMe,
|
|
Confidence: model.ConfClass, Passthrough: true,
|
|
}))
|
|
without := site("plain", node("n1", 64, 512))
|
|
|
|
r := req(2, 8, 32)
|
|
r.Compute.Prefer = []spec.Constraint{{Capability: model.CapNVMe, Count: 1}}
|
|
|
|
d := Plan([]model.Site{withNVMe, without}, r, DefaultWeights())
|
|
if !d.Feasible {
|
|
t.Fatal(d.Summary)
|
|
}
|
|
if d.Chosen.SiteID != "nvme" {
|
|
t.Errorf("prefer should have tipped the choice to nvme, got %s", d.Chosen.SiteID)
|
|
}
|
|
// but it must remain satisfiable without it
|
|
d2 := Plan([]model.Site{without}, r, DefaultWeights())
|
|
if !d2.Feasible {
|
|
t.Error("prefer must not act as a hard requirement")
|
|
}
|
|
}
|
|
|
|
// The head is cloned from a template that lives on one node, so it may only
|
|
// go there; compute nodes are diskless and go wherever there is room.
|
|
func TestHeadStaysOnTheTemplateNode(t *testing.T) {
|
|
// Fullest-first packing would put the head on pve2, the tighter node.
|
|
// pve1 has room for the head and one compute node, not two.
|
|
s := site("lab", node("pve1", 16, 256), node("pve2", 12, 256))
|
|
s.HeadNode = "pve1"
|
|
d := Plan([]model.Site{s}, req(2, 8, 32), DefaultWeights())
|
|
if !d.Feasible {
|
|
t.Fatalf("expected a placement, got: %s", d.Summary)
|
|
}
|
|
hosts := map[string]bool{}
|
|
for _, p := range d.Chosen.Placement {
|
|
hosts[p.HostNode] = true
|
|
if p.Role == "head" && p.HostNode != "pve1" {
|
|
t.Errorf("head on %s, want pve1, where the template is", p.HostNode)
|
|
}
|
|
}
|
|
if !hosts["pve2"] {
|
|
t.Errorf("nothing on pve2: compute nodes should still use the other node (%v)", hosts)
|
|
}
|
|
}
|
|
|
|
func TestHeadNodeWithoutRoomIsAClearRefusal(t *testing.T) {
|
|
s := site("lab", node("pve1", 2, 4), node("pve2", 64, 256))
|
|
s.HeadNode = "pve1"
|
|
d := Plan([]model.Site{s}, req(1, 8, 32), DefaultWeights())
|
|
if d.Feasible {
|
|
t.Fatal("placed a head on pve2, which has no template")
|
|
}
|
|
if !strings.Contains(d.Summary+strings.Join(d.Candidates[0].Reasons, ";"), "head template") {
|
|
t.Errorf("reason does not say why: %s %v", d.Summary, d.Candidates[0].Reasons)
|
|
}
|
|
}
|