Phase 3 (1/3): cluster inventory and placement
CI / test (push) Successful in 23s

The two hard constraints from plan §1.1 are enforced structurally rather than
checked after the fact: there is no code path that can combine nodes from two
fabrics, and only GPUs of the profile's class are ever considered. Getting
either wrong produces a replica that starts, passes health checks, and then
hangs or crawls under load — which looks like success until traffic arrives.

- inventory: nodes, hardware, fabric attachments, allocations. A re-registering
  agent cannot clear an operator's cordon, since a cordon is usually in place
  because something is wrong with the node. A GPU with an ECC fault or another
  process on it is not free in any useful sense. Replicas are released whole,
  because they fail whole.
- placement: highest-priority profile that fits, with every rejected candidate
  explained. "Unschedulable" on its own tells an operator nothing about which
  of six constraints bit.
- bin packing fills partly used nodes first so whole nodes stay free for large
  models that cannot be split; multi-node replicas get the emptiest machines
  instead, being the least movable thing in the cluster.
- NVLink-aware GPU selection keeps a tensor-parallel group inside one domain
  where the topology is known, falling back to lowest free indices otherwise.

Property-based tests generate 400 random clusters and assert the invariants on
every placement produced: one fabric per multi-node replica, only that fabric's
HCAs per shard, one GPU class, no unhealthy or already-allocated GPU. The
table-driven tests cover the cases someone thought of; these cover the rest.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ilkerandClaude Opus 5 committed 2026-09-16 02:07:29 +02:00
1 parent 55417546fc
commit 06672d1b85
6 files changed
+1426 -2

No files matched your search

+352
View File
@@ -0,0 +1,352 @@
// Package placement decides which nodes and GPUs a replica runs on (plan §6.2).
//
// Two constraints here are not negotiable, and everything else in this package
// exists to keep them true:
//
// - a multi-node replica never spans two fabrics
// - a tensor-parallel group never mixes GPU classes
//
// Violating either produces a replica that starts, passes health checks, and
// then hangs or crawls under load — the worst possible failure mode, because it
// looks like success until traffic arrives.
package placement
import (
"fmt"
"slices"
"sort"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Profile is the subset of a catalog profile placement needs.
type Profile struct {
Name string
Engine string
GPUClass string
// GPUsPerNode is how many GPUs each node contributes.
GPUsPerNode int
// Nodes is how many nodes the replica spans. Above 1, a fabric is required.
Nodes int
// Priority orders profiles; the highest that fits is used.
Priority int
}
// normalize fills in the defaults a catalog entry may omit.
func (p Profile) normalize() Profile {
if p.Nodes <= 0 {
p.Nodes = 1
}
if p.GPUsPerNode <= 0 {
p.GPUsPerNode = 1
}
return p
}
// NodeAssignment is one node's share of a replica.
type NodeAssignment struct {
NodeID string
Rank int
// GPUIndices are the local GPUs the shard uses.
GPUIndices []int
// HCAs are the adapters of the chosen fabric only.
HCAs []string
}
// Placement is a decision: which profile, which fabric, which nodes.
type Placement struct {
Profile Profile
// Fabric is empty for a single-node replica.
Fabric string
// Pool names the (fabric, gpu class) pair this landed in.
Pool string
Nodes []NodeAssignment
}
// Rejection records why a candidate was not used. Placement failures are
// otherwise impossible to debug: "unschedulable" tells an operator nothing
// about which of six constraints bit.
type Rejection struct {
Profile string
Pool string
Node string
Reason string
}
func (r Rejection) String() string {
where := r.Pool
if r.Node != "" {
where += "/" + r.Node
}
if where == "" {
where = "-"
}
return fmt.Sprintf("%s @ %s: %s", r.Profile, where, r.Reason)
}
// Result is the outcome of a placement attempt.
type Result struct {
Placement *Placement
// Rejections explains every candidate that was considered and discarded,
// in the order they were tried. This is what `ctl explain-placement` prints.
Rejections []Rejection
}
// Explain renders the rejections as text.
func (r Result) Explain() string {
if len(r.Rejections) == 0 {
return "no candidates were considered"
}
var b strings.Builder
for _, rej := range r.Rejections {
b.WriteString(" ")
b.WriteString(rej.String())
b.WriteString("\n")
}
return b.String()
}
// Scheduler places replicas onto an inventory.
type Scheduler struct {
inv *inventory.Store
}
// New builds a scheduler over an inventory.
func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} }
// Place picks the first profile, in priority order, that fits.
//
// avoid lists nodes already hosting this model, so replicas of one model spread
// across failure domains instead of stacking on one machine.
func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
var res Result
ordered := slices.Clone(profiles)
sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].Priority > ordered[j].Priority })
for _, raw := range ordered {
p := raw.normalize()
if p.Nodes == 1 {
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
res.Placement = pl
return res
} else {
res.Rejections = append(res.Rejections, rej...)
}
continue
}
if pl, rej := s.placeMultiNode(p, avoid); pl != nil {
res.Placement = pl
return res
} else {
res.Rejections = append(res.Rejections, rej...)
}
}
return res
}
// placeSingleNode finds one node with enough free GPUs of the right class.
//
// Nodes without a fabric are eligible here and only here: a single-node replica
// never talks to another node, so it has nothing to be reachable over.
func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Rejection) {
var rejections []Rejection
candidates := s.eligibleNodes(p, "", avoid, &rejections)
if len(candidates) == 0 {
return nil, rejections
}
// Bin packing: the node with the fewest free GPUs that still fits. Filling
// partly used nodes first keeps whole nodes free for the large models that
// cannot be split.
sort.SliceStable(candidates, func(i, j int) bool {
return len(candidates[i].free) < len(candidates[j].free)
})
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode)
if gpus == nil {
rejections = append(rejections, Rejection{
Profile: p.Name, Node: best.node.ID,
Reason: fmt.Sprintf("has %d free %s GPUs but no usable set of %d", len(best.free), p.GPUClass, p.GPUsPerNode),
})
return nil, rejections
}
return &Placement{
Profile: p,
Pool: "none/" + p.GPUClass,
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
}, rejections
}
// placeMultiNode finds N nodes on a single fabric.
//
// The fabric loop is the whole point: candidates are gathered per fabric and a
// replica is only formed from within one of them. There is deliberately no path
// that combines nodes from different fabrics, however well they would otherwise
// fit.
func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rejection) {
var rejections []Rejection
fabrics := s.inv.Fabrics()
if len(fabrics) == 0 {
return nil, []Rejection{{
Profile: p.Name,
Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes),
}}
}
for _, fabric := range fabrics {
pool := fabric + "/" + p.GPUClass
candidates := s.eligibleNodes(p, fabric, avoid, &rejections)
if len(candidates) < p.Nodes {
rejections = append(rejections, Rejection{
Profile: p.Name, Pool: pool,
Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)),
})
continue
}
// Prefer the emptiest nodes for multi-node work: a replica spanning
// nodes is the least movable thing in the cluster, so give it the
// cleanest machines rather than squeezing it into the gaps.
sort.SliceStable(candidates, func(i, j int) bool {
return len(candidates[i].free) > len(candidates[j].free)
})
assignments := make([]NodeAssignment, 0, p.Nodes)
for rank, c := range candidates[:p.Nodes] {
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode)
if gpus == nil {
assignments = nil
rejections = append(rejections, Rejection{
Profile: p.Name, Pool: pool, Node: c.node.ID,
Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode),
})
break
}
assignments = append(assignments, NodeAssignment{
NodeID: c.node.ID,
Rank: rank,
GPUIndices: gpus,
// Only this fabric's adapters, never the node's others.
HCAs: c.node.HCAsFor(fabric),
})
}
if assignments == nil {
continue
}
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}, rejections
}
return nil, rejections
}
// candidate is a node that passed the hard constraints.
type candidate struct {
node inventory.Node
free []discovery.GPU
}
// eligibleNodes applies the hard constraints and records why each node failed.
//
// fabric is empty for single-node placement, in which case attachment is not
// checked at all.
func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, rejections *[]Rejection) []candidate {
pool := "none"
if fabric != "" {
pool = fabric
}
pool += "/" + p.GPUClass
var out []candidate
for _, n := range s.inv.Nodes() {
reject := func(reason string) {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: n.ID, Reason: reason})
}
if !n.State.Schedulable() {
reject("node is " + n.State.String())
continue
}
if slices.Contains(avoid, n.ID) {
reject("already hosts this model; spreading across failure domains")
continue
}
if fabric != "" {
if !n.HasFabric(fabric) {
// Not worth reporting for every node on every fabric; it is the
// expected case, not a problem.
continue
}
if len(n.HCAsFor(fabric)) == 0 {
reject("attached to " + fabric + " but reports no adapters for it")
continue
}
}
freeIdx := s.inv.FreeGPUs(n.ID)
var free []discovery.GPU
for _, g := range n.GPUs {
if g.Class() != p.GPUClass || !slices.Contains(freeIdx, g.Index) {
continue
}
free = append(free, g)
}
if len(free) < p.GPUsPerNode {
reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode))
continue
}
out = append(out, candidate{node: n, free: free})
}
return out
}
// pickGPUs chooses which of a node's free GPUs to use.
//
// It prefers a set that shares an NVLink domain: within a node, a
// tensor-parallel group that has to fall back to PCIe for its all-reduces runs
// several times slower. NVLink topology is only available from the NVML build,
// so without it this degrades to taking the lowest indices — which is usually
// right, but only by accident of how vendors number slots.
func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) []int {
if len(free) < want {
return nil
}
if want == 1 {
return []int{free[0].Index}
}
freeSet := map[int]struct{}{}
for _, g := range free {
freeSet[g.Index] = struct{}{}
}
// Try each free GPU as a seed and grow its NVLink-connected set.
for _, seed := range free {
group := []int{seed.Index}
for _, peer := range seed.NVLinkPeers {
if _, ok := freeSet[peer]; !ok || peer == seed.Index {
continue
}
group = append(group, peer)
if len(group) == want {
slices.Sort(group)
return group
}
}
}
// No NVLink domain large enough, or no topology at all.
out := make([]int, 0, want)
for _, g := range free[:want] {
out = append(out, g.Index)
}
slices.Sort(out)
return out
}
@@ -0,0 +1,483 @@
package placement
import (
"fmt"
"math/rand/v2"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// node builds a node with n GPUs of one class.
func node(id string, class string, n int, fabrics ...string) inventory.Node {
name, mem := classToHardware(class)
nd := inventory.Node{ID: id, State: inventory.Ready}
for i := range n {
nd.GPUs = append(nd.GPUs, discovery.GPU{
Index: i, Name: name, MemoryTotalMB: mem, ComputeCapability: "9.0",
UUID: fmt.Sprintf("GPU-%s-%d", id, i),
})
}
for _, f := range fabrics {
nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{
Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200,
})
}
return nd
}
// classToHardware maps a class back to a name and size that derive it.
func classToHardware(class string) (string, int) {
switch class {
case "h100-80g":
return "NVIDIA H100 80GB HBM3", 81559
case "a100-40g":
return "Tesla A100-SXM4-40GB", 40960
default:
return "NVIDIA L40S", 46068
}
}
func store(nodes ...inventory.Node) *inventory.Store {
s := inventory.NewStore()
for _, n := range nodes {
s.Upsert(n)
}
return s
}
func TestSingleNodePlacement(t *testing.T) {
s := New(store(node("n1", "h100-80g", 8)))
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if len(res.Placement.Nodes) != 1 {
t.Fatalf("expected 1 node, got %d", len(res.Placement.Nodes))
}
if got := len(res.Placement.Nodes[0].GPUIndices); got != 4 {
t.Errorf("got %d GPUs, want 4", got)
}
if res.Placement.Fabric != "" {
t.Errorf("a single-node replica was given fabric %q", res.Placement.Fabric)
}
}
func TestNodeWithoutFabricTakesSingleNodeWork(t *testing.T) {
// A single-node replica talks to nobody, so it has nothing to be reachable
// over. Excluding fabric-less nodes here would waste them entirely.
s := New(store(node("no-ib", "h100-80g", 8)))
res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("a node without IB was refused single-node work:\n%s", res.Explain())
}
}
func TestNodeWithoutFabricRefusedMultiNode(t *testing.T) {
s := New(store(
node("no-ib-1", "h100-80g", 8),
node("no-ib-2", "h100-80g", 8),
))
res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement != nil {
t.Fatal("a multi-node replica was placed on nodes with no fabric")
}
if !strings.Contains(res.Explain(), "no node reports a fabric") {
t.Errorf("the reason is unclear:\n%s", res.Explain())
}
}
// TestNeverSpansTwoFabrics is the first hard invariant from plan §1.1.
func TestNeverSpansTwoFabrics(t *testing.T) {
// Two fabrics, one node each: together they would satisfy the count, and a
// scheduler that only counted nodes would happily combine them.
s := New(store(
node("a1", "h100-80g", 8, "ib-a"),
node("b1", "h100-80g", 8, "ib-b"),
))
res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement != nil {
var fabrics []string
for _, n := range res.Placement.Nodes {
fabrics = append(fabrics, n.NodeID)
}
t.Fatalf("a replica was spread across fabrics: %v", fabrics)
}
if !strings.Contains(res.Explain(), "only 1 qualify") {
t.Errorf("explanation should say each fabric was short of nodes:\n%s", res.Explain())
}
}
func TestMultiNodeWithinOneFabric(t *testing.T) {
s := New(store(
node("a1", "h100-80g", 8, "ib-a"),
node("a2", "h100-80g", 8, "ib-a"),
node("b1", "h100-80g", 8, "ib-b"),
))
res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if res.Placement.Fabric != "ib-a" {
t.Errorf("fabric=%q, only ib-a has two nodes", res.Placement.Fabric)
}
for _, n := range res.Placement.Nodes {
if n.NodeID == "b1" {
t.Error("a node from the other fabric was included")
}
if len(n.HCAs) == 0 {
t.Errorf("node %s got no HCAs; NCCL would pick its own", n.NodeID)
}
}
// Ranks must be assigned, since rank 0 serves HTTP.
ranks := []int{res.Placement.Nodes[0].Rank, res.Placement.Nodes[1].Rank}
slices.Sort(ranks)
if ranks[0] != 0 || ranks[1] != 1 {
t.Errorf("ranks=%v", ranks)
}
}
// TestNeverMixesGPUClasses is the second hard invariant from plan §1.1.
func TestNeverMixesGPUClasses(t *testing.T) {
// Enough GPUs in total, but split across classes.
n := node("mixed", "h100-80g", 4)
a100Name, a100Mem := classToHardware("a100-40g")
for i := 4; i < 8; i++ {
n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: a100Name, MemoryTotalMB: a100Mem})
}
s := New(store(n))
res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatal("a placement used GPUs of more than one class")
}
if !strings.Contains(res.Explain(), "has 4 free h100-80g GPUs, needs 8") {
t.Errorf("explanation should count only the matching class:\n%s", res.Explain())
}
// The 4-GPU profile does fit, using only H100s.
res = s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("the matching-class profile was refused:\n%s", res.Explain())
}
for _, idx := range res.Placement.Nodes[0].GPUIndices {
if idx >= 4 {
t.Errorf("GPU %d is an A100 and was included in an h100 placement", idx)
}
}
}
func TestProfilePriorityOrder(t *testing.T) {
s := New(store(
node("h", "h100-80g", 8, "ib-a"),
node("a", "a100-40g", 8, "ib-a"),
))
res := s.Place([]Profile{
{Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50},
{Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100},
}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if res.Placement.Profile.Name != "h100" {
t.Errorf("chose %q; the higher-priority profile fits", res.Placement.Profile.Name)
}
}
func TestFallsBackToLowerPriorityProfile(t *testing.T) {
s := New(store(node("a", "a100-40g", 8)))
res := s.Place([]Profile{
{Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100},
{Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50},
}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if res.Placement.Profile.Name != "a100" {
t.Errorf("chose %q", res.Placement.Profile.Name)
}
// The rejected first choice must still be explained.
if !strings.Contains(res.Explain(), "h100") {
t.Errorf("the higher-priority profile's rejection is missing:\n%s", res.Explain())
}
}
func TestCordonedNodeExcluded(t *testing.T) {
st := store(node("n1", "h100-80g", 8))
st.SetState("n1", inventory.Cordoned)
s := New(st)
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatal("a cordoned node was used")
}
if !strings.Contains(res.Explain(), "cordoned") {
t.Errorf("explanation:\n%s", res.Explain())
}
}
func TestUnhealthyGPUsExcluded(t *testing.T) {
n := node("n1", "h100-80g", 8)
n.GPUs[0].Health.Problems = []string{"12 uncorrected ECC errors"}
n.GPUs[1].Health.Problems = []string{"40000 MiB already allocated by another process"}
s := New(store(n))
res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatal("unhealthy GPUs were counted as available")
}
res = s.Place([]Profile{{Name: "tp6", GPUClass: "h100-80g", GPUsPerNode: 6, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("the six healthy GPUs should have been usable:\n%s", res.Explain())
}
for _, idx := range res.Placement.Nodes[0].GPUIndices {
if idx == 0 || idx == 1 {
t.Errorf("unhealthy GPU %d was placed on", idx)
}
}
}
func TestAllocatedGPUsExcluded(t *testing.T) {
st := store(node("n1", "h100-80g", 8))
st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}})
s := New(st)
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("the remaining four GPUs should fit:\n%s", res.Explain())
}
for _, idx := range res.Placement.Nodes[0].GPUIndices {
if idx < 4 {
t.Errorf("GPU %d is already allocated", idx)
}
}
// Another four will not fit.
st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{4, 5, 6, 7}})
if res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil); res.Placement != nil {
t.Fatal("a full node was placed on")
}
}
func TestAvoidSpreadsAcrossNodes(t *testing.T) {
s := New(store(
node("n1", "h100-80g", 8),
node("n2", "h100-80g", 8),
))
p := []Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}
first := s.Place(p, nil)
if first.Placement == nil {
t.Fatal("no first placement")
}
placed := first.Placement.Nodes[0].NodeID
second := s.Place(p, []string{placed})
if second.Placement == nil {
t.Fatalf("the second replica had nowhere to go:\n%s", second.Explain())
}
if second.Placement.Nodes[0].NodeID == placed {
t.Error("both replicas landed on the same node despite avoid")
}
}
func TestBinPackingPrefersPartlyUsedNodes(t *testing.T) {
// Keeping whole nodes free matters: a large model that needs all 8 GPUs
// cannot be split, so scattering small replicas would strand it.
st := store(
node("empty", "h100-80g", 8),
node("partial", "h100-80g", 8),
)
st.Allocate("partial", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}})
s := New(st)
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "partial" {
t.Errorf("placed on %q; the partly used node should be filled first", got)
}
}
func TestNVLinkGroupPreferred(t *testing.T) {
// GPUs 0-3 and 4-7 form separate NVLink domains. A TP group crossing them
// falls back to PCIe for its all-reduces and runs several times slower.
n := node("n1", "h100-80g", 8)
for i := range n.GPUs {
var peers []int
base := (i / 4) * 4
for j := base; j < base+4; j++ {
if j != i {
peers = append(peers, j)
}
}
n.GPUs[i].NVLinkPeers = peers
}
st := store(n)
// Take GPU 0, so a naive "lowest indices" pick would span the two domains.
st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0}})
s := New(st)
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement == nil {
t.Fatalf("no placement:\n%s", res.Explain())
}
got := res.Placement.Nodes[0].GPUIndices
if !slices.Equal(got, []int{4, 5, 6, 7}) {
t.Errorf("chose %v; 4-7 is the only intact NVLink domain left", got)
}
}
func TestExplainIsUsefulWhenNothingFits(t *testing.T) {
s := New(store(node("n1", "a100-40g", 2)))
res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatal("unexpected placement")
}
exp := res.Explain()
// It must name the node, the profile, and what was actually wrong.
for _, want := range []string{"n1", "tp8", "free h100-80g"} {
if !strings.Contains(exp, want) {
t.Errorf("explanation is missing %q:\n%s", want, exp)
}
}
}
func TestEmptyInventory(t *testing.T) {
s := New(inventory.NewStore())
res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil)
if res.Placement != nil {
t.Fatal("something was placed on an empty cluster")
}
}
// --- property-based invariants (plan §13) ---
// TestInvariantsHold generates random clusters and profiles and checks the two
// hard constraints on every placement produced.
//
// The table-driven tests above cover the cases someone thought of; this covers
// the ones nobody did.
func TestInvariantsHold(t *testing.T) {
rng := rand.New(rand.NewPCG(1, 2))
classes := []string{"h100-80g", "a100-40g", "l40s-45g"}
fabrics := []string{"ib-a", "ib-b", "ib-c"}
for iter := range 400 {
st := inventory.NewStore()
nodeCount := 1 + rng.IntN(6)
for i := range nodeCount {
class := classes[rng.IntN(len(classes))]
gpuCount := 1 + rng.IntN(8)
var nf []string
switch rng.IntN(4) {
case 0: // no fabric at all
case 1:
nf = []string{fabrics[rng.IntN(len(fabrics))]}
default:
nf = []string{fabrics[rng.IntN(len(fabrics))]}
if rng.IntN(3) == 0 {
// A node attached to two fabrics: allowed, and the reason
// HCAs must be selected per fabric rather than per node.
nf = append(nf, fabrics[rng.IntN(len(fabrics))])
}
}
n := node(fmt.Sprintf("n%d", i), class, gpuCount, nf...)
if rng.IntN(8) == 0 {
n.State = inventory.Cordoned
}
st.Upsert(n)
}
s := New(st)
p := Profile{
Name: "gen",
GPUClass: classes[rng.IntN(len(classes))],
GPUsPerNode: 1 + rng.IntN(8),
Nodes: 1 + rng.IntN(3),
}
res := s.Place([]Profile{p}, nil)
if res.Placement == nil {
continue
}
checkInvariants(t, st, p, res.Placement, iter)
}
}
func checkInvariants(t *testing.T, st *inventory.Store, p Profile, pl *Placement, iter int) {
t.Helper()
if len(pl.Nodes) != max(p.Nodes, 1) {
t.Fatalf("iter %d: placed on %d nodes, profile wants %d", iter, len(pl.Nodes), p.Nodes)
}
seenNodes := map[string]struct{}{}
for _, a := range pl.Nodes {
if _, dup := seenNodes[a.NodeID]; dup {
t.Fatalf("iter %d: node %s used twice in one replica", iter, a.NodeID)
}
seenNodes[a.NodeID] = struct{}{}
n, ok := st.Get(a.NodeID)
if !ok {
t.Fatalf("iter %d: placed on unknown node %s", iter, a.NodeID)
}
if !n.State.Schedulable() {
t.Fatalf("iter %d: placed on a %s node", iter, n.State)
}
if len(a.GPUIndices) != p.GPUsPerNode {
t.Fatalf("iter %d: node %s got %d GPUs, wants %d", iter, a.NodeID, len(a.GPUIndices), p.GPUsPerNode)
}
// Invariant: every GPU is of the profile's class, healthy, and real.
byIndex := map[int]discovery.GPU{}
for _, g := range n.GPUs {
byIndex[g.Index] = g
}
for _, idx := range a.GPUIndices {
g, ok := byIndex[idx]
if !ok {
t.Fatalf("iter %d: node %s has no GPU %d", iter, a.NodeID, idx)
}
if g.Class() != p.GPUClass {
t.Fatalf("iter %d: GPU %d on %s is %s, profile wants %s",
iter, idx, a.NodeID, g.Class(), p.GPUClass)
}
if !g.Health.OK() {
t.Fatalf("iter %d: unhealthy GPU %d was placed on", iter, idx)
}
}
// Invariant: a multi-node replica lives on one fabric, and each shard
// gets only that fabric's adapters.
if p.Nodes > 1 {
if pl.Fabric == "" {
t.Fatalf("iter %d: multi-node placement with no fabric", iter)
}
if !n.HasFabric(pl.Fabric) {
t.Fatalf("iter %d: node %s is not attached to %s", iter, a.NodeID, pl.Fabric)
}
want := n.HCAsFor(pl.Fabric)
if !slices.Equal(a.HCAs, want) {
t.Fatalf("iter %d: node %s got HCAs %v, want exactly %v for %s",
iter, a.NodeID, a.HCAs, want, pl.Fabric)
}
}
}
}