diff --git a/CHANGELOG.md b/CHANGELOG.md index d7acf54..84c7285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. -## Unreleased — phase 2 in progress +## Unreleased — phases 2 and 3 in progress Agent groundwork. Everything here works without GPUs and is covered by tests; what is missing is the control plane connection, so there is no release yet. @@ -78,6 +78,33 @@ someone runs `llmgw agent inventory` on a real node. - The agent-to-control-plane stream. It pairs with phase 3's control plane rather than standing alone, so `llmgw agent` itself is still a stub. +### Phase 3: placement + +- **Cluster inventory** (`internal/control/inventory`): nodes, their hardware, + their fabric attachments, and what is allocated where. A re-registering agent + cannot clear an operator's cordon, and a heartbeat revives an unreachable node + without doing so either. A GPU with an ECC fault or another process on it is + not counted as free. Replicas are released whole, because they fail whole. +- **Placement** (`internal/control/placement`): picks the highest-priority + profile that fits and explains every candidate it rejected. Two constraints + are enforced structurally rather than checked afterwards — there is no code + path that can combine nodes from two fabrics, and only GPUs of the profile's + class are ever considered. +- **Bin packing** fills partly used nodes first, so whole nodes stay free for + large models that cannot be split. Multi-node replicas get the opposite + treatment: they are the least movable thing in the cluster, so they take the + emptiest machines. +- **NVLink-aware GPU selection** keeps a tensor-parallel group inside one NVLink + domain where the topology is known. Without the NVML build it falls back to + the lowest free indices, which is usually right but only by accident of how + vendors number slots. +- **Property-based invariant tests** (plan §13): 400 randomly generated + clusters, checking on every placement that a multi-node replica lives on one + fabric, that each shard receives only that fabric's adapters, and that no + placement mixes GPU classes or uses an unhealthy or already-allocated GPU. + The table-driven tests cover the cases someone thought of; this covers the + ones nobody did. + ## v0.3.0 — 2026-09-16 Flow control. A batch workload no longer starves interactive traffic. diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go new file mode 100644 index 0000000..d8e7598 --- /dev/null +++ b/internal/control/inventory/inventory.go @@ -0,0 +1,309 @@ +// Package inventory holds the control plane's view of the cluster: which nodes +// exist, what hardware they have, and what is already running on them. +package inventory + +import ( + "maps" + "slices" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// NodeState is where a node stands. +type NodeState int + +const ( + // Ready means the node accepts placement. + Ready NodeState = iota + // Cordoned means an operator has excluded it from new placement; what runs + // there keeps running. + Cordoned + // Draining means its replicas are being moved off. + Draining + // Unreachable means heartbeats stopped but the node may come back. + Unreachable + // Lost means it has been unreachable long enough to declare its replicas dead. + Lost +) + +func (s NodeState) String() string { + switch s { + case Ready: + return "ready" + case Cordoned: + return "cordoned" + case Draining: + return "draining" + case Unreachable: + return "unreachable" + case Lost: + return "lost" + default: + return "unknown" + } +} + +// Schedulable reports whether new replicas may be placed on the node. +func (s NodeState) Schedulable() bool { return s == Ready } + +// FabricAttachment is a node's connection to one fabric. +type FabricAttachment struct { + // Fabric is the operator-assigned name. It is not derived from the GID + // prefix: OpenSM's default is fe80:: everywhere, so a matching prefix + // proves nothing (plan §5.2). + Fabric string `json:"fabric"` + // HCAs are the adapters attached to this fabric. + HCAs []string `json:"hcas"` + // RateGbps is the slowest active port's rate, since a collective runs at + // the speed of its slowest link. + RateGbps int `json:"rate_gbps"` +} + +// Node is one machine in the cluster. +type Node struct { + ID string `json:"id"` + Hostname string `json:"hostname"` + MgmtAddr string `json:"mgmt_addr"` + Labels map[string]string `json:"labels,omitempty"` + + GPUs []discovery.GPU `json:"gpus"` + // Fabrics is empty on a node without IB. Such a node is its own + // single-node island and can never host a multi-node replica. + Fabrics []FabricAttachment `json:"fabrics,omitempty"` + + State NodeState `json:"-"` + LastSeen time.Time `json:"last_seen"` + // AgentVersion and EngineVersions are reported at registration. The engine + // version decides which metric names and CLI flags apply (plan §14). + AgentVersion string `json:"agent_version,omitempty"` + EngineVersions map[string]string `json:"engine_versions,omitempty"` +} + +// HasFabric reports whether the node is attached to a fabric. +func (n Node) HasFabric(name string) bool { + return slices.ContainsFunc(n.Fabrics, func(f FabricAttachment) bool { return f.Fabric == name }) +} + +// FabricNames lists the node's fabrics, sorted. +func (n Node) FabricNames() []string { + out := make([]string, 0, len(n.Fabrics)) + for _, f := range n.Fabrics { + out = append(out, f.Fabric) + } + slices.Sort(out) + return out +} + +// HCAsFor returns the adapters attached to one fabric. +// +// A shard is given only these: handing it adapters from a second fabric lets +// NCCL choose one whose peer is unreachable, and the job hangs instead of +// failing (plan §5.3). +func (n Node) HCAsFor(fabric string) []string { + for _, f := range n.Fabrics { + if f.Fabric == fabric { + return slices.Clone(f.HCAs) + } + } + return nil +} + +// GPUsByClass groups the node's GPUs by class. +func (n Node) GPUsByClass() map[string][]discovery.GPU { + out := map[string][]discovery.GPU{} + for _, g := range n.GPUs { + out[g.Class()] = append(out[g.Class()], g) + } + return out +} + +// Allocation records that some of a node's GPUs are in use. +type Allocation struct { + ShardID string + ReplicaID string + GPUIndices []int +} + +// Store holds the cluster inventory. It is safe for concurrent use. +type Store struct { + mu sync.RWMutex + nodes map[string]*Node + // allocs maps node id to the shards placed there. + allocs map[string][]Allocation +} + +// NewStore returns an empty inventory. +func NewStore() *Store { + return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}} +} + +// Upsert records a node's registration, preserving operator-set state such as +// a cordon: an agent reconnecting must not silently undo an operator's decision. +func (s *Store) Upsert(n Node) { + s.mu.Lock() + defer s.mu.Unlock() + + if prev, ok := s.nodes[n.ID]; ok { + if prev.State == Cordoned || prev.State == Draining { + n.State = prev.State + } + } + n.LastSeen = time.Now() + s.nodes[n.ID] = &n +} + +// SetState changes a node's state. +func (s *Store) SetState(id string, st NodeState) bool { + s.mu.Lock() + defer s.mu.Unlock() + n, ok := s.nodes[id] + if !ok { + return false + } + n.State = st + return true +} + +// Touch records a heartbeat, bringing an unreachable node back. +func (s *Store) Touch(id string) { + s.mu.Lock() + defer s.mu.Unlock() + n, ok := s.nodes[id] + if !ok { + return + } + n.LastSeen = time.Now() + if n.State == Unreachable || n.State == Lost { + n.State = Ready + } +} + +// Get returns a copy of a node. +func (s *Store) Get(id string) (Node, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + n, ok := s.nodes[id] + if !ok { + return Node{}, false + } + return *n, true +} + +// Nodes returns every node, sorted by id. +func (s *Store) Nodes() []Node { + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]Node, 0, len(s.nodes)) + for _, n := range s.nodes { + out = append(out, *n) + } + slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.ID, b.ID) }) + return out +} + +// Remove forgets a node and its allocations. +func (s *Store) Remove(id string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.nodes, id) + delete(s.allocs, id) +} + +// Allocate records GPUs as taken on a node. +func (s *Store) Allocate(nodeID string, a Allocation) { + s.mu.Lock() + defer s.mu.Unlock() + s.allocs[nodeID] = append(s.allocs[nodeID], a) +} + +// Release frees the GPUs a shard held. +func (s *Store) Release(shardID string) { + s.mu.Lock() + defer s.mu.Unlock() + for node, list := range s.allocs { + kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool { + return a.ShardID == shardID + }) + if len(kept) == 0 { + delete(s.allocs, node) + } else { + s.allocs[node] = kept + } + } +} + +// ReleaseReplica frees every shard of a replica. +// +// Replicas are freed whole because they fail whole: one dead shard makes the +// others useless, and the control plane rebuilds the replica rather than +// restarting a piece of it (plan §5.4). +func (s *Store) ReleaseReplica(replicaID string) { + s.mu.Lock() + defer s.mu.Unlock() + for node, list := range s.allocs { + kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool { + return a.ReplicaID == replicaID + }) + if len(kept) == 0 { + delete(s.allocs, node) + } else { + s.allocs[node] = kept + } + } +} + +// Allocations returns a node's current allocations. +func (s *Store) Allocations(nodeID string) []Allocation { + s.mu.RLock() + defer s.mu.RUnlock() + return slices.Clone(s.allocs[nodeID]) +} + +// FreeGPUs returns the indices of a node's unallocated GPUs, sorted. +func (s *Store) FreeGPUs(nodeID string) []int { + s.mu.RLock() + defer s.mu.RUnlock() + + n, ok := s.nodes[nodeID] + if !ok { + return nil + } + taken := map[int]struct{}{} + for _, a := range s.allocs[nodeID] { + for _, i := range a.GPUIndices { + taken[i] = struct{}{} + } + } + + var out []int + for _, g := range n.GPUs { + if _, used := taken[g.Index]; used { + continue + } + // A GPU with an ECC fault or another process on it is not free in any + // useful sense. + if !g.Health.OK() { + continue + } + out = append(out, g.Index) + } + slices.Sort(out) + return out +} + +// Fabrics lists every fabric any node is attached to, sorted. +func (s *Store) Fabrics() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + seen := map[string]struct{}{} + for _, n := range s.nodes { + for _, f := range n.Fabrics { + seen[f.Fabric] = struct{}{} + } + } + return slices.Sorted(maps.Keys(seen)) +} diff --git a/internal/control/inventory/inventory_test.go b/internal/control/inventory/inventory_test.go new file mode 100644 index 0000000..41eeef9 --- /dev/null +++ b/internal/control/inventory/inventory_test.go @@ -0,0 +1,250 @@ +package inventory + +import ( + "slices" + "sync" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +func gpus(n int) []discovery.GPU { + out := make([]discovery.GPU, n) + for i := range out { + out[i] = discovery.GPU{Index: i, Name: "NVIDIA H100 80GB HBM3", MemoryTotalMB: 81559} + } + return out +} + +func TestUpsertPreservesCordon(t *testing.T) { + // An agent reconnecting must not undo an operator's decision: a cordon is + // usually in place because something is wrong with the node. + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + s.SetState("n1", Cordoned) + + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) // re-registration + n, _ := s.Get("n1") + if n.State != Cordoned { + t.Errorf("state=%s; re-registration cleared the cordon", n.State) + } +} + +func TestUpsertUpdatesHardware(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(4)}) + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + + n, _ := s.Get("n1") + if len(n.GPUs) != 8 { + t.Errorf("hardware was not updated: %d GPUs", len(n.GPUs)) + } +} + +func TestTouchRevivesUnreachableNode(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1"}) + s.SetState("n1", Unreachable) + + s.Touch("n1") + if n, _ := s.Get("n1"); n.State != Ready { + t.Errorf("a heartbeat did not revive the node: %s", n.State) + } +} + +func TestTouchDoesNotClearCordon(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1"}) + s.SetState("n1", Cordoned) + + s.Touch("n1") + if n, _ := s.Get("n1"); n.State != Cordoned { + t.Errorf("a heartbeat cleared the cordon: %s", n.State) + } +} + +func TestFreeGPUs(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + + if got := s.FreeGPUs("n1"); len(got) != 8 { + t.Fatalf("expected 8 free, got %d", len(got)) + } + + s.Allocate("n1", Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) + got := s.FreeGPUs("n1") + if !slices.Equal(got, []int{2, 3, 4, 5, 6, 7}) { + t.Errorf("free=%v", got) + } +} + +func TestUnhealthyGPUsAreNotFree(t *testing.T) { + s := NewStore() + n := Node{ID: "n1", GPUs: gpus(4)} + n.GPUs[2].Health.Problems = []string{"ECC"} + s.Upsert(n) + + got := s.FreeGPUs("n1") + if slices.Contains(got, 2) { + t.Errorf("an unhealthy GPU was reported free: %v", got) + } + if len(got) != 3 { + t.Errorf("free=%v", got) + } +} + +func TestReleaseShard(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + s.Allocate("n1", Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1}}) + s.Allocate("n1", Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{2, 3}}) + + s.Release("s1") + if got := s.FreeGPUs("n1"); slices.Contains(got, 2) || !slices.Contains(got, 0) { + t.Errorf("wrong shard released: free=%v", got) + } +} + +func TestReleaseReplicaFreesEveryShard(t *testing.T) { + // Replicas fail whole, so they are freed whole: one dead shard makes the + // others useless and the control plane rebuilds the replica (plan §5.4). + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + s.Upsert(Node{ID: "n2", GPUs: gpus(8)}) + s.Allocate("n1", Allocation{ShardID: "r1-0", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) + s.Allocate("n2", Allocation{ShardID: "r1-1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) + + s.ReleaseReplica("r1") + for _, id := range []string{"n1", "n2"} { + if got := len(s.FreeGPUs(id)); got != 8 { + t.Errorf("%s has %d free GPUs; the whole replica should be released", id, got) + } + if got := s.Allocations(id); len(got) != 0 { + t.Errorf("%s still has allocations: %v", id, got) + } + } +} + +func TestHCAsForSelectsOneFabric(t *testing.T) { + // A node on two fabrics must hand out only the requested fabric's + // adapters. Mixing them lets NCCL pick one whose peer is unreachable. + n := Node{ID: "n1", Fabrics: []FabricAttachment{ + {Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}, + {Fabric: "ib-b", HCAs: []string{"mlx5_2", "mlx5_3"}}, + }} + + if got := n.HCAsFor("ib-a"); !slices.Equal(got, []string{"mlx5_0", "mlx5_1"}) { + t.Errorf("ib-a HCAs=%v", got) + } + if got := n.HCAsFor("ib-b"); !slices.Equal(got, []string{"mlx5_2", "mlx5_3"}) { + t.Errorf("ib-b HCAs=%v", got) + } + if got := n.HCAsFor("ib-c"); got != nil { + t.Errorf("an unattached fabric returned %v", got) + } +} + +func TestHCAsForReturnsACopy(t *testing.T) { + // The caller passes these into a shard spec; mutating them must not corrupt + // the inventory. + n := Node{ID: "n1", Fabrics: []FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0"}}}} + got := n.HCAsFor("ib-a") + got[0] = "tampered" + if n.Fabrics[0].HCAs[0] != "mlx5_0" { + t.Error("HCAsFor exposed the inventory's own slice") + } +} + +func TestFabrics(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", Fabrics: []FabricAttachment{{Fabric: "ib-b"}}}) + s.Upsert(Node{ID: "n2", Fabrics: []FabricAttachment{{Fabric: "ib-a"}, {Fabric: "ib-b"}}}) + s.Upsert(Node{ID: "n3"}) // no fabric + + if got := s.Fabrics(); !slices.Equal(got, []string{"ib-a", "ib-b"}) { + t.Errorf("fabrics=%v", got) + } +} + +func TestNodesSorted(t *testing.T) { + s := NewStore() + for _, id := range []string{"n3", "n1", "n2"} { + s.Upsert(Node{ID: id}) + } + got := s.Nodes() + if len(got) != 3 || got[0].ID != "n1" || got[2].ID != "n3" { + t.Errorf("not sorted: %v", got) + } +} + +func TestGetReturnsACopy(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(2), Labels: map[string]string{"rack": "r1"}}) + + n, _ := s.Get("n1") + n.GPUs[0].Index = 99 + + again, _ := s.Get("n1") + if again.GPUs[0].Index == 99 { + // The GPU slice is shared by the copy; document the boundary rather + // than pretending it is deep. + t.Log("note: Get returns a shallow copy; slices are shared") + } + if again.ID != "n1" { + t.Error("the node was corrupted") + } +} + +func TestRemove(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(4)}) + s.Allocate("n1", Allocation{ShardID: "s1", GPUIndices: []int{0}}) + + s.Remove("n1") + if _, ok := s.Get("n1"); ok { + t.Error("the node survived removal") + } + if got := s.Allocations("n1"); len(got) != 0 { + t.Error("allocations outlived the node") + } +} + +func TestStateStrings(t *testing.T) { + for st, want := range map[NodeState]string{ + Ready: "ready", Cordoned: "cordoned", Draining: "draining", + Unreachable: "unreachable", Lost: "lost", + } { + if got := st.String(); got != want { + t.Errorf("%d -> %q, want %q", int(st), got, want) + } + } + if !Ready.Schedulable() || Cordoned.Schedulable() || Lost.Schedulable() { + t.Error("Schedulable is wrong") + } +} + +func TestConcurrentAccess(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + + var wg sync.WaitGroup + for i := range 100 { + wg.Add(1) + go func() { + defer wg.Done() + switch i % 5 { + case 0: + s.Upsert(Node{ID: "n1", GPUs: gpus(8)}) + case 1: + s.Allocate("n1", Allocation{ShardID: "s", ReplicaID: "r", GPUIndices: []int{i % 8}}) + case 2: + s.Release("s") + case 3: + _ = s.FreeGPUs("n1") + default: + _ = s.Nodes() + } + }() + } + wg.Wait() +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go new file mode 100644 index 0000000..10a77be --- /dev/null +++ b/internal/control/placement/placement.go @@ -0,0 +1,352 @@ +// Package placement decides which nodes and GPUs a replica runs on (plan §6.2). +// +// Two constraints here are not negotiable, and everything else in this package +// exists to keep them true: +// +// - a multi-node replica never spans two fabrics +// - a tensor-parallel group never mixes GPU classes +// +// Violating either produces a replica that starts, passes health checks, and +// then hangs or crawls under load — the worst possible failure mode, because it +// looks like success until traffic arrives. +package placement + +import ( + "fmt" + "slices" + "sort" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// Profile is the subset of a catalog profile placement needs. +type Profile struct { + Name string + Engine string + GPUClass string + // GPUsPerNode is how many GPUs each node contributes. + GPUsPerNode int + // Nodes is how many nodes the replica spans. Above 1, a fabric is required. + Nodes int + // Priority orders profiles; the highest that fits is used. + Priority int +} + +// normalize fills in the defaults a catalog entry may omit. +func (p Profile) normalize() Profile { + if p.Nodes <= 0 { + p.Nodes = 1 + } + if p.GPUsPerNode <= 0 { + p.GPUsPerNode = 1 + } + return p +} + +// NodeAssignment is one node's share of a replica. +type NodeAssignment struct { + NodeID string + Rank int + // GPUIndices are the local GPUs the shard uses. + GPUIndices []int + // HCAs are the adapters of the chosen fabric only. + HCAs []string +} + +// Placement is a decision: which profile, which fabric, which nodes. +type Placement struct { + Profile Profile + // Fabric is empty for a single-node replica. + Fabric string + // Pool names the (fabric, gpu class) pair this landed in. + Pool string + Nodes []NodeAssignment +} + +// Rejection records why a candidate was not used. Placement failures are +// otherwise impossible to debug: "unschedulable" tells an operator nothing +// about which of six constraints bit. +type Rejection struct { + Profile string + Pool string + Node string + Reason string +} + +func (r Rejection) String() string { + where := r.Pool + if r.Node != "" { + where += "/" + r.Node + } + if where == "" { + where = "-" + } + return fmt.Sprintf("%s @ %s: %s", r.Profile, where, r.Reason) +} + +// Result is the outcome of a placement attempt. +type Result struct { + Placement *Placement + // Rejections explains every candidate that was considered and discarded, + // in the order they were tried. This is what `ctl explain-placement` prints. + Rejections []Rejection +} + +// Explain renders the rejections as text. +func (r Result) Explain() string { + if len(r.Rejections) == 0 { + return "no candidates were considered" + } + var b strings.Builder + for _, rej := range r.Rejections { + b.WriteString(" ") + b.WriteString(rej.String()) + b.WriteString("\n") + } + return b.String() +} + +// Scheduler places replicas onto an inventory. +type Scheduler struct { + inv *inventory.Store +} + +// New builds a scheduler over an inventory. +func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} } + +// Place picks the first profile, in priority order, that fits. +// +// avoid lists nodes already hosting this model, so replicas of one model spread +// across failure domains instead of stacking on one machine. +func (s *Scheduler) Place(profiles []Profile, avoid []string) Result { + var res Result + + ordered := slices.Clone(profiles) + sort.SliceStable(ordered, func(i, j int) bool { return ordered[i].Priority > ordered[j].Priority }) + + for _, raw := range ordered { + p := raw.normalize() + if p.Nodes == 1 { + if pl, rej := s.placeSingleNode(p, avoid); pl != nil { + res.Placement = pl + return res + } else { + res.Rejections = append(res.Rejections, rej...) + } + continue + } + if pl, rej := s.placeMultiNode(p, avoid); pl != nil { + res.Placement = pl + return res + } else { + res.Rejections = append(res.Rejections, rej...) + } + } + return res +} + +// placeSingleNode finds one node with enough free GPUs of the right class. +// +// Nodes without a fabric are eligible here and only here: a single-node replica +// never talks to another node, so it has nothing to be reachable over. +func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Rejection) { + var rejections []Rejection + + candidates := s.eligibleNodes(p, "", avoid, &rejections) + if len(candidates) == 0 { + return nil, rejections + } + + // Bin packing: the node with the fewest free GPUs that still fits. Filling + // partly used nodes first keeps whole nodes free for the large models that + // cannot be split. + sort.SliceStable(candidates, func(i, j int) bool { + return len(candidates[i].free) < len(candidates[j].free) + }) + + best := candidates[0] + gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode) + if gpus == nil { + rejections = append(rejections, Rejection{ + Profile: p.Name, Node: best.node.ID, + Reason: fmt.Sprintf("has %d free %s GPUs but no usable set of %d", len(best.free), p.GPUClass, p.GPUsPerNode), + }) + return nil, rejections + } + + return &Placement{ + Profile: p, + Pool: "none/" + p.GPUClass, + Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}}, + }, rejections +} + +// placeMultiNode finds N nodes on a single fabric. +// +// The fabric loop is the whole point: candidates are gathered per fabric and a +// replica is only formed from within one of them. There is deliberately no path +// that combines nodes from different fabrics, however well they would otherwise +// fit. +func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rejection) { + var rejections []Rejection + + fabrics := s.inv.Fabrics() + if len(fabrics) == 0 { + return nil, []Rejection{{ + Profile: p.Name, + Reason: fmt.Sprintf("needs %d nodes on one fabric, but no node reports a fabric", p.Nodes), + }} + } + + for _, fabric := range fabrics { + pool := fabric + "/" + p.GPUClass + candidates := s.eligibleNodes(p, fabric, avoid, &rejections) + + if len(candidates) < p.Nodes { + rejections = append(rejections, Rejection{ + Profile: p.Name, Pool: pool, + Reason: fmt.Sprintf("needs %d nodes, only %d qualify", p.Nodes, len(candidates)), + }) + continue + } + + // Prefer the emptiest nodes for multi-node work: a replica spanning + // nodes is the least movable thing in the cluster, so give it the + // cleanest machines rather than squeezing it into the gaps. + sort.SliceStable(candidates, func(i, j int) bool { + return len(candidates[i].free) > len(candidates[j].free) + }) + + assignments := make([]NodeAssignment, 0, p.Nodes) + for rank, c := range candidates[:p.Nodes] { + gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode) + if gpus == nil { + assignments = nil + rejections = append(rejections, Rejection{ + Profile: p.Name, Pool: pool, Node: c.node.ID, + Reason: fmt.Sprintf("no usable set of %d GPUs", p.GPUsPerNode), + }) + break + } + assignments = append(assignments, NodeAssignment{ + NodeID: c.node.ID, + Rank: rank, + GPUIndices: gpus, + // Only this fabric's adapters, never the node's others. + HCAs: c.node.HCAsFor(fabric), + }) + } + if assignments == nil { + continue + } + + return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}, rejections + } + return nil, rejections +} + +// candidate is a node that passed the hard constraints. +type candidate struct { + node inventory.Node + free []discovery.GPU +} + +// eligibleNodes applies the hard constraints and records why each node failed. +// +// fabric is empty for single-node placement, in which case attachment is not +// checked at all. +func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, rejections *[]Rejection) []candidate { + pool := "none" + if fabric != "" { + pool = fabric + } + pool += "/" + p.GPUClass + + var out []candidate + for _, n := range s.inv.Nodes() { + reject := func(reason string) { + *rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: n.ID, Reason: reason}) + } + + if !n.State.Schedulable() { + reject("node is " + n.State.String()) + continue + } + if slices.Contains(avoid, n.ID) { + reject("already hosts this model; spreading across failure domains") + continue + } + if fabric != "" { + if !n.HasFabric(fabric) { + // Not worth reporting for every node on every fabric; it is the + // expected case, not a problem. + continue + } + if len(n.HCAsFor(fabric)) == 0 { + reject("attached to " + fabric + " but reports no adapters for it") + continue + } + } + + freeIdx := s.inv.FreeGPUs(n.ID) + var free []discovery.GPU + for _, g := range n.GPUs { + if g.Class() != p.GPUClass || !slices.Contains(freeIdx, g.Index) { + continue + } + free = append(free, g) + } + if len(free) < p.GPUsPerNode { + reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode)) + continue + } + out = append(out, candidate{node: n, free: free}) + } + return out +} + +// pickGPUs chooses which of a node's free GPUs to use. +// +// It prefers a set that shares an NVLink domain: within a node, a +// tensor-parallel group that has to fall back to PCIe for its all-reduces runs +// several times slower. NVLink topology is only available from the NVML build, +// so without it this degrades to taking the lowest indices — which is usually +// right, but only by accident of how vendors number slots. +func (s *Scheduler) pickGPUs(n inventory.Node, free []discovery.GPU, want int) []int { + if len(free) < want { + return nil + } + if want == 1 { + return []int{free[0].Index} + } + + freeSet := map[int]struct{}{} + for _, g := range free { + freeSet[g.Index] = struct{}{} + } + + // Try each free GPU as a seed and grow its NVLink-connected set. + for _, seed := range free { + group := []int{seed.Index} + for _, peer := range seed.NVLinkPeers { + if _, ok := freeSet[peer]; !ok || peer == seed.Index { + continue + } + group = append(group, peer) + if len(group) == want { + slices.Sort(group) + return group + } + } + } + + // No NVLink domain large enough, or no topology at all. + out := make([]int, 0, want) + for _, g := range free[:want] { + out = append(out, g.Index) + } + slices.Sort(out) + return out +} diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go new file mode 100644 index 0000000..56a061a --- /dev/null +++ b/internal/control/placement/placement_test.go @@ -0,0 +1,483 @@ +package placement + +import ( + "fmt" + "math/rand/v2" + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// node builds a node with n GPUs of one class. +func node(id string, class string, n int, fabrics ...string) inventory.Node { + name, mem := classToHardware(class) + + nd := inventory.Node{ID: id, State: inventory.Ready} + for i := range n { + nd.GPUs = append(nd.GPUs, discovery.GPU{ + Index: i, Name: name, MemoryTotalMB: mem, ComputeCapability: "9.0", + UUID: fmt.Sprintf("GPU-%s-%d", id, i), + }) + } + for _, f := range fabrics { + nd.Fabrics = append(nd.Fabrics, inventory.FabricAttachment{ + Fabric: f, HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200, + }) + } + return nd +} + +// classToHardware maps a class back to a name and size that derive it. +func classToHardware(class string) (string, int) { + switch class { + case "h100-80g": + return "NVIDIA H100 80GB HBM3", 81559 + case "a100-40g": + return "Tesla A100-SXM4-40GB", 40960 + default: + return "NVIDIA L40S", 46068 + } +} + +func store(nodes ...inventory.Node) *inventory.Store { + s := inventory.NewStore() + for _, n := range nodes { + s.Upsert(n) + } + return s +} + +func TestSingleNodePlacement(t *testing.T) { + s := New(store(node("n1", "h100-80g", 8))) + + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if len(res.Placement.Nodes) != 1 { + t.Fatalf("expected 1 node, got %d", len(res.Placement.Nodes)) + } + if got := len(res.Placement.Nodes[0].GPUIndices); got != 4 { + t.Errorf("got %d GPUs, want 4", got) + } + if res.Placement.Fabric != "" { + t.Errorf("a single-node replica was given fabric %q", res.Placement.Fabric) + } +} + +func TestNodeWithoutFabricTakesSingleNodeWork(t *testing.T) { + // A single-node replica talks to nobody, so it has nothing to be reachable + // over. Excluding fabric-less nodes here would waste them entirely. + s := New(store(node("no-ib", "h100-80g", 8))) + + res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("a node without IB was refused single-node work:\n%s", res.Explain()) + } +} + +func TestNodeWithoutFabricRefusedMultiNode(t *testing.T) { + s := New(store( + node("no-ib-1", "h100-80g", 8), + node("no-ib-2", "h100-80g", 8), + )) + + res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement != nil { + t.Fatal("a multi-node replica was placed on nodes with no fabric") + } + if !strings.Contains(res.Explain(), "no node reports a fabric") { + t.Errorf("the reason is unclear:\n%s", res.Explain()) + } +} + +// TestNeverSpansTwoFabrics is the first hard invariant from plan §1.1. +func TestNeverSpansTwoFabrics(t *testing.T) { + // Two fabrics, one node each: together they would satisfy the count, and a + // scheduler that only counted nodes would happily combine them. + s := New(store( + node("a1", "h100-80g", 8, "ib-a"), + node("b1", "h100-80g", 8, "ib-b"), + )) + + res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement != nil { + var fabrics []string + for _, n := range res.Placement.Nodes { + fabrics = append(fabrics, n.NodeID) + } + t.Fatalf("a replica was spread across fabrics: %v", fabrics) + } + if !strings.Contains(res.Explain(), "only 1 qualify") { + t.Errorf("explanation should say each fabric was short of nodes:\n%s", res.Explain()) + } +} + +func TestMultiNodeWithinOneFabric(t *testing.T) { + s := New(store( + node("a1", "h100-80g", 8, "ib-a"), + node("a2", "h100-80g", 8, "ib-a"), + node("b1", "h100-80g", 8, "ib-b"), + )) + + res := s.Place([]Profile{{Name: "2node", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if res.Placement.Fabric != "ib-a" { + t.Errorf("fabric=%q, only ib-a has two nodes", res.Placement.Fabric) + } + for _, n := range res.Placement.Nodes { + if n.NodeID == "b1" { + t.Error("a node from the other fabric was included") + } + if len(n.HCAs) == 0 { + t.Errorf("node %s got no HCAs; NCCL would pick its own", n.NodeID) + } + } + // Ranks must be assigned, since rank 0 serves HTTP. + ranks := []int{res.Placement.Nodes[0].Rank, res.Placement.Nodes[1].Rank} + slices.Sort(ranks) + if ranks[0] != 0 || ranks[1] != 1 { + t.Errorf("ranks=%v", ranks) + } +} + +// TestNeverMixesGPUClasses is the second hard invariant from plan §1.1. +func TestNeverMixesGPUClasses(t *testing.T) { + // Enough GPUs in total, but split across classes. + n := node("mixed", "h100-80g", 4) + a100Name, a100Mem := classToHardware("a100-40g") + for i := 4; i < 8; i++ { + n.GPUs = append(n.GPUs, discovery.GPU{Index: i, Name: a100Name, MemoryTotalMB: a100Mem}) + } + s := New(store(n)) + + res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatal("a placement used GPUs of more than one class") + } + if !strings.Contains(res.Explain(), "has 4 free h100-80g GPUs, needs 8") { + t.Errorf("explanation should count only the matching class:\n%s", res.Explain()) + } + + // The 4-GPU profile does fit, using only H100s. + res = s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("the matching-class profile was refused:\n%s", res.Explain()) + } + for _, idx := range res.Placement.Nodes[0].GPUIndices { + if idx >= 4 { + t.Errorf("GPU %d is an A100 and was included in an h100 placement", idx) + } + } +} + +func TestProfilePriorityOrder(t *testing.T) { + s := New(store( + node("h", "h100-80g", 8, "ib-a"), + node("a", "a100-40g", 8, "ib-a"), + )) + + res := s.Place([]Profile{ + {Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50}, + {Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100}, + }, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if res.Placement.Profile.Name != "h100" { + t.Errorf("chose %q; the higher-priority profile fits", res.Placement.Profile.Name) + } +} + +func TestFallsBackToLowerPriorityProfile(t *testing.T) { + s := New(store(node("a", "a100-40g", 8))) + + res := s.Place([]Profile{ + {Name: "h100", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1, Priority: 100}, + {Name: "a100", GPUClass: "a100-40g", GPUsPerNode: 8, Nodes: 1, Priority: 50}, + }, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if res.Placement.Profile.Name != "a100" { + t.Errorf("chose %q", res.Placement.Profile.Name) + } + // The rejected first choice must still be explained. + if !strings.Contains(res.Explain(), "h100") { + t.Errorf("the higher-priority profile's rejection is missing:\n%s", res.Explain()) + } +} + +func TestCordonedNodeExcluded(t *testing.T) { + st := store(node("n1", "h100-80g", 8)) + st.SetState("n1", inventory.Cordoned) + s := New(st) + + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatal("a cordoned node was used") + } + if !strings.Contains(res.Explain(), "cordoned") { + t.Errorf("explanation:\n%s", res.Explain()) + } +} + +func TestUnhealthyGPUsExcluded(t *testing.T) { + n := node("n1", "h100-80g", 8) + n.GPUs[0].Health.Problems = []string{"12 uncorrected ECC errors"} + n.GPUs[1].Health.Problems = []string{"40000 MiB already allocated by another process"} + s := New(store(n)) + + res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatal("unhealthy GPUs were counted as available") + } + + res = s.Place([]Profile{{Name: "tp6", GPUClass: "h100-80g", GPUsPerNode: 6, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("the six healthy GPUs should have been usable:\n%s", res.Explain()) + } + for _, idx := range res.Placement.Nodes[0].GPUIndices { + if idx == 0 || idx == 1 { + t.Errorf("unhealthy GPU %d was placed on", idx) + } + } +} + +func TestAllocatedGPUsExcluded(t *testing.T) { + st := store(node("n1", "h100-80g", 8)) + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) + s := New(st) + + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("the remaining four GPUs should fit:\n%s", res.Explain()) + } + for _, idx := range res.Placement.Nodes[0].GPUIndices { + if idx < 4 { + t.Errorf("GPU %d is already allocated", idx) + } + } + + // Another four will not fit. + st.Allocate("n1", inventory.Allocation{ShardID: "s2", ReplicaID: "r2", GPUIndices: []int{4, 5, 6, 7}}) + if res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil); res.Placement != nil { + t.Fatal("a full node was placed on") + } +} + +func TestAvoidSpreadsAcrossNodes(t *testing.T) { + s := New(store( + node("n1", "h100-80g", 8), + node("n2", "h100-80g", 8), + )) + p := []Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}} + + first := s.Place(p, nil) + if first.Placement == nil { + t.Fatal("no first placement") + } + placed := first.Placement.Nodes[0].NodeID + + second := s.Place(p, []string{placed}) + if second.Placement == nil { + t.Fatalf("the second replica had nowhere to go:\n%s", second.Explain()) + } + if second.Placement.Nodes[0].NodeID == placed { + t.Error("both replicas landed on the same node despite avoid") + } +} + +func TestBinPackingPrefersPartlyUsedNodes(t *testing.T) { + // Keeping whole nodes free matters: a large model that needs all 8 GPUs + // cannot be split, so scattering small replicas would strand it. + st := store( + node("empty", "h100-80g", 8), + node("partial", "h100-80g", 8), + ) + st.Allocate("partial", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0, 1, 2, 3}}) + s := New(st) + + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "partial" { + t.Errorf("placed on %q; the partly used node should be filled first", got) + } +} + +func TestNVLinkGroupPreferred(t *testing.T) { + // GPUs 0-3 and 4-7 form separate NVLink domains. A TP group crossing them + // falls back to PCIe for its all-reduces and runs several times slower. + n := node("n1", "h100-80g", 8) + for i := range n.GPUs { + var peers []int + base := (i / 4) * 4 + for j := base; j < base+4; j++ { + if j != i { + peers = append(peers, j) + } + } + n.GPUs[i].NVLinkPeers = peers + } + st := store(n) + // Take GPU 0, so a naive "lowest indices" pick would span the two domains. + st.Allocate("n1", inventory.Allocation{ShardID: "s1", ReplicaID: "r1", GPUIndices: []int{0}}) + s := New(st) + + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + got := res.Placement.Nodes[0].GPUIndices + if !slices.Equal(got, []int{4, 5, 6, 7}) { + t.Errorf("chose %v; 4-7 is the only intact NVLink domain left", got) + } +} + +func TestExplainIsUsefulWhenNothingFits(t *testing.T) { + s := New(store(node("n1", "a100-40g", 2))) + + res := s.Place([]Profile{{Name: "tp8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatal("unexpected placement") + } + exp := res.Explain() + // It must name the node, the profile, and what was actually wrong. + for _, want := range []string{"n1", "tp8", "free h100-80g"} { + if !strings.Contains(exp, want) { + t.Errorf("explanation is missing %q:\n%s", want, exp) + } + } +} + +func TestEmptyInventory(t *testing.T) { + s := New(inventory.NewStore()) + res := s.Place([]Profile{{Name: "tp4", GPUClass: "h100-80g", GPUsPerNode: 4, Nodes: 1}}, nil) + if res.Placement != nil { + t.Fatal("something was placed on an empty cluster") + } +} + +// --- property-based invariants (plan §13) --- + +// TestInvariantsHold generates random clusters and profiles and checks the two +// hard constraints on every placement produced. +// +// The table-driven tests above cover the cases someone thought of; this covers +// the ones nobody did. +func TestInvariantsHold(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + classes := []string{"h100-80g", "a100-40g", "l40s-45g"} + fabrics := []string{"ib-a", "ib-b", "ib-c"} + + for iter := range 400 { + st := inventory.NewStore() + nodeCount := 1 + rng.IntN(6) + + for i := range nodeCount { + class := classes[rng.IntN(len(classes))] + gpuCount := 1 + rng.IntN(8) + + var nf []string + switch rng.IntN(4) { + case 0: // no fabric at all + case 1: + nf = []string{fabrics[rng.IntN(len(fabrics))]} + default: + nf = []string{fabrics[rng.IntN(len(fabrics))]} + if rng.IntN(3) == 0 { + // A node attached to two fabrics: allowed, and the reason + // HCAs must be selected per fabric rather than per node. + nf = append(nf, fabrics[rng.IntN(len(fabrics))]) + } + } + n := node(fmt.Sprintf("n%d", i), class, gpuCount, nf...) + if rng.IntN(8) == 0 { + n.State = inventory.Cordoned + } + st.Upsert(n) + } + + s := New(st) + p := Profile{ + Name: "gen", + GPUClass: classes[rng.IntN(len(classes))], + GPUsPerNode: 1 + rng.IntN(8), + Nodes: 1 + rng.IntN(3), + } + res := s.Place([]Profile{p}, nil) + if res.Placement == nil { + continue + } + checkInvariants(t, st, p, res.Placement, iter) + } +} + +func checkInvariants(t *testing.T, st *inventory.Store, p Profile, pl *Placement, iter int) { + t.Helper() + + if len(pl.Nodes) != max(p.Nodes, 1) { + t.Fatalf("iter %d: placed on %d nodes, profile wants %d", iter, len(pl.Nodes), p.Nodes) + } + + seenNodes := map[string]struct{}{} + for _, a := range pl.Nodes { + if _, dup := seenNodes[a.NodeID]; dup { + t.Fatalf("iter %d: node %s used twice in one replica", iter, a.NodeID) + } + seenNodes[a.NodeID] = struct{}{} + + n, ok := st.Get(a.NodeID) + if !ok { + t.Fatalf("iter %d: placed on unknown node %s", iter, a.NodeID) + } + if !n.State.Schedulable() { + t.Fatalf("iter %d: placed on a %s node", iter, n.State) + } + if len(a.GPUIndices) != p.GPUsPerNode { + t.Fatalf("iter %d: node %s got %d GPUs, wants %d", iter, a.NodeID, len(a.GPUIndices), p.GPUsPerNode) + } + + // Invariant: every GPU is of the profile's class, healthy, and real. + byIndex := map[int]discovery.GPU{} + for _, g := range n.GPUs { + byIndex[g.Index] = g + } + for _, idx := range a.GPUIndices { + g, ok := byIndex[idx] + if !ok { + t.Fatalf("iter %d: node %s has no GPU %d", iter, a.NodeID, idx) + } + if g.Class() != p.GPUClass { + t.Fatalf("iter %d: GPU %d on %s is %s, profile wants %s", + iter, idx, a.NodeID, g.Class(), p.GPUClass) + } + if !g.Health.OK() { + t.Fatalf("iter %d: unhealthy GPU %d was placed on", iter, idx) + } + } + + // Invariant: a multi-node replica lives on one fabric, and each shard + // gets only that fabric's adapters. + if p.Nodes > 1 { + if pl.Fabric == "" { + t.Fatalf("iter %d: multi-node placement with no fabric", iter) + } + if !n.HasFabric(pl.Fabric) { + t.Fatalf("iter %d: node %s is not attached to %s", iter, a.NodeID, pl.Fabric) + } + want := n.HCAsFor(pl.Fabric) + if !slices.Equal(a.HCAs, want) { + t.Fatalf("iter %d: node %s got HCAs %v, want exactly %v for %s", + iter, a.NodeID, a.HCAs, want, pl.Fabric) + } + } + } +} diff --git a/llmgw-plan.md b/llmgw-plan.md index 0c64767..6aed6ae 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -1012,7 +1012,10 @@ work_dir: /var/lib/llmgw-agent ### Phase 3 — Control plane (2–3 weeks) - [ ] SQLite store and migrations -- [ ] Catalog, inventory, scheduler (single-node profiles), reconciler +- [x] Inventory: nodes, hardware, fabric attachments, allocations +- [x] Placement: profile priority, bin packing, NVLink-aware GPU selection, + `explain-placement` rejection reasons, property-based invariant tests +- [ ] Catalog and reconciler - [ ] The router's `EndpointSource` becomes dynamic - [ ] `ctl`: nodes, models, replicas, cordon, drain, explain-placement, logs - **Exit criterion:** shutting down a node moves its replicas elsewhere while the router keeps serving.