Files
llmgw/internal/control/inventory/inventory.go
T
Ilker ManapandClaude Opus 5 c13c567cc5
CI / test (push) Successful in 1m32s
Know where the weights are, without owning the storage
Storage will be different at every installation — Lustre, NFS over RDMA, an
image exported with SRP or NVMe-oF, local NVMe, an rsync — so llmgw observes
it rather than managing it. An agent told which directories to look in
reports what it can see; how the files got there is nobody's business here,
and a node told nothing reports nothing and behaves exactly as every node
did before this existed.

That fact buys two things. Placement prefers a node that already holds the
model over one that must first move tens of gigabytes, as a preference and
never a filter: fetching them is a valid answer, and refusing to place would
turn a slow start into no start at all. And ctl status shows what each
fabric holds.

Per fabric because fabrics are not connected to one another. Weights in
ib-a are not weights in ib-b — the only path between them is the management
network — so a model seeded into one fabric leaves the rest of the cluster
unable to serve it without a slow copy, and nothing said so before now. One
shared filer counts the same as twenty local copies, because "can this
fabric serve it" is the question being asked.

Three distinctions the tests hold to, because each is a way to get this
quietly wrong. A node that was never asked to look must rank with one that
looked and has nothing, or enabling the feature on a single node would
drain every other. An empty directory is not weights, because a
half-finished download looks exactly like one and would send placement to a
node whose engine then fails to load. And local storage is never mistaken
for shared: a filer read as local costs an unnecessary copy, while local
read as shared would have the cluster believe a model is everywhere when it
is on one node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 08:21:16 +02:00

347 lines
9.4 KiB
Go

// Package inventory holds the control plane's view of the cluster: which nodes
// exist, what hardware they have, and what is already running on them.
package inventory
import (
"maps"
"slices"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// NodeState is where a node stands.
type NodeState int
const (
// Ready means the node accepts placement.
Ready NodeState = iota
// Cordoned means an operator has excluded it from new placement; what runs
// there keeps running.
Cordoned
// Draining means its replicas are being moved off.
Draining
// Unreachable means heartbeats stopped but the node may come back.
Unreachable
// Lost means it has been unreachable long enough to declare its replicas dead.
Lost
)
func (s NodeState) String() string {
switch s {
case Ready:
return "ready"
case Cordoned:
return "cordoned"
case Draining:
return "draining"
case Unreachable:
return "unreachable"
case Lost:
return "lost"
default:
return "unknown"
}
}
// Schedulable reports whether new replicas may be placed on the node.
func (s NodeState) Schedulable() bool { return s == Ready }
// FabricAttachment is a node's connection to one fabric.
// The koanf tags are not decoration. Without them the fields bind from their Go
// names, so the documented `name:` key silently produced an empty fabric and
// `rate_gbps` was never read at all — and the agent then refused to start with
// "fabrics[0]: name is required", naming the key the operator had just written.
// Every fabric in this project was configured in a test until the first one was
// configured from a file.
type FabricAttachment struct {
// Fabric is the operator-assigned name. It is not derived from the GID
// prefix: OpenSM's default is fe80:: everywhere, so a matching prefix
// proves nothing (plan §5.2).
Fabric string `json:"fabric" koanf:"name"`
// HCAs are the adapters attached to this fabric.
HCAs []string `json:"hcas" koanf:"hcas"`
// RateGbps is the slowest active port's rate, since a collective runs at
// the speed of its slowest link.
RateGbps int `json:"rate_gbps" koanf:"rate_gbps"`
}
// Node is one machine in the cluster.
type Node struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
MgmtAddr string `json:"mgmt_addr"`
Labels map[string]string `json:"labels,omitempty"`
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
// it, which is treated as no constraint — the same as every node before
// this field existed.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
// Weights are the model files this node can see, however they got there.
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
// single-node island and can never host a multi-node replica.
Fabrics []FabricAttachment `json:"fabrics,omitempty"`
State NodeState `json:"-"`
LastSeen time.Time `json:"last_seen"`
// AgentVersion and EngineVersions are reported at registration. The engine
// version decides which metric names and CLI flags apply (plan §14).
AgentVersion string `json:"agent_version,omitempty"`
EngineVersions map[string]string `json:"engine_versions,omitempty"`
}
// HasFabric reports whether the node is attached to a fabric.
func (n Node) HasFabric(name string) bool {
return slices.ContainsFunc(n.Fabrics, func(f FabricAttachment) bool { return f.Fabric == name })
}
// FabricNames lists the node's fabrics, sorted.
func (n Node) FabricNames() []string {
out := make([]string, 0, len(n.Fabrics))
for _, f := range n.Fabrics {
out = append(out, f.Fabric)
}
slices.Sort(out)
return out
}
// HCAsFor returns the adapters attached to one fabric.
//
// A shard is given only these: handing it adapters from a second fabric lets
// NCCL choose one whose peer is unreachable, and the job hangs instead of
// failing (plan §5.3).
func (n Node) HCAsFor(fabric string) []string {
for _, f := range n.Fabrics {
if f.Fabric == fabric {
return slices.Clone(f.HCAs)
}
}
return nil
}
// GPUsByClass groups the node's GPUs by class.
func (n Node) GPUsByClass() map[string][]discovery.GPU {
out := map[string][]discovery.GPU{}
for _, g := range n.GPUs {
out[g.Class()] = append(out[g.Class()], g)
}
return out
}
// Allocation records that some of a node's GPUs are in use.
type Allocation struct {
ShardID string
ReplicaID string
GPUIndices []int
}
// Store holds the cluster inventory. It is safe for concurrent use.
type Store struct {
mu sync.RWMutex
nodes map[string]*Node
// allocs maps node id to the shards placed there.
allocs map[string][]Allocation
}
// NewStore returns an empty inventory.
func NewStore() *Store {
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
}
// Upsert records a node's registration, preserving operator-set state such as
// a cordon: an agent reconnecting must not silently undo an operator's decision.
func (s *Store) Upsert(n Node) {
s.mu.Lock()
defer s.mu.Unlock()
if prev, ok := s.nodes[n.ID]; ok {
if prev.State == Cordoned || prev.State == Draining {
n.State = prev.State
}
}
n.LastSeen = time.Now()
s.nodes[n.ID] = &n
}
// SetState changes a node's state.
func (s *Store) SetState(id string, st NodeState) bool {
s.mu.Lock()
defer s.mu.Unlock()
n, ok := s.nodes[id]
if !ok {
return false
}
n.State = st
return true
}
// Touch records a heartbeat, bringing an unreachable node back.
func (s *Store) Touch(id string) {
s.mu.Lock()
defer s.mu.Unlock()
n, ok := s.nodes[id]
if !ok {
return
}
n.LastSeen = time.Now()
if n.State == Unreachable || n.State == Lost {
n.State = Ready
}
}
// Get returns a copy of a node.
func (s *Store) Get(id string) (Node, bool) {
s.mu.RLock()
defer s.mu.RUnlock()
n, ok := s.nodes[id]
if !ok {
return Node{}, false
}
return *n, true
}
// Nodes returns every node, sorted by id.
func (s *Store) Nodes() []Node {
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]Node, 0, len(s.nodes))
for _, n := range s.nodes {
out = append(out, *n)
}
slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.ID, b.ID) })
return out
}
// Remove forgets a node and its allocations.
func (s *Store) Remove(id string) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.nodes, id)
delete(s.allocs, id)
}
// Allocate records GPUs as taken on a node.
func (s *Store) Allocate(nodeID string, a Allocation) {
s.mu.Lock()
defer s.mu.Unlock()
s.allocs[nodeID] = append(s.allocs[nodeID], a)
}
// Release frees the GPUs a shard held.
func (s *Store) Release(shardID string) {
s.mu.Lock()
defer s.mu.Unlock()
for node, list := range s.allocs {
kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool {
return a.ShardID == shardID
})
if len(kept) == 0 {
delete(s.allocs, node)
} else {
s.allocs[node] = kept
}
}
}
// ReleaseReplica frees every shard of a replica.
//
// Replicas are freed whole because they fail whole: one dead shard makes the
// others useless, and the control plane rebuilds the replica rather than
// restarting a piece of it (plan §5.4).
func (s *Store) ReleaseReplica(replicaID string) {
s.mu.Lock()
defer s.mu.Unlock()
for node, list := range s.allocs {
kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool {
return a.ReplicaID == replicaID
})
if len(kept) == 0 {
delete(s.allocs, node)
} else {
s.allocs[node] = kept
}
}
}
// Allocations returns a node's current allocations.
func (s *Store) Allocations(nodeID string) []Allocation {
s.mu.RLock()
defer s.mu.RUnlock()
return slices.Clone(s.allocs[nodeID])
}
// FreeGPUs returns the indices of a node's unallocated GPUs, sorted.
func (s *Store) FreeGPUs(nodeID string) []int {
s.mu.RLock()
defer s.mu.RUnlock()
n, ok := s.nodes[nodeID]
if !ok {
return nil
}
taken := map[int]struct{}{}
for _, a := range s.allocs[nodeID] {
for _, i := range a.GPUIndices {
taken[i] = struct{}{}
}
}
var out []int
for _, g := range n.GPUs {
if _, used := taken[g.Index]; used {
continue
}
// A GPU with an ECC fault or another process on it is not free in any
// useful sense.
if !g.Health.OK() {
continue
}
out = append(out, g.Index)
}
slices.Sort(out)
return out
}
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
// an engine will resolve itself — cannot be checked from here, and guessing
// would be worse than the honest answer that nothing is known.
func (n Node) HasWeights(source string) bool {
for _, w := range n.Weights {
if w.Path == source {
return true
}
}
return false
}
// KnowsItsWeights reports whether this node was asked to look at all.
//
// The distinction that keeps the feature optional: a node reporting no weights
// because nobody configured a directory must not be excluded from placement,
// while a node that looked and genuinely has none should be.
func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 }
// Fabrics lists every fabric any node is attached to, sorted.
func (s *Store) Fabrics() []string {
s.mu.RLock()
defer s.mu.RUnlock()
seen := map[string]struct{}{}
for _, n := range s.nodes {
for _, f := range n.Fabrics {
seen[f.Fabric] = struct{}{}
}
}
return slices.Sorted(maps.Keys(seen))
}