// Package inventory holds the control plane's view of the cluster: which nodes // exist, what hardware they have, and what is already running on them. package inventory import ( "maps" "slices" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) // NodeState is where a node stands. type NodeState int const ( // Ready means the node accepts placement. Ready NodeState = iota // Cordoned means an operator has excluded it from new placement; what runs // there keeps running. Cordoned // Draining means its replicas are being moved off. Draining // Unreachable means heartbeats stopped but the node may come back. Unreachable // Lost means it has been unreachable long enough to declare its replicas dead. Lost ) func (s NodeState) String() string { switch s { case Ready: return "ready" case Cordoned: return "cordoned" case Draining: return "draining" case Unreachable: return "unreachable" case Lost: return "lost" default: return "unknown" } } // Schedulable reports whether new replicas may be placed on the node. func (s NodeState) Schedulable() bool { return s == Ready } // FabricAttachment is a node's connection to one fabric. // The koanf tags are not decoration. Without them the fields bind from their Go // names, so the documented `name:` key silently produced an empty fabric and // `rate_gbps` was never read at all — and the agent then refused to start with // "fabrics[0]: name is required", naming the key the operator had just written. // Every fabric in this project was configured in a test until the first one was // configured from a file. type FabricAttachment struct { // Fabric is the operator-assigned name. It is not derived from the GID // prefix: OpenSM's default is fe80:: everywhere, so a matching prefix // proves nothing (plan §5.2). Fabric string `json:"fabric" koanf:"name"` // HCAs are the adapters attached to this fabric. HCAs []string `json:"hcas" koanf:"hcas"` // RateGbps is the slowest active port's rate, since a collective runs at // the speed of its slowest link. RateGbps int `json:"rate_gbps" koanf:"rate_gbps"` } // Node is one machine in the cluster. type Node struct { ID string `json:"id"` Hostname string `json:"hostname"` MgmtAddr string `json:"mgmt_addr"` Labels map[string]string `json:"labels,omitempty"` // MemoryTotalMB is the machine's RAM. Zero means the agent did not report // it, which is treated as no constraint — the same as every node before // this field existed. MemoryTotalMB int `json:"memory_total_mb,omitempty"` // Weights are the model files this node can see, however they got there. // Empty means the node was not asked to look, which is not the same as // holding none — so it constrains nothing. Weights []discovery.LocalWeights `json:"weights,omitempty"` // Storage is the disk those weights sit on, one entry per filesystem. // // Empty means the node was not asked, not that it has no disk — so an // agent too old to report it, or one with no model directories, is left // unconstrained rather than treated as full. Storage []discovery.Storage `json:"storage,omitempty"` GPUs []discovery.GPU `json:"gpus"` // Fabrics is empty on a node without IB. Such a node is its own // single-node island and can never host a multi-node replica. Fabrics []FabricAttachment `json:"fabrics,omitempty"` State NodeState `json:"-"` LastSeen time.Time `json:"last_seen"` // AgentVersion and EngineVersions are reported at registration. The engine // version decides which metric names and CLI flags apply (plan §14). AgentVersion string `json:"agent_version,omitempty"` EngineVersions map[string]string `json:"engine_versions,omitempty"` // EnginesProbed says EngineVersions is an answer, so an empty map means // "has none" rather than "an agent too old to say". EnginesProbed bool `json:"engines_probed,omitempty"` // Host is the operating system and processor, as the agent reads them. Host discovery.Host `json:"host"` // Provision is where installing each wanted engine stands on this node. Provision []EngineInstall `json:"provision,omitempty"` // Load is how busy the machine was at the last heartbeat that measured // it. Nil before the first one, and from an agent too old to say. Not // persisted: it describes a moment, and a restored one would be stale. Load *Load `json:"-"` } // Load is a node's use of its CPU and memory, and each engine's share. type Load struct { At time.Time CPUPercent float64 CPUCount int Load1 float64 Load5 float64 Load15 float64 MemUsedBytes int64 MemAvailBytes int64 // Shards is each engine's use, by shard ID. Shards map[string]ShardLoad } // ShardLoad is one engine's use: cores busy, and resident memory. type ShardLoad struct { CPUCores float64 RSSBytes int64 } // EngineInstall is where one engine's installation stands on a node. type EngineInstall struct { Engine string `json:"engine"` Version string `json:"version"` Variant string `json:"variant,omitempty"` // State is "installing", "ready", "failed" or "unsupported". State string `json:"state"` Detail string `json:"detail,omitempty"` } // HasFabric reports whether the node is attached to a fabric. func (n Node) HasFabric(name string) bool { return slices.ContainsFunc(n.Fabrics, func(f FabricAttachment) bool { return f.Fabric == name }) } // FabricNames lists the node's fabrics, sorted. func (n Node) FabricNames() []string { out := make([]string, 0, len(n.Fabrics)) for _, f := range n.Fabrics { out = append(out, f.Fabric) } slices.Sort(out) return out } // HCAsFor returns the adapters attached to one fabric. // // A shard is given only these: handing it adapters from a second fabric lets // NCCL choose one whose peer is unreachable, and the job hangs instead of // failing (plan §5.3). func (n Node) HCAsFor(fabric string) []string { for _, f := range n.Fabrics { if f.Fabric == fabric { return slices.Clone(f.HCAs) } } return nil } // GPUsByClass groups the node's GPUs by class. func (n Node) GPUsByClass() map[string][]discovery.GPU { out := map[string][]discovery.GPU{} for _, g := range n.GPUs { out[g.Class()] = append(out[g.Class()], g) } return out } // Allocation records that some of a node's GPUs are in use. type Allocation struct { ShardID string ReplicaID string GPUIndices []int // MemoryGB is the host RAM this shard's profile asked for. // // Recorded for the same reason the GPU indices are: it was claimed, and // placement has to stop handing it out twice. Without it a node's memory // was compared against its total on every decision, so two models each // declaring most of a machine were both placed on it — and the second one // is what the kernel kills. // // Zero for a profile that declared nothing, which constrains nothing and // is counted as nothing. MemoryGB int } // Store holds the cluster inventory. It is safe for concurrent use. type Store struct { mu sync.RWMutex nodes map[string]*Node // allocs maps node id to the shards placed there. allocs map[string][]Allocation } // NewStore returns an empty inventory. func NewStore() *Store { return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}} } // Upsert records a node's registration, preserving operator-set state such as // a cordon: an agent reconnecting must not silently undo an operator's decision. func (s *Store) Upsert(n Node) { s.mu.Lock() defer s.mu.Unlock() if prev, ok := s.nodes[n.ID]; ok { if prev.State == Cordoned || prev.State == Draining { n.State = prev.State } } n.LastSeen = time.Now() s.nodes[n.ID] = &n } // SetState changes a node's state. func (s *Store) SetState(id string, st NodeState) bool { s.mu.Lock() defer s.mu.Unlock() n, ok := s.nodes[id] if !ok { return false } n.State = st return true } // Touch records a heartbeat, bringing an unreachable node back. func (s *Store) Touch(id string) { s.mu.Lock() defer s.mu.Unlock() n, ok := s.nodes[id] if !ok { return } n.LastSeen = time.Now() if n.State == Unreachable || n.State == Lost { n.State = Ready } } // Get returns a copy of a node. func (s *Store) Get(id string) (Node, bool) { s.mu.RLock() defer s.mu.RUnlock() n, ok := s.nodes[id] if !ok { return Node{}, false } return *n, true } // Nodes returns every node, sorted by id. func (s *Store) Nodes() []Node { s.mu.RLock() defer s.mu.RUnlock() out := make([]Node, 0, len(s.nodes)) for _, n := range s.nodes { out = append(out, *n) } slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.ID, b.ID) }) return out } // Remove forgets a node and its allocations. func (s *Store) Remove(id string) { s.mu.Lock() defer s.mu.Unlock() delete(s.nodes, id) delete(s.allocs, id) } // Allocate records GPUs as taken on a node. func (s *Store) Allocate(nodeID string, a Allocation) { s.mu.Lock() defer s.mu.Unlock() s.allocs[nodeID] = append(s.allocs[nodeID], a) } // Release frees the GPUs a shard held. func (s *Store) Release(shardID string) { s.mu.Lock() defer s.mu.Unlock() for node, list := range s.allocs { kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool { return a.ShardID == shardID }) if len(kept) == 0 { delete(s.allocs, node) } else { s.allocs[node] = kept } } } // ReleaseReplica frees every shard of a replica. // // Replicas are freed whole because they fail whole: one dead shard makes the // others useless, and the control plane rebuilds the replica rather than // restarting a piece of it (plan §5.4). func (s *Store) ReleaseReplica(replicaID string) { s.mu.Lock() defer s.mu.Unlock() for node, list := range s.allocs { kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool { return a.ReplicaID == replicaID }) if len(kept) == 0 { delete(s.allocs, node) } else { s.allocs[node] = kept } } } // Allocations returns a node's current allocations. func (s *Store) Allocations(nodeID string) []Allocation { s.mu.RLock() defer s.mu.RUnlock() return slices.Clone(s.allocs[nodeID]) } // FreeGPUs returns the indices of a node's unallocated GPUs, sorted. func (s *Store) FreeGPUs(nodeID string) []int { s.mu.RLock() defer s.mu.RUnlock() n, ok := s.nodes[nodeID] if !ok { return nil } taken := map[int]struct{}{} for _, a := range s.allocs[nodeID] { for _, i := range a.GPUIndices { taken[i] = struct{}{} } } var out []int for _, g := range n.GPUs { if _, used := taken[g.Index]; used { continue } // A GPU with an ECC fault or another process on it is not free in any // useful sense. if !g.Health.OK() { continue } out = append(out, g.Index) } slices.Sort(out) return out } // LocalDisk sums the space on filesystems this node fills itself. // // Shared filesystems are left out on purpose. Space on a filer belongs to // everyone mounting it, and counting it per node would let twenty nodes each // believe they have the same free terabyte — then place twenty models into it. // A node whose weights are all on a shared mount therefore reports no local // disk, which is correct: it has none to spend. // // Both zero means nothing is known — an agent too old to report storage, or a // node with no model directories — and callers must read that as unknown // rather than as a full disk. func (n Node) LocalDisk() (free, total int64) { for _, s := range n.Storage { if s.Shared { continue } free += s.FreeBytes total += s.TotalBytes } return free, total } // HasWeights reports whether this node can see the given source. // // An exact path match, deliberately. A source that is not a path — a model id // an engine will resolve itself — cannot be checked from here, and guessing // would be worse than the honest answer that nothing is known. func (n Node) HasWeights(source string) bool { for _, w := range n.Weights { if w.Path == source { return true } } return false } // KnowsItsWeights reports whether this node was asked to look at all. // // The distinction that keeps the feature optional: a node reporting no weights // because nobody configured a directory must not be excluded from placement, // while a node that looked and genuinely has none should be. func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 } // Fabrics lists every fabric any node is attached to, sorted. func (s *Store) Fabrics() []string { s.mu.RLock() defer s.mu.RUnlock() seen := map[string]struct{}{} for _, n := range s.nodes { for _, f := range n.Fabrics { seen[f.Fabric] = struct{}{} } } return slices.Sorted(maps.Keys(seen)) }