llmgw/internal/control/inventory/inventory.go

// Package inventory holds the control plane's view of the cluster: which nodes
// exist, what hardware they have, and what is already running on them.
package inventory

import (
	"maps"
	"slices"
	"strings"
	"sync"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)

// NodeState is where a node stands.
type NodeState int

const (
	// Ready means the node accepts placement.
	Ready NodeState = iota
	// Cordoned means an operator has excluded it from new placement; what runs
	// there keeps running.
	Cordoned
	// Draining means its replicas are being moved off.
	Draining
	// Unreachable means heartbeats stopped but the node may come back.
	Unreachable
	// Lost means it has been unreachable long enough to declare its replicas dead.
	Lost
)

func (s NodeState) String() string {
	switch s {
	case Ready:
		return "ready"
	case Cordoned:
		return "cordoned"
	case Draining:
		return "draining"
	case Unreachable:
		return "unreachable"
	case Lost:
		return "lost"
	default:
		return "unknown"
	}
}

// Schedulable reports whether new replicas may be placed on the node.
func (s NodeState) Schedulable() bool { return s == Ready }

// FabricAttachment is a node's connection to one fabric.
// The koanf tags are not decoration. Without them the fields bind from their Go
// names, so the documented `name:` key silently produced an empty fabric and
// `rate_gbps` was never read at all — and the agent then refused to start with
// "fabrics[0]: name is required", naming the key the operator had just written.
// Every fabric in this project was configured in a test until the first one was
// configured from a file.
type FabricAttachment struct {
	// Fabric is the operator-assigned name. It is not derived from the GID
	// prefix: OpenSM's default is fe80:: everywhere, so a matching prefix
	// proves nothing (plan §5.2).
	Fabric string `json:"fabric" koanf:"name"`
	// HCAs are the adapters attached to this fabric.
	HCAs []string `json:"hcas" koanf:"hcas"`
	// RateGbps is the slowest active port's rate, since a collective runs at
	// the speed of its slowest link.
	RateGbps int `json:"rate_gbps" koanf:"rate_gbps"`
}

// Node is one machine in the cluster.
type Node struct {
	ID       string            `json:"id"`
	Hostname string            `json:"hostname"`
	MgmtAddr string            `json:"mgmt_addr"`
	Labels   map[string]string `json:"labels,omitempty"`

	// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
	// it, which is treated as no constraint — the same as every node before
	// this field existed.
	MemoryTotalMB int `json:"memory_total_mb,omitempty"`

	// Weights are the model files this node can see, however they got there.
	// Empty means the node was not asked to look, which is not the same as
	// holding none — so it constrains nothing.
	Weights []discovery.LocalWeights `json:"weights,omitempty"`
	// Storage is the disk those weights sit on, one entry per filesystem.
	//
	// Empty means the node was not asked, not that it has no disk — so an
	// agent too old to report it, or one with no model directories, is left
	// unconstrained rather than treated as full.
	Storage []discovery.Storage `json:"storage,omitempty"`

	GPUs []discovery.GPU `json:"gpus"`
	// Fabrics is empty on a node without IB. Such a node is its own
	// single-node island and can never host a multi-node replica.
	Fabrics []FabricAttachment `json:"fabrics,omitempty"`

	State    NodeState `json:"-"`
	LastSeen time.Time `json:"last_seen"`
	// AgentVersion and EngineVersions are reported at registration. The engine
	// version decides which metric names and CLI flags apply (plan §14).
	AgentVersion   string            `json:"agent_version,omitempty"`
	EngineVersions map[string]string `json:"engine_versions,omitempty"`
	// EnginesProbed says EngineVersions is an answer, so an empty map means
	// "has none" rather than "an agent too old to say".
	EnginesProbed bool `json:"engines_probed,omitempty"`

	// Host is the operating system and processor, as the agent reads them.
	Host discovery.Host `json:"host"`
	// Provision is where installing each wanted engine stands on this node.
	Provision []EngineInstall `json:"provision,omitempty"`

	// Load is how busy the machine was at the last heartbeat that measured
	// it. Nil before the first one, and from an agent too old to say. Not
	// persisted: it describes a moment, and a restored one would be stale.
	Load *Load `json:"-"`
}

// Load is a node's use of its CPU and memory, and each engine's share.
type Load struct {
	At            time.Time
	CPUPercent    float64
	CPUCount      int
	Load1         float64
	Load5         float64
	Load15        float64
	MemUsedBytes  int64
	MemAvailBytes int64
	// Shards is each engine's use, by shard ID.
	Shards map[string]ShardLoad
}

// ShardLoad is one engine's use: cores busy, and resident memory.
type ShardLoad struct {
	CPUCores float64
	RSSBytes int64
}

// EngineInstall is where one engine's installation stands on a node.
type EngineInstall struct {
	Engine  string `json:"engine"`
	Version string `json:"version"`
	Variant string `json:"variant,omitempty"`
	// State is "installing", "ready", "failed" or "unsupported".
	State  string `json:"state"`
	Detail string `json:"detail,omitempty"`
}

// HasFabric reports whether the node is attached to a fabric.
func (n Node) HasFabric(name string) bool {
	return slices.ContainsFunc(n.Fabrics, func(f FabricAttachment) bool { return f.Fabric == name })
}

// FabricNames lists the node's fabrics, sorted.
func (n Node) FabricNames() []string {
	out := make([]string, 0, len(n.Fabrics))
	for _, f := range n.Fabrics {
		out = append(out, f.Fabric)
	}
	slices.Sort(out)
	return out
}

// HCAsFor returns the adapters attached to one fabric.
//
// A shard is given only these: handing it adapters from a second fabric lets
// NCCL choose one whose peer is unreachable, and the job hangs instead of
// failing (plan §5.3).
func (n Node) HCAsFor(fabric string) []string {
	for _, f := range n.Fabrics {
		if f.Fabric == fabric {
			return slices.Clone(f.HCAs)
		}
	}
	return nil
}

// GPUsByClass groups the node's GPUs by class.
func (n Node) GPUsByClass() map[string][]discovery.GPU {
	out := map[string][]discovery.GPU{}
	for _, g := range n.GPUs {
		out[g.Class()] = append(out[g.Class()], g)
	}
	return out
}

// Allocation records that some of a node's GPUs are in use.
type Allocation struct {
	ShardID    string
	ReplicaID  string
	GPUIndices []int
	// MemoryGB is the host RAM this shard's profile asked for.
	//
	// Recorded for the same reason the GPU indices are: it was claimed, and
	// placement has to stop handing it out twice. Without it a node's memory
	// was compared against its total on every decision, so two models each
	// declaring most of a machine were both placed on it — and the second one
	// is what the kernel kills.
	//
	// Zero for a profile that declared nothing, which constrains nothing and
	// is counted as nothing.
	MemoryGB int
}

// Store holds the cluster inventory. It is safe for concurrent use.
type Store struct {
	mu    sync.RWMutex
	nodes map[string]*Node
	// allocs maps node id to the shards placed there.
	allocs map[string][]Allocation
}

// NewStore returns an empty inventory.
func NewStore() *Store {
	return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
}

// Upsert records a node's registration, preserving operator-set state such as
// a cordon: an agent reconnecting must not silently undo an operator's decision.
func (s *Store) Upsert(n Node) {
	s.mu.Lock()
	defer s.mu.Unlock()

	if prev, ok := s.nodes[n.ID]; ok {
		if prev.State == Cordoned || prev.State == Draining {
			n.State = prev.State
		}
	}
	n.LastSeen = time.Now()
	s.nodes[n.ID] = &n
}

// SetState changes a node's state.
func (s *Store) SetState(id string, st NodeState) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	n, ok := s.nodes[id]
	if !ok {
		return false
	}
	n.State = st
	return true
}

// Touch records a heartbeat, bringing an unreachable node back.
func (s *Store) Touch(id string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	n, ok := s.nodes[id]
	if !ok {
		return
	}
	n.LastSeen = time.Now()
	if n.State == Unreachable || n.State == Lost {
		n.State = Ready
	}
}

// Get returns a copy of a node.
func (s *Store) Get(id string) (Node, bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()
	n, ok := s.nodes[id]
	if !ok {
		return Node{}, false
	}
	return *n, true
}

// Nodes returns every node, sorted by id.
func (s *Store) Nodes() []Node {
	s.mu.RLock()
	defer s.mu.RUnlock()
	out := make([]Node, 0, len(s.nodes))
	for _, n := range s.nodes {
		out = append(out, *n)
	}
	slices.SortFunc(out, func(a, b Node) int { return strings.Compare(a.ID, b.ID) })
	return out
}

// Remove forgets a node and its allocations.
func (s *Store) Remove(id string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	delete(s.nodes, id)
	delete(s.allocs, id)
}

// Allocate records GPUs as taken on a node.
func (s *Store) Allocate(nodeID string, a Allocation) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.allocs[nodeID] = append(s.allocs[nodeID], a)
}

// Release frees the GPUs a shard held.
func (s *Store) Release(shardID string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	for node, list := range s.allocs {
		kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool {
			return a.ShardID == shardID
		})
		if len(kept) == 0 {
			delete(s.allocs, node)
		} else {
			s.allocs[node] = kept
		}
	}
}

// ReleaseReplica frees every shard of a replica.
//
// Replicas are freed whole because they fail whole: one dead shard makes the
// others useless, and the control plane rebuilds the replica rather than
// restarting a piece of it (plan §5.4).
func (s *Store) ReleaseReplica(replicaID string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	for node, list := range s.allocs {
		kept := slices.DeleteFunc(slices.Clone(list), func(a Allocation) bool {
			return a.ReplicaID == replicaID
		})
		if len(kept) == 0 {
			delete(s.allocs, node)
		} else {
			s.allocs[node] = kept
		}
	}
}

// Allocations returns a node's current allocations.
func (s *Store) Allocations(nodeID string) []Allocation {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return slices.Clone(s.allocs[nodeID])
}

// FreeGPUs returns the indices of a node's unallocated GPUs, sorted.
func (s *Store) FreeGPUs(nodeID string) []int {
	s.mu.RLock()
	defer s.mu.RUnlock()

	n, ok := s.nodes[nodeID]
	if !ok {
		return nil
	}
	taken := map[int]struct{}{}
	for _, a := range s.allocs[nodeID] {
		for _, i := range a.GPUIndices {
			taken[i] = struct{}{}
		}
	}

	var out []int
	for _, g := range n.GPUs {
		if _, used := taken[g.Index]; used {
			continue
		}
		// A GPU with an ECC fault or another process on it is not free in any
		// useful sense.
		if !g.Health.OK() {
			continue
		}
		out = append(out, g.Index)
	}
	slices.Sort(out)
	return out
}

// LocalDisk sums the space on filesystems this node fills itself.
//
// Shared filesystems are left out on purpose. Space on a filer belongs to
// everyone mounting it, and counting it per node would let twenty nodes each
// believe they have the same free terabyte — then place twenty models into it.
// A node whose weights are all on a shared mount therefore reports no local
// disk, which is correct: it has none to spend.
//
// Both zero means nothing is known — an agent too old to report storage, or a
// node with no model directories — and callers must read that as unknown
// rather than as a full disk.
func (n Node) LocalDisk() (free, total int64) {
	for _, s := range n.Storage {
		if s.Shared {
			continue
		}
		free += s.FreeBytes
		total += s.TotalBytes
	}
	return free, total
}

// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
// an engine will resolve itself — cannot be checked from here, and guessing
// would be worse than the honest answer that nothing is known.
func (n Node) HasWeights(source string) bool {
	for _, w := range n.Weights {
		if w.Path == source {
			return true
		}
	}
	return false
}

// KnowsItsWeights reports whether this node was asked to look at all.
//
// The distinction that keeps the feature optional: a node reporting no weights
// because nobody configured a directory must not be excluded from placement,
// while a node that looked and genuinely has none should be.
func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 }

// Fabrics lists every fabric any node is attached to, sorted.
func (s *Store) Fabrics() []string {
	s.mu.RLock()
	defer s.mu.RUnlock()

	seen := map[string]struct{}{}
	for _, n := range s.nodes {
		for _, f := range n.Fabrics {
			seen[f.Fabric] = struct{}{}
		}
	}
	return slices.Sorted(maps.Keys(seen))
}