Let a node say how much room it has left
CI / test (push) Failing after 1m31s

Disk was the one resource nothing reported. `statfs` was already being called
on every model directory, but only to ask whether the filesystem was a network
one — the free space it returns in the same struct was thrown away. So the
only way to find out why a node was filling up was to ssh to it, which is what
this system exists to avoid.

Reported per filesystem rather than per configured directory. Free space
belongs to the disk, not the path: two model directories on one disk are one
pool of bytes, and counting them separately would let the control plane add
24 GB to 24 GB and place a 40 GB model on a node that has 24.

Shared filesystems are summed separately from local ones and left out of a
node's own total. Space on a filer belongs to everyone mounting it, so twenty
nodes would otherwise each report the same free terabyte, and a scheduler
would place twenty models into room for one.

Nothing reported means nothing is known, everywhere: an agent too old to send
storage and a node with no model directories both send nothing, and both must
read as unknown rather than as a full disk. `ctl nodes` and the dashboard show
a dash, the metric is absent rather than zero, and a missing directory — which
is what an unmounted filer looks like — is skipped rather than reported as
having no space.

Shown as free out of total, because the first is useless without the second:
8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root
partition, and only the second stops a model being placed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-18 09:38:57 +02:00
1 parent b5177b70f6
commit 753f4e3265
13 files changed
+515 -83

No files matched your search

+19 -3
View File
@@ -62,9 +62,21 @@ type NodeView struct {
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
Models []string `json:"models_on_disk,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
Models []string `json:"models_on_disk,omitempty"`
// DiskFreeGB and DiskTotalGB are the filesystems holding those weights,
// summed. Zero means the agent did not report any, which an older agent
// and a node with no model directories both do — so it reads as unknown
// rather than as a full disk.
//
// Summed because an operator asking "can this node take another model"
// wants one number; the per-filesystem detail is in the node's own record
// for anyone who needs it. Shared filesystems are left out of the sum: the
// space on a filer is not this node's to spend, and adding it would let a
// fleet of twenty nodes each believe it has the same free terabyte.
DiskFreeGB int `json:"disk_free_gb,omitempty"`
DiskTotalGB int `json:"disk_total_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -166,6 +178,10 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
if free, total := n.LocalDisk(); total > 0 {
v.DiskFreeGB = int(free / (1 << 30))
v.DiskTotalGB = int(total / (1 << 30))
}
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
+16
View File
@@ -30,6 +30,14 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
})
}
for _, s := range r.GetStorage() {
n.Storage = append(n.Storage, discovery.Storage{
Path: s.GetPath(),
FreeBytes: s.GetFreeBytes(),
TotalBytes: s.GetTotalBytes(),
Shared: s.GetShared(),
})
}
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: int(g.GetIndex()),
@@ -75,6 +83,14 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
})
}
for _, s := range inv.Storage {
r.Storage = append(r.Storage, &llmgwv1.Storage{
Path: s.Path,
FreeBytes: s.FreeBytes,
TotalBytes: s.TotalBytes,
Shared: s.Shared,
})
}
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
Index: int32(g.Index),
@@ -45,6 +45,8 @@ type Collector struct {
nodeGPUsFree *prometheus.Desc
nodeMemory *prometheus.Desc
nodeSeen *prometheus.Desc
nodeDiskFree *prometheus.Desc
nodeDiskSize *prometheus.Desc
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
@@ -76,6 +78,15 @@ func New(src Source) *Collector {
"A node's total RAM, 0 when it does not report one.", node, nil),
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
// Only the disk a node fills itself: space on a shared filer is not
// this node's to spend, and summing it per node would have a fleet of
// twenty each report the same free terabyte. Absent rather than zero
// when nothing was reported, so an alert on "free is low" does not
// fire for every node that was never asked about its disk.
nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes",
"Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil),
nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes",
"Size of those filesystems, so free space can be read as a fraction.", node, nil),
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
"A GPU's framebuffer size.", gpu, nil),
@@ -111,6 +122,7 @@ func New(src Source) *Collector {
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -143,6 +155,15 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
float64(n.MemoryTotalMB)*1024*1024, n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
now.Sub(n.LastSeen).Seconds(), n.ID)
// Nothing reported means nothing is known, so no sample at all. A zero
// here would read as a full disk and page someone about a node that
// was simply never asked.
if free, total := n.LocalDisk(); total > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue,
float64(free), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue,
float64(total), n.ID)
}
byClass := map[string]int{}
for _, g := range n.GPUs {
+28
View File
@@ -84,6 +84,12 @@ type Node struct {
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
// Storage is the disk those weights sit on, one entry per filesystem.
//
// Empty means the node was not asked, not that it has no disk — so an
// agent too old to report it, or one with no model directories, is left
// unconstrained rather than treated as full.
Storage []discovery.Storage `json:"storage,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
@@ -310,6 +316,28 @@ func (s *Store) FreeGPUs(nodeID string) []int {
return out
}
// LocalDisk sums the space on filesystems this node fills itself.
//
// Shared filesystems are left out on purpose. Space on a filer belongs to
// everyone mounting it, and counting it per node would let twenty nodes each
// believe they have the same free terabyte — then place twenty models into it.
// A node whose weights are all on a shared mount therefore reports no local
// disk, which is correct: it has none to spend.
//
// Both zero means nothing is known — an agent too old to report storage, or a
// node with no model directories — and callers must read that as unknown
// rather than as a full disk.
func (n Node) LocalDisk() (free, total int64) {
for _, s := range n.Storage {
if s.Shared {
continue
}
free += s.FreeBytes
total += s.TotalBytes
}
return free, total
}
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
@@ -248,3 +248,42 @@ func TestConcurrentAccess(t *testing.T) {
}
wg.Wait()
}
func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) {
// Twenty nodes mounting one filer have 500 GB between them, not ten
// terabytes. Counting a shared mount per node is how a scheduler places
// twenty models into space for one.
n := Node{Storage: []discovery.Storage{
{Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true},
{Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30},
}}
free, total := n.LocalDisk()
if free != 20<<30 || total != 60<<30 {
t.Errorf("shared space was counted as local: free=%d total=%d", free, total)
}
}
func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) {
// An agent too old to report storage, and a node with no model directories,
// both send nothing. Reading that as a full disk would take every such node
// out of placement for a reason that is not true.
free, total := Node{}.LocalDisk()
if free != 0 || total != 0 {
t.Errorf("a node with no storage reported %d/%d", free, total)
}
// The contract callers rely on: total of zero is the signal for "unknown",
// which is why nothing may read free alone.
if total != 0 {
t.Error("callers cannot distinguish unknown from full")
}
}
func TestLocalDiskSumsSeveralDisks(t *testing.T) {
n := Node{Storage: []discovery.Storage{
{Path: "/a", FreeBytes: 10, TotalBytes: 100},
{Path: "/b", FreeBytes: 5, TotalBytes: 50},
}}
if free, total := n.LocalDisk(); free != 15 || total != 150 {
t.Errorf("free=%d total=%d, want 15/150", free, total)
}
}