Let a node say how much room it has left
CI / test (push) Failing after 1m31s

Disk was the one resource nothing reported. `statfs` was already being called
on every model directory, but only to ask whether the filesystem was a network
one — the free space it returns in the same struct was thrown away. So the
only way to find out why a node was filling up was to ssh to it, which is what
this system exists to avoid.

Reported per filesystem rather than per configured directory. Free space
belongs to the disk, not the path: two model directories on one disk are one
pool of bytes, and counting them separately would let the control plane add
24 GB to 24 GB and place a 40 GB model on a node that has 24.

Shared filesystems are summed separately from local ones and left out of a
node's own total. Space on a filer belongs to everyone mounting it, so twenty
nodes would otherwise each report the same free terabyte, and a scheduler
would place twenty models into room for one.

Nothing reported means nothing is known, everywhere: an agent too old to send
storage and a node with no model directories both send nothing, and both must
read as unknown rather than as a full disk. `ctl nodes` and the dashboard show
a dash, the metric is absent rather than zero, and a missing directory — which
is what an unmounted filer looks like — is skipped rather than reported as
having no space.

Shown as free out of total, because the first is useless without the second:
8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root
partition, and only the second stops a model being placed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-18 09:38:57 +02:00
1 parent b5177b70f6
commit 753f4e3265
13 files changed
+515 -83

No files matched your search

+181 -77
View File
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -356,7 +356,16 @@ type Register struct {
// Empty means the node was not asked to look, which is not the same as having
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"`
Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"`
// storage is the disk behind those weights, one entry per filesystem.
//
// Reported so an operator can see a node filling up before a fetch dies at
// ninety percent, and so placement can refuse a node that cannot hold a
// model rather than discovering it during the download.
//
// Empty means the node was not asked to look, exactly as for weights, and
// constrains nothing.
Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -468,6 +477,91 @@ func (x *Register) GetWeights() []*LocalWeights {
return nil
}
func (x *Register) GetStorage() []*Storage {
if x != nil {
return x.Storage
}
return nil
}
// Storage is one filesystem a node keeps weights on.
//
// Per filesystem rather than per configured directory: free space belongs to
// the disk, and two model directories on one disk are one pool of bytes.
type Storage struct {
state protoimpl.MessageState `protogen:"open.v1"`
// path is one directory on this filesystem, so the operator recognises it.
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
// free_bytes is what a non-root process can write — not the root reserve,
// which is space the agent cannot have.
FreeBytes int64 `protobuf:"varint,2,opt,name=free_bytes,json=freeBytes,proto3" json:"free_bytes,omitempty"`
TotalBytes int64 `protobuf:"varint,3,opt,name=total_bytes,json=totalBytes,proto3" json:"total_bytes,omitempty"`
// shared marks a filesystem the node did not fill. Free space on one is
// shared with every other node using it, so two nodes each reporting 500 GB
// free on the same filer have 500 GB between them and not a terabyte.
Shared bool `protobuf:"varint,4,opt,name=shared,proto3" json:"shared,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Storage) Reset() {
*x = Storage{}
mi := &file_llmgw_v1_agent_proto_msgTypes[3]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *Storage) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*Storage) ProtoMessage() {}
func (x *Storage) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[3]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use Storage.ProtoReflect.Descriptor instead.
func (*Storage) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
}
func (x *Storage) GetPath() string {
if x != nil {
return x.Path
}
return ""
}
func (x *Storage) GetFreeBytes() int64 {
if x != nil {
return x.FreeBytes
}
return 0
}
func (x *Storage) GetTotalBytes() int64 {
if x != nil {
return x.TotalBytes
}
return 0
}
func (x *Storage) GetShared() bool {
if x != nil {
return x.Shared
}
return false
}
// RunningShard describes an engine already running on a node.
//
// It carries enough for the control plane to rebuild the replica it belongs to,
@@ -498,7 +592,7 @@ type RunningShard struct {
func (x *RunningShard) Reset() {
*x = RunningShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[3]
mi := &file_llmgw_v1_agent_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -510,7 +604,7 @@ func (x *RunningShard) String() string {
func (*RunningShard) ProtoMessage() {}
func (x *RunningShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[3]
mi := &file_llmgw_v1_agent_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -523,7 +617,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use RunningShard.ProtoReflect.Descriptor instead.
func (*RunningShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
}
func (x *RunningShard) GetReplicaId() string {
@@ -653,7 +747,7 @@ type GPU struct {
func (x *GPU) Reset() {
*x = GPU{}
mi := &file_llmgw_v1_agent_proto_msgTypes[4]
mi := &file_llmgw_v1_agent_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -665,7 +759,7 @@ func (x *GPU) String() string {
func (*GPU) ProtoMessage() {}
func (x *GPU) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[4]
mi := &file_llmgw_v1_agent_proto_msgTypes[5]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -678,7 +772,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message {
// Deprecated: Use GPU.ProtoReflect.Descriptor instead.
func (*GPU) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
}
func (x *GPU) GetIndex() int32 {
@@ -800,7 +894,7 @@ type FabricAttachment struct {
func (x *FabricAttachment) Reset() {
*x = FabricAttachment{}
mi := &file_llmgw_v1_agent_proto_msgTypes[5]
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -812,7 +906,7 @@ func (x *FabricAttachment) String() string {
func (*FabricAttachment) ProtoMessage() {}
func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[5]
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -825,7 +919,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead.
func (*FabricAttachment) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
}
func (x *FabricAttachment) GetFabric() string {
@@ -867,7 +961,7 @@ type LocalWeights struct {
func (x *LocalWeights) Reset() {
*x = LocalWeights{}
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -879,7 +973,7 @@ func (x *LocalWeights) String() string {
func (*LocalWeights) ProtoMessage() {}
func (x *LocalWeights) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -892,7 +986,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead.
func (*LocalWeights) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
}
func (x *LocalWeights) GetPath() string {
@@ -929,7 +1023,7 @@ type Heartbeat struct {
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -941,7 +1035,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -954,7 +1048,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
}
func (x *Heartbeat) GetNodeId() string {
@@ -996,7 +1090,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1008,7 +1102,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1021,7 +1115,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1102,7 +1196,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1114,7 +1208,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1127,7 +1221,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
}
func (x *StartShard) GetReplicaId() string {
@@ -1247,7 +1341,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1259,7 +1353,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1272,7 +1366,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *EngineProfile) GetName() string {
@@ -1362,7 +1456,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1374,7 +1468,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1387,7 +1481,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *StopShard) GetShardId() string {
@@ -1407,7 +1501,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1419,7 +1513,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1432,7 +1526,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *DrainShard) GetShardId() string {
@@ -1454,7 +1548,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1466,7 +1560,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1479,7 +1573,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *Ack) GetMessageId() string {
@@ -1518,7 +1612,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
"\x03msg\"\xfa\x04\n" +
"\x03msg\"\xa7\x05\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -1531,13 +1625,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
"\x0fmemory_total_mb\x18\n" +
" \x01(\x03R\rmemoryTotalMb\x120\n" +
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" +
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
"\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x97\x03\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" +
"\aStorage\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" +
"\vtotal_bytes\x18\x03 \x01(\x03R\n" +
"totalBytes\x12\x16\n" +
"\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" +
"\fRunningShard\x12\x1d\n" +
"\n" +
"replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" +
@@ -1675,52 +1777,54 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
(*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage
(*Register)(nil), // 3: llmgw.v1.Register
(*RunningShard)(nil), // 4: llmgw.v1.RunningShard
(*GPU)(nil), // 5: llmgw.v1.GPU
(*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights
(*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat
(*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus
(*StartShard)(nil), // 10: llmgw.v1.StartShard
(*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile
(*StopShard)(nil), // 12: llmgw.v1.StopShard
(*DrainShard)(nil), // 13: llmgw.v1.DrainShard
(*Ack)(nil), // 14: llmgw.v1.Ack
nil, // 15: llmgw.v1.Register.LabelsEntry
nil, // 16: llmgw.v1.Register.EngineVersionsEntry
nil, // 17: llmgw.v1.EngineProfile.EnvEntry
(*Storage)(nil), // 4: llmgw.v1.Storage
(*RunningShard)(nil), // 5: llmgw.v1.RunningShard
(*GPU)(nil), // 6: llmgw.v1.GPU
(*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment
(*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights
(*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat
(*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus
(*StartShard)(nil), // 11: llmgw.v1.StartShard
(*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile
(*StopShard)(nil), // 13: llmgw.v1.StopShard
(*DrainShard)(nil), // 14: llmgw.v1.DrainShard
(*Ack)(nil), // 15: llmgw.v1.Ack
nil, // 16: llmgw.v1.Register.LabelsEntry
nil, // 17: llmgw.v1.Register.EngineVersionsEntry
nil, // 18: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
18, // [18:19] is the sub-list for method output_type
17, // [17:18] is the sub-list for method input_type
17, // [17:17] is the sub-list for extension type_name
17, // [17:17] is the sub-list for extension extendee
0, // [0:17] is the sub-list for field type_name
9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
19, // [19:20] is the sub-list for method output_type
18, // [18:19] is the sub-list for method input_type
18, // [18:18] is the sub-list for extension type_name
18, // [18:18] is the sub-list for extension extendee
0, // [0:18] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -1745,7 +1849,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 17,
NumMessages: 18,
NumExtensions: 0,
NumServices: 1,
},
+27
View File
@@ -90,6 +90,33 @@ message Register {
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
repeated LocalWeights weights = 11;
// storage is the disk behind those weights, one entry per filesystem.
//
// Reported so an operator can see a node filling up before a fetch dies at
// ninety percent, and so placement can refuse a node that cannot hold a
// model rather than discovering it during the download.
//
// Empty means the node was not asked to look, exactly as for weights, and
// constrains nothing.
repeated Storage storage = 12;
}
// Storage is one filesystem a node keeps weights on.
//
// Per filesystem rather than per configured directory: free space belongs to
// the disk, and two model directories on one disk are one pool of bytes.
message Storage {
// path is one directory on this filesystem, so the operator recognises it.
string path = 1;
// free_bytes is what a non-root process can write — not the root reserve,
// which is space the agent cannot have.
int64 free_bytes = 2;
int64 total_bytes = 3;
// shared marks a filesystem the node did not fill. Free space on one is
// shared with every other node using it, so two nodes each reporting 500 GB
// free on the same filer have 500 GB between them and not a terabyte.
bool shared = 4;
}
// RunningShard describes an engine already running on a node.
+3
View File
@@ -51,6 +51,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
// holds has nothing to do with its cards, and a node with none is exactly
// where a CPU profile will look for them.
inv.Weights = ScanWeights(opts.ModelDirs)
// And the disk they sit on. A node can hold a model today and be unable to
// take the next one, which the file list alone never shows.
inv.Storage = ScanStorage(opts.ModelDirs)
if !opts.SkipGPUs {
switch {
+3
View File
@@ -114,6 +114,9 @@ type Inventory struct {
// Weights are the model files this node can see. Empty means it was not
// asked to look.
Weights []LocalWeights `json:"weights,omitempty"`
// Storage is the disk behind those weights, one entry per filesystem.
// Empty means the same thing it means for Weights: not asked.
Storage []Storage `json:"storage,omitempty"`
}
// Classes returns the distinct GPU classes present, sorted.
+90
View File
@@ -0,0 +1,90 @@
package discovery
import (
"os"
"syscall"
)
// Storage is one filesystem a node keeps weights on.
//
// One entry per filesystem rather than per configured directory, because free
// space belongs to the disk and not to the path. Two model directories on the
// same disk are one pool of bytes, and reporting them separately would let the
// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24.
type Storage struct {
// Path is one directory on this filesystem — the first one configured that
// landed here, so the operator recognises it.
Path string `json:"path"`
// FreeBytes is what a non-root process can still write. Deliberately not
// the root reserve: the agent does not run as root, so the reserve is space
// it cannot have.
FreeBytes int64 `json:"free_bytes"`
// TotalBytes is the filesystem's size, so a reading of "8 GB free" can be
// told apart on a 10 TB filer and on a 10 GB root partition.
TotalBytes int64 `json:"total_bytes"`
// Shared marks a filesystem the node did not have to fill. Free space on
// one is shared with every other node using it, so two nodes reporting
// 500 GB free on the same filer have 500 GB between them, not a terabyte.
Shared bool `json:"shared,omitempty"`
}
// ScanStorage reports the filesystems behind the given model directories.
//
// Empty in, empty out, and a directory that cannot be read is skipped rather
// than reported as full: a mount that has not come back yet looks exactly like
// a missing directory, and claiming zero free bytes would take the node out of
// placement for a reason that is not true. Absent is not zero here, as
// everywhere else in this system.
func ScanStorage(dirs []string) []Storage {
var out []Storage
seen := map[uint64]int{} // filesystem id -> index in out
for _, dir := range dirs {
if dir == "" {
continue
}
dev, ok := filesystemID(dir)
if !ok {
continue
}
if i, dup := seen[dev]; dup {
// Same disk, already counted. Keep the first path: it is the one
// the operator wrote first and so the one they will recognise.
_ = i
continue
}
var st syscall.Statfs_t
if err := syscall.Statfs(dir, &st); err != nil {
continue
}
// Bsize is int32 on some platforms and int64 on others, which is why
// every field here is widened explicitly rather than assumed.
block := int64(st.Bsize)
seen[dev] = len(out)
out = append(out, Storage{
Path: dir,
FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose
TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose
Shared: isNetworkFS(dir),
})
}
return out
}
// filesystemID identifies the disk a path is on.
//
// By device number rather than by the statfs filesystem id, because the field
// holding that id is spelled differently on Linux and on macOS and this has to
// compile on both — the agent runs on one and is developed on the other.
func filesystemID(path string) (uint64, bool) {
fi, err := os.Stat(path)
if err != nil {
return 0, false
}
st, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return 0, false
}
return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux
}
+63
View File
@@ -0,0 +1,63 @@
package discovery
import (
"os"
"path/filepath"
"testing"
)
func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) {
// The reason this matters: free space belongs to the disk, not to the path.
// Counting two directories separately lets the control plane add 24 GB to
// 24 GB and place a 40 GB model on a node that has 24.
base := t.TempDir()
a := filepath.Join(base, "models-a")
b := filepath.Join(base, "models-b")
for _, d := range []string{a, b} {
if err := os.Mkdir(d, 0o755); err != nil {
t.Fatal(err)
}
}
got := ScanStorage([]string{a, b})
if len(got) != 1 {
t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got)
}
if got[0].Path != a {
t.Errorf("kept %q rather than the first directory configured", got[0].Path)
}
}
func TestStorageReportsSomethingPlausible(t *testing.T) {
got := ScanStorage([]string{t.TempDir()})
if len(got) != 1 {
t.Fatalf("a real directory reported %d filesystems", len(got))
}
s := got[0]
if s.TotalBytes <= 0 {
t.Error("a filesystem with no size at all")
}
if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes {
t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes)
}
}
func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) {
// A mount that has not come back yet looks exactly like a missing
// directory. Reporting it as zero free bytes would take the node out of
// placement for a reason that is not true; reporting nothing says only
// that nothing is known, which is the honest answer.
got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")})
if len(got) != 0 {
t.Errorf("a missing directory was reported as a filesystem: %+v", got)
}
}
func TestNoDirectoriesConstrainNothing(t *testing.T) {
if got := ScanStorage(nil); len(got) != 0 {
t.Errorf("scanning nothing found %+v", got)
}
if got := ScanStorage([]string{""}); len(got) != 0 {
t.Errorf("an empty path found %+v", got)
}
}
+20 -2
View File
@@ -149,10 +149,11 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
for _, n := range nodes {
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n",
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
}
_ = w.Flush()
@@ -221,6 +222,23 @@ func memColumn(gb int) string {
return fmt.Sprintf("%dG", gb)
}
// diskColumn renders free space against the size it is free out of.
//
// Both, because they answer different questions and the first is useless
// without the second: 8 GB free is comfortable on a 10 TB filer and nearly
// full on a 10 GB root partition, and it is the second case that stops a
// model from being placed.
//
// A dash for a node that reported nothing, which an older agent and a node
// with no model directories both do. It has to read as "not known" rather
// than as "no space", or every unconfigured node looks like it is full.
func diskColumn(freeGB, totalGB int) string {
if totalGB <= 0 {
return "-"
}
return fmt.Sprintf("%dG/%dG", freeGB, totalGB)
}
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "models",
+19 -3
View File
@@ -62,9 +62,21 @@ type NodeView struct {
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
Models []string `json:"models_on_disk,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
Models []string `json:"models_on_disk,omitempty"`
// DiskFreeGB and DiskTotalGB are the filesystems holding those weights,
// summed. Zero means the agent did not report any, which an older agent
// and a node with no model directories both do — so it reads as unknown
// rather than as a full disk.
//
// Summed because an operator asking "can this node take another model"
// wants one number; the per-filesystem detail is in the node's own record
// for anyone who needs it. Shared filesystems are left out of the sum: the
// space on a filer is not this node's to spend, and adding it would let a
// fleet of twenty nodes each believe it has the same free terabyte.
DiskFreeGB int `json:"disk_free_gb,omitempty"`
DiskTotalGB int `json:"disk_total_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -166,6 +178,10 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
if free, total := n.LocalDisk(); total > 0 {
v.DiskFreeGB = int(free / (1 << 30))
v.DiskTotalGB = int(total / (1 << 30))
}
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
+16
View File
@@ -30,6 +30,14 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
})
}
for _, s := range r.GetStorage() {
n.Storage = append(n.Storage, discovery.Storage{
Path: s.GetPath(),
FreeBytes: s.GetFreeBytes(),
TotalBytes: s.GetTotalBytes(),
Shared: s.GetShared(),
})
}
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: int(g.GetIndex()),
@@ -75,6 +83,14 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
})
}
for _, s := range inv.Storage {
r.Storage = append(r.Storage, &llmgwv1.Storage{
Path: s.Path,
FreeBytes: s.FreeBytes,
TotalBytes: s.TotalBytes,
Shared: s.Shared,
})
}
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
Index: int32(g.Index),
@@ -45,6 +45,8 @@ type Collector struct {
nodeGPUsFree *prometheus.Desc
nodeMemory *prometheus.Desc
nodeSeen *prometheus.Desc
nodeDiskFree *prometheus.Desc
nodeDiskSize *prometheus.Desc
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
@@ -76,6 +78,15 @@ func New(src Source) *Collector {
"A node's total RAM, 0 when it does not report one.", node, nil),
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
// Only the disk a node fills itself: space on a shared filer is not
// this node's to spend, and summing it per node would have a fleet of
// twenty each report the same free terabyte. Absent rather than zero
// when nothing was reported, so an alert on "free is low" does not
// fire for every node that was never asked about its disk.
nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes",
"Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil),
nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes",
"Size of those filesystems, so free space can be read as a fraction.", node, nil),
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
"A GPU's framebuffer size.", gpu, nil),
@@ -111,6 +122,7 @@ func New(src Source) *Collector {
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -143,6 +155,15 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
float64(n.MemoryTotalMB)*1024*1024, n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
now.Sub(n.LastSeen).Seconds(), n.ID)
// Nothing reported means nothing is known, so no sample at all. A zero
// here would read as a full disk and page someone about a node that
// was simply never asked.
if free, total := n.LocalDisk(); total > 0 {
ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue,
float64(free), n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue,
float64(total), n.ID)
}
byClass := map[string]int{}
for _, g := range n.GPUs {
+28
View File
@@ -84,6 +84,12 @@ type Node struct {
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
// Storage is the disk those weights sit on, one entry per filesystem.
//
// Empty means the node was not asked, not that it has no disk — so an
// agent too old to report it, or one with no model directories, is left
// unconstrained rather than treated as full.
Storage []discovery.Storage `json:"storage,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
@@ -310,6 +316,28 @@ func (s *Store) FreeGPUs(nodeID string) []int {
return out
}
// LocalDisk sums the space on filesystems this node fills itself.
//
// Shared filesystems are left out on purpose. Space on a filer belongs to
// everyone mounting it, and counting it per node would let twenty nodes each
// believe they have the same free terabyte — then place twenty models into it.
// A node whose weights are all on a shared mount therefore reports no local
// disk, which is correct: it has none to spend.
//
// Both zero means nothing is known — an agent too old to report storage, or a
// node with no model directories — and callers must read that as unknown
// rather than as a full disk.
func (n Node) LocalDisk() (free, total int64) {
for _, s := range n.Storage {
if s.Shared {
continue
}
free += s.FreeBytes
total += s.TotalBytes
}
return free, total
}
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
@@ -248,3 +248,42 @@ func TestConcurrentAccess(t *testing.T) {
}
wg.Wait()
}
func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) {
// Twenty nodes mounting one filer have 500 GB between them, not ten
// terabytes. Counting a shared mount per node is how a scheduler places
// twenty models into space for one.
n := Node{Storage: []discovery.Storage{
{Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true},
{Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30},
}}
free, total := n.LocalDisk()
if free != 20<<30 || total != 60<<30 {
t.Errorf("shared space was counted as local: free=%d total=%d", free, total)
}
}
func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) {
// An agent too old to report storage, and a node with no model directories,
// both send nothing. Reading that as a full disk would take every such node
// out of placement for a reason that is not true.
free, total := Node{}.LocalDisk()
if free != 0 || total != 0 {
t.Errorf("a node with no storage reported %d/%d", free, total)
}
// The contract callers rely on: total of zero is the signal for "unknown",
// which is why nothing may read free alone.
if total != 0 {
t.Error("callers cannot distinguish unknown from full")
}
}
func TestLocalDiskSumsSeveralDisks(t *testing.T) {
n := Node{Storage: []discovery.Storage{
{Path: "/a", FreeBytes: 10, TotalBytes: 100},
{Path: "/b", FreeBytes: 5, TotalBytes: 50},
}}
if free, total := n.LocalDisk(); free != 15 || total != 150 {
t.Errorf("free=%d total=%d, want 15/150", free, total)
}
}
+5 -1
View File
@@ -98,12 +98,16 @@ requested whenever a fallback served it.</p>
<h2>Nodes</h2>
{{if .Nodes}}
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
<table><tr><th>Node</th><th>State</th><th class="num">GPUs</th><th class="num">RAM</th><th class="num">Disk</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
{{range .Nodes}}<tr>
<td><code>{{.ID}}</code>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}</td>
<td class="state state-{{.State}}">{{.State}}</td>
<td class="num">{{.GPUFree}}/{{.GPUTotal}}</td>
<td class="num">{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}</td>
{{/* Free out of total, because the first is meaningless without the
second, and an em dash where nothing was reported: an agent that
was never asked about its disk must not read as a full one. */}}
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
<td>{{join .Classes}}</td>
<td>{{join .Fabrics}}</td>
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>