From 753f4e32654799fb16949eadfd26d7322b66bf2e Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 18 Sep 2026 09:38:57 +0200 Subject: [PATCH] Let a node say how much room it has left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disk was the one resource nothing reported. `statfs` was already being called on every model directory, but only to ask whether the filesystem was a network one — the free space it returns in the same struct was thrown away. So the only way to find out why a node was filling up was to ssh to it, which is what this system exists to avoid. Reported per filesystem rather than per configured directory. Free space belongs to the disk, not the path: two model directories on one disk are one pool of bytes, and counting them separately would let the control plane add 24 GB to 24 GB and place a 40 GB model on a node that has 24. Shared filesystems are summed separately from local ones and left out of a node's own total. Space on a filer belongs to everyone mounting it, so twenty nodes would otherwise each report the same free terabyte, and a scheduler would place twenty models into room for one. Nothing reported means nothing is known, everywhere: an agent too old to send storage and a node with no model directories both send nothing, and both must read as unknown rather than as a full disk. `ctl nodes` and the dashboard show a dash, the metric is absent rather than zero, and a missing directory — which is what an unmounted filer looks like — is skipped rather than reported as having no space. Shown as free out of total, because the first is useless without the second: 8 GB free is comfortable on a 10 TB filer and nearly full on a 10 GB root partition, and only the second stops a model being placed. Co-Authored-By: Claude Opus 5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 258 ++++++++++++------ api/proto/llmgw/v1/agent.proto | 27 ++ internal/agent/discovery/discover.go | 3 + internal/agent/discovery/gpu.go | 3 + internal/agent/discovery/storage.go | 90 ++++++ internal/agent/discovery/storage_test.go | 63 +++++ internal/cli/ctl.go | 22 +- internal/control/adminapi/api.go | 22 +- internal/control/agentsvc/convert.go | 16 ++ internal/control/fleetmetrics/fleetmetrics.go | 21 ++ internal/control/inventory/inventory.go | 28 ++ internal/control/inventory/inventory_test.go | 39 +++ internal/web/templates/dashboard.html | 6 +- 13 files changed, 515 insertions(+), 83 deletions(-) create mode 100644 internal/agent/discovery/storage.go create mode 100644 internal/agent/discovery/storage_test.go diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 11c560cc..d6d8ecf1 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0} } // AgentMessage is anything a node tells the control plane. @@ -356,7 +356,16 @@ type Register struct { // Empty means the node was not asked to look, which is not the same as having // nothing — an agent with no model directories configured behaves exactly as // every agent did before this existed, and constrains nothing. - Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"` + Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"` + // storage is the disk behind those weights, one entry per filesystem. + // + // Reported so an operator can see a node filling up before a fetch dies at + // ninety percent, and so placement can refuse a node that cannot hold a + // model rather than discovering it during the download. + // + // Empty means the node was not asked to look, exactly as for weights, and + // constrains nothing. + Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -468,6 +477,91 @@ func (x *Register) GetWeights() []*LocalWeights { return nil } +func (x *Register) GetStorage() []*Storage { + if x != nil { + return x.Storage + } + return nil +} + +// Storage is one filesystem a node keeps weights on. +// +// Per filesystem rather than per configured directory: free space belongs to +// the disk, and two model directories on one disk are one pool of bytes. +type Storage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // path is one directory on this filesystem, so the operator recognises it. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // free_bytes is what a non-root process can write — not the root reserve, + // which is space the agent cannot have. + FreeBytes int64 `protobuf:"varint,2,opt,name=free_bytes,json=freeBytes,proto3" json:"free_bytes,omitempty"` + TotalBytes int64 `protobuf:"varint,3,opt,name=total_bytes,json=totalBytes,proto3" json:"total_bytes,omitempty"` + // shared marks a filesystem the node did not fill. Free space on one is + // shared with every other node using it, so two nodes each reporting 500 GB + // free on the same filer have 500 GB between them and not a terabyte. + Shared bool `protobuf:"varint,4,opt,name=shared,proto3" json:"shared,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Storage) Reset() { + *x = Storage{} + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Storage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Storage) ProtoMessage() {} + +func (x *Storage) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Storage.ProtoReflect.Descriptor instead. +func (*Storage) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *Storage) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *Storage) GetFreeBytes() int64 { + if x != nil { + return x.FreeBytes + } + return 0 +} + +func (x *Storage) GetTotalBytes() int64 { + if x != nil { + return x.TotalBytes + } + return 0 +} + +func (x *Storage) GetShared() bool { + if x != nil { + return x.Shared + } + return false +} + // RunningShard describes an engine already running on a node. // // It carries enough for the control plane to rebuild the replica it belongs to, @@ -498,7 +592,7 @@ type RunningShard struct { func (x *RunningShard) Reset() { *x = RunningShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -510,7 +604,7 @@ func (x *RunningShard) String() string { func (*RunningShard) ProtoMessage() {} func (x *RunningShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -523,7 +617,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningShard.ProtoReflect.Descriptor instead. func (*RunningShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} } func (x *RunningShard) GetReplicaId() string { @@ -653,7 +747,7 @@ type GPU struct { func (x *GPU) Reset() { *x = GPU{} - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -665,7 +759,7 @@ func (x *GPU) String() string { func (*GPU) ProtoMessage() {} func (x *GPU) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -678,7 +772,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message { // Deprecated: Use GPU.ProtoReflect.Descriptor instead. func (*GPU) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} } func (x *GPU) GetIndex() int32 { @@ -800,7 +894,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -812,7 +906,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -825,7 +919,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} } func (x *FabricAttachment) GetFabric() string { @@ -867,7 +961,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -879,7 +973,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -892,7 +986,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *LocalWeights) GetPath() string { @@ -929,7 +1023,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -941,7 +1035,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -954,7 +1048,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *Heartbeat) GetNodeId() string { @@ -996,7 +1090,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1008,7 +1102,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1021,7 +1115,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *ShardStatus) GetNodeId() string { @@ -1102,7 +1196,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1114,7 +1208,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1127,7 +1221,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *StartShard) GetReplicaId() string { @@ -1247,7 +1341,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1353,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1366,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *EngineProfile) GetName() string { @@ -1362,7 +1456,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1374,7 +1468,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1387,7 +1481,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *StopShard) GetShardId() string { @@ -1407,7 +1501,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1419,7 +1513,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1432,7 +1526,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *DrainShard) GetShardId() string { @@ -1454,7 +1548,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1466,7 +1560,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1479,7 +1573,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *Ack) GetMessageId() string { @@ -1518,7 +1612,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + - "\x03msg\"\xfa\x04\n" + + "\x03msg\"\xa7\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -1531,13 +1625,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" + "\x0fmemory_total_mb\x18\n" + " \x01(\x03R\rmemoryTotalMb\x120\n" + - "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" + + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x97\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" + + "\aStorage\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + + "\n" + + "free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" + + "\vtotal_bytes\x18\x03 \x01(\x03R\n" + + "totalBytes\x12\x16\n" + + "\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" + "\fRunningShard\x12\x1d\n" + "\n" + "replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" + @@ -1675,52 +1777,54 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage (*Register)(nil), // 3: llmgw.v1.Register - (*RunningShard)(nil), // 4: llmgw.v1.RunningShard - (*GPU)(nil), // 5: llmgw.v1.GPU - (*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights - (*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat - (*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus - (*StartShard)(nil), // 10: llmgw.v1.StartShard - (*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile - (*StopShard)(nil), // 12: llmgw.v1.StopShard - (*DrainShard)(nil), // 13: llmgw.v1.DrainShard - (*Ack)(nil), // 14: llmgw.v1.Ack - nil, // 15: llmgw.v1.Register.LabelsEntry - nil, // 16: llmgw.v1.Register.EngineVersionsEntry - nil, // 17: llmgw.v1.EngineProfile.EnvEntry + (*Storage)(nil), // 4: llmgw.v1.Storage + (*RunningShard)(nil), // 5: llmgw.v1.RunningShard + (*GPU)(nil), // 6: llmgw.v1.GPU + (*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights + (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat + (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus + (*StartShard)(nil), // 11: llmgw.v1.StartShard + (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile + (*StopShard)(nil), // 13: llmgw.v1.StopShard + (*DrainShard)(nil), // 14: llmgw.v1.DrainShard + (*Ack)(nil), // 15: llmgw.v1.Ack + nil, // 16: llmgw.v1.Register.LabelsEntry + nil, // 17: llmgw.v1.Register.EngineVersionsEntry + nil, // 18: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 18, // [18:19] is the sub-list for method output_type - 17, // [17:18] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 19, // [19:20] is the sub-list for method output_type + 18, // [18:19] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1745,7 +1849,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 17, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 39604c70..8fc6c2f7 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -90,6 +90,33 @@ message Register { // nothing — an agent with no model directories configured behaves exactly as // every agent did before this existed, and constrains nothing. repeated LocalWeights weights = 11; + + // storage is the disk behind those weights, one entry per filesystem. + // + // Reported so an operator can see a node filling up before a fetch dies at + // ninety percent, and so placement can refuse a node that cannot hold a + // model rather than discovering it during the download. + // + // Empty means the node was not asked to look, exactly as for weights, and + // constrains nothing. + repeated Storage storage = 12; +} + +// Storage is one filesystem a node keeps weights on. +// +// Per filesystem rather than per configured directory: free space belongs to +// the disk, and two model directories on one disk are one pool of bytes. +message Storage { + // path is one directory on this filesystem, so the operator recognises it. + string path = 1; + // free_bytes is what a non-root process can write — not the root reserve, + // which is space the agent cannot have. + int64 free_bytes = 2; + int64 total_bytes = 3; + // shared marks a filesystem the node did not fill. Free space on one is + // shared with every other node using it, so two nodes each reporting 500 GB + // free on the same filer have 500 GB between them and not a terabyte. + bool shared = 4; } // RunningShard describes an engine already running on a node. diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 812acbe3..275cc31d 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -51,6 +51,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e // holds has nothing to do with its cards, and a node with none is exactly // where a CPU profile will look for them. inv.Weights = ScanWeights(opts.ModelDirs) + // And the disk they sit on. A node can hold a model today and be unable to + // take the next one, which the file list alone never shows. + inv.Storage = ScanStorage(opts.ModelDirs) if !opts.SkipGPUs { switch { diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index cd94b5c1..4d475bf1 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -114,6 +114,9 @@ type Inventory struct { // Weights are the model files this node can see. Empty means it was not // asked to look. Weights []LocalWeights `json:"weights,omitempty"` + // Storage is the disk behind those weights, one entry per filesystem. + // Empty means the same thing it means for Weights: not asked. + Storage []Storage `json:"storage,omitempty"` } // Classes returns the distinct GPU classes present, sorted. diff --git a/internal/agent/discovery/storage.go b/internal/agent/discovery/storage.go new file mode 100644 index 00000000..fc8a9881 --- /dev/null +++ b/internal/agent/discovery/storage.go @@ -0,0 +1,90 @@ +package discovery + +import ( + "os" + "syscall" +) + +// Storage is one filesystem a node keeps weights on. +// +// One entry per filesystem rather than per configured directory, because free +// space belongs to the disk and not to the path. Two model directories on the +// same disk are one pool of bytes, and reporting them separately would let the +// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24. +type Storage struct { + // Path is one directory on this filesystem — the first one configured that + // landed here, so the operator recognises it. + Path string `json:"path"` + // FreeBytes is what a non-root process can still write. Deliberately not + // the root reserve: the agent does not run as root, so the reserve is space + // it cannot have. + FreeBytes int64 `json:"free_bytes"` + // TotalBytes is the filesystem's size, so a reading of "8 GB free" can be + // told apart on a 10 TB filer and on a 10 GB root partition. + TotalBytes int64 `json:"total_bytes"` + // Shared marks a filesystem the node did not have to fill. Free space on + // one is shared with every other node using it, so two nodes reporting + // 500 GB free on the same filer have 500 GB between them, not a terabyte. + Shared bool `json:"shared,omitempty"` +} + +// ScanStorage reports the filesystems behind the given model directories. +// +// Empty in, empty out, and a directory that cannot be read is skipped rather +// than reported as full: a mount that has not come back yet looks exactly like +// a missing directory, and claiming zero free bytes would take the node out of +// placement for a reason that is not true. Absent is not zero here, as +// everywhere else in this system. +func ScanStorage(dirs []string) []Storage { + var out []Storage + seen := map[uint64]int{} // filesystem id -> index in out + + for _, dir := range dirs { + if dir == "" { + continue + } + dev, ok := filesystemID(dir) + if !ok { + continue + } + if i, dup := seen[dev]; dup { + // Same disk, already counted. Keep the first path: it is the one + // the operator wrote first and so the one they will recognise. + _ = i + continue + } + + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + continue + } + // Bsize is int32 on some platforms and int64 on others, which is why + // every field here is widened explicitly rather than assumed. + block := int64(st.Bsize) + seen[dev] = len(out) + out = append(out, Storage{ + Path: dir, + FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose + TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose + Shared: isNetworkFS(dir), + }) + } + return out +} + +// filesystemID identifies the disk a path is on. +// +// By device number rather than by the statfs filesystem id, because the field +// holding that id is spelled differently on Linux and on macOS and this has to +// compile on both — the agent runs on one and is developed on the other. +func filesystemID(path string) (uint64, bool) { + fi, err := os.Stat(path) + if err != nil { + return 0, false + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux +} diff --git a/internal/agent/discovery/storage_test.go b/internal/agent/discovery/storage_test.go new file mode 100644 index 00000000..b8f7a090 --- /dev/null +++ b/internal/agent/discovery/storage_test.go @@ -0,0 +1,63 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) { + // The reason this matters: free space belongs to the disk, not to the path. + // Counting two directories separately lets the control plane add 24 GB to + // 24 GB and place a 40 GB model on a node that has 24. + base := t.TempDir() + a := filepath.Join(base, "models-a") + b := filepath.Join(base, "models-b") + for _, d := range []string{a, b} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + + got := ScanStorage([]string{a, b}) + if len(got) != 1 { + t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got) + } + if got[0].Path != a { + t.Errorf("kept %q rather than the first directory configured", got[0].Path) + } +} + +func TestStorageReportsSomethingPlausible(t *testing.T) { + got := ScanStorage([]string{t.TempDir()}) + if len(got) != 1 { + t.Fatalf("a real directory reported %d filesystems", len(got)) + } + s := got[0] + if s.TotalBytes <= 0 { + t.Error("a filesystem with no size at all") + } + if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes { + t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes) + } +} + +func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) { + // A mount that has not come back yet looks exactly like a missing + // directory. Reporting it as zero free bytes would take the node out of + // placement for a reason that is not true; reporting nothing says only + // that nothing is known, which is the honest answer. + got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")}) + if len(got) != 0 { + t.Errorf("a missing directory was reported as a filesystem: %+v", got) + } +} + +func TestNoDirectoriesConstrainNothing(t *testing.T) { + if got := ScanStorage(nil); len(got) != 0 { + t.Errorf("scanning nothing found %+v", got) + } + if got := ScanStorage([]string{""}); len(got) != 0 { + t.Errorf("an empty path found %+v", got) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index cce20d30..b98c888b 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -149,10 +149,11 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n", + fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n", n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), + diskColumn(n.DiskFreeGB, n.DiskTotalGB), join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -221,6 +222,23 @@ func memColumn(gb int) string { return fmt.Sprintf("%dG", gb) } +// diskColumn renders free space against the size it is free out of. +// +// Both, because they answer different questions and the first is useless +// without the second: 8 GB free is comfortable on a 10 TB filer and nearly +// full on a 10 GB root partition, and it is the second case that stops a +// model from being placed. +// +// A dash for a node that reported nothing, which an older agent and a node +// with no model directories both do. It has to read as "not known" rather +// than as "no space", or every unconfigured node looks like it is full. +func diskColumn(freeGB, totalGB int) string { + if totalGB <= 0 { + return "-" + } + return fmt.Sprintf("%dG/%dG", freeGB, totalGB) +} + func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return &cobra.Command{ Use: "models", diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index b9f09c16..2b9792ac 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -62,9 +62,21 @@ type NodeView struct { MemoryGB int `json:"memory_gb,omitempty"` // Models are the weights this node can see. Absent means it was not asked // to look, which is not the same as holding none. - Models []string `json:"models_on_disk,omitempty"` - Classes []string `json:"gpu_classes,omitempty"` - Fabrics []string `json:"fabrics,omitempty"` + Models []string `json:"models_on_disk,omitempty"` + // DiskFreeGB and DiskTotalGB are the filesystems holding those weights, + // summed. Zero means the agent did not report any, which an older agent + // and a node with no model directories both do — so it reads as unknown + // rather than as a full disk. + // + // Summed because an operator asking "can this node take another model" + // wants one number; the per-filesystem detail is in the node's own record + // for anyone who needs it. Shared filesystems are left out of the sum: the + // space on a filer is not this node's to spend, and adding it would let a + // fleet of twenty nodes each believe it has the same free terabyte. + DiskFreeGB int `json:"disk_free_gb,omitempty"` + DiskTotalGB int `json:"disk_total_gb,omitempty"` + Classes []string `json:"gpu_classes,omitempty"` + Fabrics []string `json:"fabrics,omitempty"` // Unhealthy lists GPUs kept out of placement and why, which is usually the // answer when a node looks idle but nothing will schedule onto it. @@ -166,6 +178,10 @@ func (a *API) Nodes() []NodeView { Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Replicas: replicasOn[n.ID], } + if free, total := n.LocalDisk(); total > 0 { + v.DiskFreeGB = int(free / (1 << 30)) + v.DiskTotalGB = int(total / (1 << 30)) + } for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index e8d00089..4c23a663 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -30,6 +30,14 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(), }) } + for _, s := range r.GetStorage() { + n.Storage = append(n.Storage, discovery.Storage{ + Path: s.GetPath(), + FreeBytes: s.GetFreeBytes(), + TotalBytes: s.GetTotalBytes(), + Shared: s.GetShared(), + }) + } for _, g := range r.GetGpus() { n.GPUs = append(n.GPUs, discovery.GPU{ Index: int(g.GetIndex()), @@ -75,6 +83,14 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared, }) } + for _, s := range inv.Storage { + r.Storage = append(r.Storage, &llmgwv1.Storage{ + Path: s.Path, + FreeBytes: s.FreeBytes, + TotalBytes: s.TotalBytes, + Shared: s.Shared, + }) + } for _, g := range inv.GPUs { r.Gpus = append(r.Gpus, &llmgwv1.GPU{ Index: int32(g.Index), diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index a9b39bf3..b5b9c9ae 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -45,6 +45,8 @@ type Collector struct { nodeGPUsFree *prometheus.Desc nodeMemory *prometheus.Desc nodeSeen *prometheus.Desc + nodeDiskFree *prometheus.Desc + nodeDiskSize *prometheus.Desc gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc @@ -76,6 +78,15 @@ func New(src Source) *Collector { "A node's total RAM, 0 when it does not report one.", node, nil), nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds", "Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil), + // Only the disk a node fills itself: space on a shared filer is not + // this node's to spend, and summing it per node would have a fleet of + // twenty each report the same free terabyte. Absent rather than zero + // when nothing was reported, so an alert on "free is low" does not + // fire for every node that was never asked about its disk. + nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes", + "Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil), + nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes", + "Size of those filesystems, so free space can be read as a fraction.", node, nil), gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes", "A GPU's framebuffer size.", gpu, nil), @@ -111,6 +122,7 @@ func New(src Source) *Collector { func (c *Collector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, + c.nodeDiskFree, c.nodeDiskSize, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -143,6 +155,15 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { float64(n.MemoryTotalMB)*1024*1024, n.ID) ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue, now.Sub(n.LastSeen).Seconds(), n.ID) + // Nothing reported means nothing is known, so no sample at all. A zero + // here would read as a full disk and page someone about a node that + // was simply never asked. + if free, total := n.LocalDisk(); total > 0 { + ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue, + float64(free), n.ID) + ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue, + float64(total), n.ID) + } byClass := map[string]int{} for _, g := range n.GPUs { diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 24830798..a58a3e9f 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -84,6 +84,12 @@ type Node struct { // Empty means the node was not asked to look, which is not the same as // holding none — so it constrains nothing. Weights []discovery.LocalWeights `json:"weights,omitempty"` + // Storage is the disk those weights sit on, one entry per filesystem. + // + // Empty means the node was not asked, not that it has no disk — so an + // agent too old to report it, or one with no model directories, is left + // unconstrained rather than treated as full. + Storage []discovery.Storage `json:"storage,omitempty"` GPUs []discovery.GPU `json:"gpus"` // Fabrics is empty on a node without IB. Such a node is its own @@ -310,6 +316,28 @@ func (s *Store) FreeGPUs(nodeID string) []int { return out } +// LocalDisk sums the space on filesystems this node fills itself. +// +// Shared filesystems are left out on purpose. Space on a filer belongs to +// everyone mounting it, and counting it per node would let twenty nodes each +// believe they have the same free terabyte — then place twenty models into it. +// A node whose weights are all on a shared mount therefore reports no local +// disk, which is correct: it has none to spend. +// +// Both zero means nothing is known — an agent too old to report storage, or a +// node with no model directories — and callers must read that as unknown +// rather than as a full disk. +func (n Node) LocalDisk() (free, total int64) { + for _, s := range n.Storage { + if s.Shared { + continue + } + free += s.FreeBytes + total += s.TotalBytes + } + return free, total +} + // HasWeights reports whether this node can see the given source. // // An exact path match, deliberately. A source that is not a path — a model id diff --git a/internal/control/inventory/inventory_test.go b/internal/control/inventory/inventory_test.go index 41eeef93..f6e72dfc 100644 --- a/internal/control/inventory/inventory_test.go +++ b/internal/control/inventory/inventory_test.go @@ -248,3 +248,42 @@ func TestConcurrentAccess(t *testing.T) { } wg.Wait() } + +func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) { + // Twenty nodes mounting one filer have 500 GB between them, not ten + // terabytes. Counting a shared mount per node is how a scheduler places + // twenty models into space for one. + n := Node{Storage: []discovery.Storage{ + {Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true}, + {Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30}, + }} + free, total := n.LocalDisk() + if free != 20<<30 || total != 60<<30 { + t.Errorf("shared space was counted as local: free=%d total=%d", free, total) + } +} + +func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) { + // An agent too old to report storage, and a node with no model directories, + // both send nothing. Reading that as a full disk would take every such node + // out of placement for a reason that is not true. + free, total := Node{}.LocalDisk() + if free != 0 || total != 0 { + t.Errorf("a node with no storage reported %d/%d", free, total) + } + // The contract callers rely on: total of zero is the signal for "unknown", + // which is why nothing may read free alone. + if total != 0 { + t.Error("callers cannot distinguish unknown from full") + } +} + +func TestLocalDiskSumsSeveralDisks(t *testing.T) { + n := Node{Storage: []discovery.Storage{ + {Path: "/a", FreeBytes: 10, TotalBytes: 100}, + {Path: "/b", FreeBytes: 5, TotalBytes: 50}, + }} + if free, total := n.LocalDisk(); free != 15 || total != 150 { + t.Errorf("free=%d total=%d, want 15/150", free, total) + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 62038322..b2556f1b 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -98,12 +98,16 @@ requested whenever a fallback served it.

Nodes

{{if .Nodes}} - +
NodeStateGPUsRAMClassesFabricsModelsReplicasLast seen
{{range .Nodes}} + {{/* Free out of total, because the first is meaningless without the + second, and an em dash where nothing was reported: an agent that + was never asked about its disk must not read as a full one. */}} +
NodeStateGPUsRAMDiskClassesFabricsModelsReplicasLast seen
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} {{.State}} {{.GPUFree}}/{{.GPUTotal}} {{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}} {{join .Classes}} {{join .Fabrics}} {{if .Models}}{{len .Models}}{{else}}—{{end}}