diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go
index 11c560cc..d6d8ecf1 100644
--- a/api/gen/llmgw/v1/agent.pb.go
+++ b/api/gen/llmgw/v1/agent.pb.go
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -356,7 +356,16 @@ type Register struct {
// Empty means the node was not asked to look, which is not the same as having
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
- Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"`
+ Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"`
+ // storage is the disk behind those weights, one entry per filesystem.
+ //
+ // Reported so an operator can see a node filling up before a fetch dies at
+ // ninety percent, and so placement can refuse a node that cannot hold a
+ // model rather than discovering it during the download.
+ //
+ // Empty means the node was not asked to look, exactly as for weights, and
+ // constrains nothing.
+ Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -468,6 +477,91 @@ func (x *Register) GetWeights() []*LocalWeights {
return nil
}
+func (x *Register) GetStorage() []*Storage {
+ if x != nil {
+ return x.Storage
+ }
+ return nil
+}
+
+// Storage is one filesystem a node keeps weights on.
+//
+// Per filesystem rather than per configured directory: free space belongs to
+// the disk, and two model directories on one disk are one pool of bytes.
+type Storage struct {
+ state protoimpl.MessageState `protogen:"open.v1"`
+ // path is one directory on this filesystem, so the operator recognises it.
+ Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
+ // free_bytes is what a non-root process can write — not the root reserve,
+ // which is space the agent cannot have.
+ FreeBytes int64 `protobuf:"varint,2,opt,name=free_bytes,json=freeBytes,proto3" json:"free_bytes,omitempty"`
+ TotalBytes int64 `protobuf:"varint,3,opt,name=total_bytes,json=totalBytes,proto3" json:"total_bytes,omitempty"`
+ // shared marks a filesystem the node did not fill. Free space on one is
+ // shared with every other node using it, so two nodes each reporting 500 GB
+ // free on the same filer have 500 GB between them and not a terabyte.
+ Shared bool `protobuf:"varint,4,opt,name=shared,proto3" json:"shared,omitempty"`
+ unknownFields protoimpl.UnknownFields
+ sizeCache protoimpl.SizeCache
+}
+
+func (x *Storage) Reset() {
+ *x = Storage{}
+ mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+}
+
+func (x *Storage) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Storage) ProtoMessage() {}
+
+func (x *Storage) ProtoReflect() protoreflect.Message {
+ mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ if x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use Storage.ProtoReflect.Descriptor instead.
+func (*Storage) Descriptor() ([]byte, []int) {
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *Storage) GetPath() string {
+ if x != nil {
+ return x.Path
+ }
+ return ""
+}
+
+func (x *Storage) GetFreeBytes() int64 {
+ if x != nil {
+ return x.FreeBytes
+ }
+ return 0
+}
+
+func (x *Storage) GetTotalBytes() int64 {
+ if x != nil {
+ return x.TotalBytes
+ }
+ return 0
+}
+
+func (x *Storage) GetShared() bool {
+ if x != nil {
+ return x.Shared
+ }
+ return false
+}
+
// RunningShard describes an engine already running on a node.
//
// It carries enough for the control plane to rebuild the replica it belongs to,
@@ -498,7 +592,7 @@ type RunningShard struct {
func (x *RunningShard) Reset() {
*x = RunningShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[4]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -510,7 +604,7 @@ func (x *RunningShard) String() string {
func (*RunningShard) ProtoMessage() {}
func (x *RunningShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[3]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[4]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -523,7 +617,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use RunningShard.ProtoReflect.Descriptor instead.
func (*RunningShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
}
func (x *RunningShard) GetReplicaId() string {
@@ -653,7 +747,7 @@ type GPU struct {
func (x *GPU) Reset() {
*x = GPU{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[4]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[5]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -665,7 +759,7 @@ func (x *GPU) String() string {
func (*GPU) ProtoMessage() {}
func (x *GPU) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[4]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[5]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -678,7 +772,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message {
// Deprecated: Use GPU.ProtoReflect.Descriptor instead.
func (*GPU) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
}
func (x *GPU) GetIndex() int32 {
@@ -800,7 +894,7 @@ type FabricAttachment struct {
func (x *FabricAttachment) Reset() {
*x = FabricAttachment{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[5]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -812,7 +906,7 @@ func (x *FabricAttachment) String() string {
func (*FabricAttachment) ProtoMessage() {}
func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[5]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -825,7 +919,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message {
// Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead.
func (*FabricAttachment) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
}
func (x *FabricAttachment) GetFabric() string {
@@ -867,7 +961,7 @@ type LocalWeights struct {
func (x *LocalWeights) Reset() {
*x = LocalWeights{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[6]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -879,7 +973,7 @@ func (x *LocalWeights) String() string {
func (*LocalWeights) ProtoMessage() {}
func (x *LocalWeights) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[6]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -892,7 +986,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message {
// Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead.
func (*LocalWeights) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
}
func (x *LocalWeights) GetPath() string {
@@ -929,7 +1023,7 @@ type Heartbeat struct {
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[7]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -941,7 +1035,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[7]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -954,7 +1048,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
}
func (x *Heartbeat) GetNodeId() string {
@@ -996,7 +1090,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[8]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1008,7 +1102,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[8]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1021,7 +1115,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1102,7 +1196,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[9]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1114,7 +1208,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[9]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1127,7 +1221,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
}
func (x *StartShard) GetReplicaId() string {
@@ -1247,7 +1341,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1259,7 +1353,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[10]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1272,7 +1366,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *EngineProfile) GetName() string {
@@ -1362,7 +1456,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1374,7 +1468,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[11]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1387,7 +1481,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *StopShard) GetShardId() string {
@@ -1407,7 +1501,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1419,7 +1513,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[12]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1432,7 +1526,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *DrainShard) GetShardId() string {
@@ -1454,7 +1548,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
- mi := &file_llmgw_v1_agent_proto_msgTypes[13]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1466,7 +1560,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
- mi := &file_llmgw_v1_agent_proto_msgTypes[13]
+ mi := &file_llmgw_v1_agent_proto_msgTypes[14]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1479,7 +1573,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
- return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
+ return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
}
func (x *Ack) GetMessageId() string {
@@ -1518,7 +1612,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
- "\x03msg\"\xfa\x04\n" +
+ "\x03msg\"\xa7\x05\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -1531,13 +1625,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
"\x0fmemory_total_mb\x18\n" +
" \x01(\x03R\rmemoryTotalMb\x120\n" +
- "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" +
+ "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" +
+ "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
- "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x97\x03\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" +
+ "\aStorage\x12\x12\n" +
+ "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
+ "\n" +
+ "free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" +
+ "\vtotal_bytes\x18\x03 \x01(\x03R\n" +
+ "totalBytes\x12\x16\n" +
+ "\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" +
"\fRunningShard\x12\x1d\n" +
"\n" +
"replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" +
@@ -1675,52 +1777,54 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17)
+var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
(*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage
(*Register)(nil), // 3: llmgw.v1.Register
- (*RunningShard)(nil), // 4: llmgw.v1.RunningShard
- (*GPU)(nil), // 5: llmgw.v1.GPU
- (*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment
- (*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights
- (*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat
- (*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus
- (*StartShard)(nil), // 10: llmgw.v1.StartShard
- (*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile
- (*StopShard)(nil), // 12: llmgw.v1.StopShard
- (*DrainShard)(nil), // 13: llmgw.v1.DrainShard
- (*Ack)(nil), // 14: llmgw.v1.Ack
- nil, // 15: llmgw.v1.Register.LabelsEntry
- nil, // 16: llmgw.v1.Register.EngineVersionsEntry
- nil, // 17: llmgw.v1.EngineProfile.EnvEntry
+ (*Storage)(nil), // 4: llmgw.v1.Storage
+ (*RunningShard)(nil), // 5: llmgw.v1.RunningShard
+ (*GPU)(nil), // 6: llmgw.v1.GPU
+ (*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment
+ (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights
+ (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat
+ (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus
+ (*StartShard)(nil), // 11: llmgw.v1.StartShard
+ (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile
+ (*StopShard)(nil), // 13: llmgw.v1.StopShard
+ (*DrainShard)(nil), // 14: llmgw.v1.DrainShard
+ (*Ack)(nil), // 15: llmgw.v1.Ack
+ nil, // 16: llmgw.v1.Register.LabelsEntry
+ nil, // 17: llmgw.v1.Register.EngineVersionsEntry
+ nil, // 18: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
- 8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
- 9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
- 10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
- 12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
- 13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
- 14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
- 15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
- 5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
- 6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
- 16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
- 4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
- 7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
- 0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
- 0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
- 11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
- 17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
- 1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
- 2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
- 18, // [18:19] is the sub-list for method output_type
- 17, // [17:18] is the sub-list for method input_type
- 17, // [17:17] is the sub-list for extension type_name
- 17, // [17:17] is the sub-list for extension extendee
- 0, // [0:17] is the sub-list for field type_name
+ 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
+ 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
+ 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
+ 13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
+ 14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
+ 15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
+ 16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
+ 6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
+ 7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
+ 17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
+ 5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
+ 8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
+ 4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
+ 0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
+ 0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
+ 12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
+ 18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
+ 1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
+ 2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
+ 19, // [19:20] is the sub-list for method output_type
+ 18, // [18:19] is the sub-list for method input_type
+ 18, // [18:18] is the sub-list for extension type_name
+ 18, // [18:18] is the sub-list for extension extendee
+ 0, // [0:18] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -1745,7 +1849,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
- NumMessages: 17,
+ NumMessages: 18,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto
index 39604c70..8fc6c2f7 100644
--- a/api/proto/llmgw/v1/agent.proto
+++ b/api/proto/llmgw/v1/agent.proto
@@ -90,6 +90,33 @@ message Register {
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
repeated LocalWeights weights = 11;
+
+ // storage is the disk behind those weights, one entry per filesystem.
+ //
+ // Reported so an operator can see a node filling up before a fetch dies at
+ // ninety percent, and so placement can refuse a node that cannot hold a
+ // model rather than discovering it during the download.
+ //
+ // Empty means the node was not asked to look, exactly as for weights, and
+ // constrains nothing.
+ repeated Storage storage = 12;
+}
+
+// Storage is one filesystem a node keeps weights on.
+//
+// Per filesystem rather than per configured directory: free space belongs to
+// the disk, and two model directories on one disk are one pool of bytes.
+message Storage {
+ // path is one directory on this filesystem, so the operator recognises it.
+ string path = 1;
+ // free_bytes is what a non-root process can write — not the root reserve,
+ // which is space the agent cannot have.
+ int64 free_bytes = 2;
+ int64 total_bytes = 3;
+ // shared marks a filesystem the node did not fill. Free space on one is
+ // shared with every other node using it, so two nodes each reporting 500 GB
+ // free on the same filer have 500 GB between them and not a terabyte.
+ bool shared = 4;
}
// RunningShard describes an engine already running on a node.
diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go
index 812acbe3..275cc31d 100644
--- a/internal/agent/discovery/discover.go
+++ b/internal/agent/discovery/discover.go
@@ -51,6 +51,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
// holds has nothing to do with its cards, and a node with none is exactly
// where a CPU profile will look for them.
inv.Weights = ScanWeights(opts.ModelDirs)
+ // And the disk they sit on. A node can hold a model today and be unable to
+ // take the next one, which the file list alone never shows.
+ inv.Storage = ScanStorage(opts.ModelDirs)
if !opts.SkipGPUs {
switch {
diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go
index cd94b5c1..4d475bf1 100644
--- a/internal/agent/discovery/gpu.go
+++ b/internal/agent/discovery/gpu.go
@@ -114,6 +114,9 @@ type Inventory struct {
// Weights are the model files this node can see. Empty means it was not
// asked to look.
Weights []LocalWeights `json:"weights,omitempty"`
+ // Storage is the disk behind those weights, one entry per filesystem.
+ // Empty means the same thing it means for Weights: not asked.
+ Storage []Storage `json:"storage,omitempty"`
}
// Classes returns the distinct GPU classes present, sorted.
diff --git a/internal/agent/discovery/storage.go b/internal/agent/discovery/storage.go
new file mode 100644
index 00000000..fc8a9881
--- /dev/null
+++ b/internal/agent/discovery/storage.go
@@ -0,0 +1,90 @@
+package discovery
+
+import (
+ "os"
+ "syscall"
+)
+
+// Storage is one filesystem a node keeps weights on.
+//
+// One entry per filesystem rather than per configured directory, because free
+// space belongs to the disk and not to the path. Two model directories on the
+// same disk are one pool of bytes, and reporting them separately would let the
+// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24.
+type Storage struct {
+ // Path is one directory on this filesystem — the first one configured that
+ // landed here, so the operator recognises it.
+ Path string `json:"path"`
+ // FreeBytes is what a non-root process can still write. Deliberately not
+ // the root reserve: the agent does not run as root, so the reserve is space
+ // it cannot have.
+ FreeBytes int64 `json:"free_bytes"`
+ // TotalBytes is the filesystem's size, so a reading of "8 GB free" can be
+ // told apart on a 10 TB filer and on a 10 GB root partition.
+ TotalBytes int64 `json:"total_bytes"`
+ // Shared marks a filesystem the node did not have to fill. Free space on
+ // one is shared with every other node using it, so two nodes reporting
+ // 500 GB free on the same filer have 500 GB between them, not a terabyte.
+ Shared bool `json:"shared,omitempty"`
+}
+
+// ScanStorage reports the filesystems behind the given model directories.
+//
+// Empty in, empty out, and a directory that cannot be read is skipped rather
+// than reported as full: a mount that has not come back yet looks exactly like
+// a missing directory, and claiming zero free bytes would take the node out of
+// placement for a reason that is not true. Absent is not zero here, as
+// everywhere else in this system.
+func ScanStorage(dirs []string) []Storage {
+ var out []Storage
+ seen := map[uint64]int{} // filesystem id -> index in out
+
+ for _, dir := range dirs {
+ if dir == "" {
+ continue
+ }
+ dev, ok := filesystemID(dir)
+ if !ok {
+ continue
+ }
+ if i, dup := seen[dev]; dup {
+ // Same disk, already counted. Keep the first path: it is the one
+ // the operator wrote first and so the one they will recognise.
+ _ = i
+ continue
+ }
+
+ var st syscall.Statfs_t
+ if err := syscall.Statfs(dir, &st); err != nil {
+ continue
+ }
+ // Bsize is int32 on some platforms and int64 on others, which is why
+ // every field here is widened explicitly rather than assumed.
+ block := int64(st.Bsize)
+ seen[dev] = len(out)
+ out = append(out, Storage{
+ Path: dir,
+ FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose
+ TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose
+ Shared: isNetworkFS(dir),
+ })
+ }
+ return out
+}
+
+// filesystemID identifies the disk a path is on.
+//
+// By device number rather than by the statfs filesystem id, because the field
+// holding that id is spelled differently on Linux and on macOS and this has to
+// compile on both — the agent runs on one and is developed on the other.
+func filesystemID(path string) (uint64, bool) {
+ fi, err := os.Stat(path)
+ if err != nil {
+ return 0, false
+ }
+ st, ok := fi.Sys().(*syscall.Stat_t)
+ if !ok {
+ return 0, false
+ }
+ return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux
+}
diff --git a/internal/agent/discovery/storage_test.go b/internal/agent/discovery/storage_test.go
new file mode 100644
index 00000000..b8f7a090
--- /dev/null
+++ b/internal/agent/discovery/storage_test.go
@@ -0,0 +1,63 @@
+package discovery
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) {
+ // The reason this matters: free space belongs to the disk, not to the path.
+ // Counting two directories separately lets the control plane add 24 GB to
+ // 24 GB and place a 40 GB model on a node that has 24.
+ base := t.TempDir()
+ a := filepath.Join(base, "models-a")
+ b := filepath.Join(base, "models-b")
+ for _, d := range []string{a, b} {
+ if err := os.Mkdir(d, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ got := ScanStorage([]string{a, b})
+ if len(got) != 1 {
+ t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got)
+ }
+ if got[0].Path != a {
+ t.Errorf("kept %q rather than the first directory configured", got[0].Path)
+ }
+}
+
+func TestStorageReportsSomethingPlausible(t *testing.T) {
+ got := ScanStorage([]string{t.TempDir()})
+ if len(got) != 1 {
+ t.Fatalf("a real directory reported %d filesystems", len(got))
+ }
+ s := got[0]
+ if s.TotalBytes <= 0 {
+ t.Error("a filesystem with no size at all")
+ }
+ if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes {
+ t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes)
+ }
+}
+
+func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) {
+ // A mount that has not come back yet looks exactly like a missing
+ // directory. Reporting it as zero free bytes would take the node out of
+ // placement for a reason that is not true; reporting nothing says only
+ // that nothing is known, which is the honest answer.
+ got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")})
+ if len(got) != 0 {
+ t.Errorf("a missing directory was reported as a filesystem: %+v", got)
+ }
+}
+
+func TestNoDirectoriesConstrainNothing(t *testing.T) {
+ if got := ScanStorage(nil); len(got) != 0 {
+ t.Errorf("scanning nothing found %+v", got)
+ }
+ if got := ScanStorage([]string{""}); len(got) != 0 {
+ t.Errorf("an empty path found %+v", got)
+ }
+}
diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go
index cce20d30..b98c888b 100644
--- a/internal/cli/ctl.go
+++ b/internal/cli/ctl.go
@@ -149,10 +149,11 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
- fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
+ fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
for _, n := range nodes {
- fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n",
+ fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
+ diskColumn(n.DiskFreeGB, n.DiskTotalGB),
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
}
_ = w.Flush()
@@ -221,6 +222,23 @@ func memColumn(gb int) string {
return fmt.Sprintf("%dG", gb)
}
+// diskColumn renders free space against the size it is free out of.
+//
+// Both, because they answer different questions and the first is useless
+// without the second: 8 GB free is comfortable on a 10 TB filer and nearly
+// full on a 10 GB root partition, and it is the second case that stops a
+// model from being placed.
+//
+// A dash for a node that reported nothing, which an older agent and a node
+// with no model directories both do. It has to read as "not known" rather
+// than as "no space", or every unconfigured node looks like it is full.
+func diskColumn(freeGB, totalGB int) string {
+ if totalGB <= 0 {
+ return "-"
+ }
+ return fmt.Sprintf("%dG/%dG", freeGB, totalGB)
+}
+
func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "models",
diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go
index b9f09c16..2b9792ac 100644
--- a/internal/control/adminapi/api.go
+++ b/internal/control/adminapi/api.go
@@ -62,9 +62,21 @@ type NodeView struct {
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
- Models []string `json:"models_on_disk,omitempty"`
- Classes []string `json:"gpu_classes,omitempty"`
- Fabrics []string `json:"fabrics,omitempty"`
+ Models []string `json:"models_on_disk,omitempty"`
+ // DiskFreeGB and DiskTotalGB are the filesystems holding those weights,
+ // summed. Zero means the agent did not report any, which an older agent
+ // and a node with no model directories both do — so it reads as unknown
+ // rather than as a full disk.
+ //
+ // Summed because an operator asking "can this node take another model"
+ // wants one number; the per-filesystem detail is in the node's own record
+ // for anyone who needs it. Shared filesystems are left out of the sum: the
+ // space on a filer is not this node's to spend, and adding it would let a
+ // fleet of twenty nodes each believe it has the same free terabyte.
+ DiskFreeGB int `json:"disk_free_gb,omitempty"`
+ DiskTotalGB int `json:"disk_total_gb,omitempty"`
+ Classes []string `json:"gpu_classes,omitempty"`
+ Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -166,6 +178,10 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
+ if free, total := n.LocalDisk(); total > 0 {
+ v.DiskFreeGB = int(free / (1 << 30))
+ v.DiskTotalGB = int(total / (1 << 30))
+ }
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go
index e8d00089..4c23a663 100644
--- a/internal/control/agentsvc/convert.go
+++ b/internal/control/agentsvc/convert.go
@@ -30,6 +30,14 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
})
}
+ for _, s := range r.GetStorage() {
+ n.Storage = append(n.Storage, discovery.Storage{
+ Path: s.GetPath(),
+ FreeBytes: s.GetFreeBytes(),
+ TotalBytes: s.GetTotalBytes(),
+ Shared: s.GetShared(),
+ })
+ }
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: int(g.GetIndex()),
@@ -75,6 +83,14 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
})
}
+ for _, s := range inv.Storage {
+ r.Storage = append(r.Storage, &llmgwv1.Storage{
+ Path: s.Path,
+ FreeBytes: s.FreeBytes,
+ TotalBytes: s.TotalBytes,
+ Shared: s.Shared,
+ })
+ }
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
Index: int32(g.Index),
diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go
index a9b39bf3..b5b9c9ae 100644
--- a/internal/control/fleetmetrics/fleetmetrics.go
+++ b/internal/control/fleetmetrics/fleetmetrics.go
@@ -45,6 +45,8 @@ type Collector struct {
nodeGPUsFree *prometheus.Desc
nodeMemory *prometheus.Desc
nodeSeen *prometheus.Desc
+ nodeDiskFree *prometheus.Desc
+ nodeDiskSize *prometheus.Desc
gpuMemoryTotal *prometheus.Desc
gpuMemoryUsed *prometheus.Desc
@@ -76,6 +78,15 @@ func New(src Source) *Collector {
"A node's total RAM, 0 when it does not report one.", node, nil),
nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds",
"Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil),
+ // Only the disk a node fills itself: space on a shared filer is not
+ // this node's to spend, and summing it per node would have a fleet of
+ // twenty each report the same free terabyte. Absent rather than zero
+ // when nothing was reported, so an alert on "free is low" does not
+ // fire for every node that was never asked about its disk.
+ nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes",
+ "Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil),
+ nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes",
+ "Size of those filesystems, so free space can be read as a fraction.", node, nil),
gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes",
"A GPU's framebuffer size.", gpu, nil),
@@ -111,6 +122,7 @@ func New(src Source) *Collector {
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
for _, d := range []*prometheus.Desc{
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
+ c.nodeDiskFree, c.nodeDiskSize,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -143,6 +155,15 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
float64(n.MemoryTotalMB)*1024*1024, n.ID)
ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue,
now.Sub(n.LastSeen).Seconds(), n.ID)
+ // Nothing reported means nothing is known, so no sample at all. A zero
+ // here would read as a full disk and page someone about a node that
+ // was simply never asked.
+ if free, total := n.LocalDisk(); total > 0 {
+ ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue,
+ float64(free), n.ID)
+ ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue,
+ float64(total), n.ID)
+ }
byClass := map[string]int{}
for _, g := range n.GPUs {
diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go
index 24830798..a58a3e9f 100644
--- a/internal/control/inventory/inventory.go
+++ b/internal/control/inventory/inventory.go
@@ -84,6 +84,12 @@ type Node struct {
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
+ // Storage is the disk those weights sit on, one entry per filesystem.
+ //
+ // Empty means the node was not asked, not that it has no disk — so an
+ // agent too old to report it, or one with no model directories, is left
+ // unconstrained rather than treated as full.
+ Storage []discovery.Storage `json:"storage,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
@@ -310,6 +316,28 @@ func (s *Store) FreeGPUs(nodeID string) []int {
return out
}
+// LocalDisk sums the space on filesystems this node fills itself.
+//
+// Shared filesystems are left out on purpose. Space on a filer belongs to
+// everyone mounting it, and counting it per node would let twenty nodes each
+// believe they have the same free terabyte — then place twenty models into it.
+// A node whose weights are all on a shared mount therefore reports no local
+// disk, which is correct: it has none to spend.
+//
+// Both zero means nothing is known — an agent too old to report storage, or a
+// node with no model directories — and callers must read that as unknown
+// rather than as a full disk.
+func (n Node) LocalDisk() (free, total int64) {
+ for _, s := range n.Storage {
+ if s.Shared {
+ continue
+ }
+ free += s.FreeBytes
+ total += s.TotalBytes
+ }
+ return free, total
+}
+
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
diff --git a/internal/control/inventory/inventory_test.go b/internal/control/inventory/inventory_test.go
index 41eeef93..f6e72dfc 100644
--- a/internal/control/inventory/inventory_test.go
+++ b/internal/control/inventory/inventory_test.go
@@ -248,3 +248,42 @@ func TestConcurrentAccess(t *testing.T) {
}
wg.Wait()
}
+
+func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) {
+ // Twenty nodes mounting one filer have 500 GB between them, not ten
+ // terabytes. Counting a shared mount per node is how a scheduler places
+ // twenty models into space for one.
+ n := Node{Storage: []discovery.Storage{
+ {Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true},
+ {Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30},
+ }}
+ free, total := n.LocalDisk()
+ if free != 20<<30 || total != 60<<30 {
+ t.Errorf("shared space was counted as local: free=%d total=%d", free, total)
+ }
+}
+
+func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) {
+ // An agent too old to report storage, and a node with no model directories,
+ // both send nothing. Reading that as a full disk would take every such node
+ // out of placement for a reason that is not true.
+ free, total := Node{}.LocalDisk()
+ if free != 0 || total != 0 {
+ t.Errorf("a node with no storage reported %d/%d", free, total)
+ }
+ // The contract callers rely on: total of zero is the signal for "unknown",
+ // which is why nothing may read free alone.
+ if total != 0 {
+ t.Error("callers cannot distinguish unknown from full")
+ }
+}
+
+func TestLocalDiskSumsSeveralDisks(t *testing.T) {
+ n := Node{Storage: []discovery.Storage{
+ {Path: "/a", FreeBytes: 10, TotalBytes: 100},
+ {Path: "/b", FreeBytes: 5, TotalBytes: 50},
+ }}
+ if free, total := n.LocalDisk(); free != 15 || total != 150 {
+ t.Errorf("free=%d total=%d, want 15/150", free, total)
+ }
+}
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html
index 62038322..b2556f1b 100644
--- a/internal/web/templates/dashboard.html
+++ b/internal/web/templates/dashboard.html
@@ -98,12 +98,16 @@ requested whenever a fallback served it.
Nodes
{{if .Nodes}}
-| Node | State | GPUs | RAM | Classes | Fabrics | Models | Replicas | Last seen |
+| Node | State | GPUs | RAM | Disk | Classes | Fabrics | Models | Replicas | Last seen |
{{range .Nodes}}
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} |
{{.State}} |
{{.GPUFree}}/{{.GPUTotal}} |
{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}} |
+ {{/* Free out of total, because the first is meaningless without the
+ second, and an em dash where nothing was reported: an agent that
+ was never asked about its disk must not read as a full one. */}}
+ {{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}} |
{{join .Classes}} |
{{join .Fabrics}} |
{{if .Models}}{{len .Models}}{{else}}—{{end}} |