diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 11c560cc..d6d8ecf1 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9, 0} } // AgentMessage is anything a node tells the control plane. @@ -356,7 +356,16 @@ type Register struct { // Empty means the node was not asked to look, which is not the same as having // nothing — an agent with no model directories configured behaves exactly as // every agent did before this existed, and constrains nothing. - Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"` + Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"` + // storage is the disk behind those weights, one entry per filesystem. + // + // Reported so an operator can see a node filling up before a fetch dies at + // ninety percent, and so placement can refuse a node that cannot hold a + // model rather than discovering it during the download. + // + // Empty means the node was not asked to look, exactly as for weights, and + // constrains nothing. + Storage []*Storage `protobuf:"bytes,12,rep,name=storage,proto3" json:"storage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -468,6 +477,91 @@ func (x *Register) GetWeights() []*LocalWeights { return nil } +func (x *Register) GetStorage() []*Storage { + if x != nil { + return x.Storage + } + return nil +} + +// Storage is one filesystem a node keeps weights on. +// +// Per filesystem rather than per configured directory: free space belongs to +// the disk, and two model directories on one disk are one pool of bytes. +type Storage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // path is one directory on this filesystem, so the operator recognises it. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // free_bytes is what a non-root process can write — not the root reserve, + // which is space the agent cannot have. + FreeBytes int64 `protobuf:"varint,2,opt,name=free_bytes,json=freeBytes,proto3" json:"free_bytes,omitempty"` + TotalBytes int64 `protobuf:"varint,3,opt,name=total_bytes,json=totalBytes,proto3" json:"total_bytes,omitempty"` + // shared marks a filesystem the node did not fill. Free space on one is + // shared with every other node using it, so two nodes each reporting 500 GB + // free on the same filer have 500 GB between them and not a terabyte. + Shared bool `protobuf:"varint,4,opt,name=shared,proto3" json:"shared,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Storage) Reset() { + *x = Storage{} + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Storage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Storage) ProtoMessage() {} + +func (x *Storage) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Storage.ProtoReflect.Descriptor instead. +func (*Storage) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *Storage) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *Storage) GetFreeBytes() int64 { + if x != nil { + return x.FreeBytes + } + return 0 +} + +func (x *Storage) GetTotalBytes() int64 { + if x != nil { + return x.TotalBytes + } + return 0 +} + +func (x *Storage) GetShared() bool { + if x != nil { + return x.Shared + } + return false +} + // RunningShard describes an engine already running on a node. // // It carries enough for the control plane to rebuild the replica it belongs to, @@ -498,7 +592,7 @@ type RunningShard struct { func (x *RunningShard) Reset() { *x = RunningShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -510,7 +604,7 @@ func (x *RunningShard) String() string { func (*RunningShard) ProtoMessage() {} func (x *RunningShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[3] + mi := &file_llmgw_v1_agent_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -523,7 +617,7 @@ func (x *RunningShard) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningShard.ProtoReflect.Descriptor instead. func (*RunningShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{3} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} } func (x *RunningShard) GetReplicaId() string { @@ -653,7 +747,7 @@ type GPU struct { func (x *GPU) Reset() { *x = GPU{} - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -665,7 +759,7 @@ func (x *GPU) String() string { func (*GPU) ProtoMessage() {} func (x *GPU) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[4] + mi := &file_llmgw_v1_agent_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -678,7 +772,7 @@ func (x *GPU) ProtoReflect() protoreflect.Message { // Deprecated: Use GPU.ProtoReflect.Descriptor instead. func (*GPU) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{4} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} } func (x *GPU) GetIndex() int32 { @@ -800,7 +894,7 @@ type FabricAttachment struct { func (x *FabricAttachment) Reset() { *x = FabricAttachment{} - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -812,7 +906,7 @@ func (x *FabricAttachment) String() string { func (*FabricAttachment) ProtoMessage() {} func (x *FabricAttachment) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[5] + mi := &file_llmgw_v1_agent_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -825,7 +919,7 @@ func (x *FabricAttachment) ProtoReflect() protoreflect.Message { // Deprecated: Use FabricAttachment.ProtoReflect.Descriptor instead. func (*FabricAttachment) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{5} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} } func (x *FabricAttachment) GetFabric() string { @@ -867,7 +961,7 @@ type LocalWeights struct { func (x *LocalWeights) Reset() { *x = LocalWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -879,7 +973,7 @@ func (x *LocalWeights) String() string { func (*LocalWeights) ProtoMessage() {} func (x *LocalWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -892,7 +986,7 @@ func (x *LocalWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. func (*LocalWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *LocalWeights) GetPath() string { @@ -929,7 +1023,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -941,7 +1035,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -954,7 +1048,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *Heartbeat) GetNodeId() string { @@ -996,7 +1090,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1008,7 +1102,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1021,7 +1115,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *ShardStatus) GetNodeId() string { @@ -1102,7 +1196,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1114,7 +1208,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1127,7 +1221,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *StartShard) GetReplicaId() string { @@ -1247,7 +1341,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1259,7 +1353,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1272,7 +1366,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *EngineProfile) GetName() string { @@ -1362,7 +1456,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1374,7 +1468,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1387,7 +1481,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *StopShard) GetShardId() string { @@ -1407,7 +1501,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1419,7 +1513,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1432,7 +1526,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *DrainShard) GetShardId() string { @@ -1454,7 +1548,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1466,7 +1560,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1479,7 +1573,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *Ack) GetMessageId() string { @@ -1518,7 +1612,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + - "\x03msg\"\xfa\x04\n" + + "\x03msg\"\xa7\x05\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -1531,13 +1625,21 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" + "\x0fmemory_total_mb\x18\n" + " \x01(\x03R\rmemoryTotalMb\x120\n" + - "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" + + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12+\n" + + "\astorage\x18\f \x03(\v2\x11.llmgw.v1.StorageR\astorage\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + "\x13EngineVersionsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x97\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"u\n" + + "\aStorage\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + + "\n" + + "free_bytes\x18\x02 \x01(\x03R\tfreeBytes\x12\x1f\n" + + "\vtotal_bytes\x18\x03 \x01(\x03R\n" + + "totalBytes\x12\x16\n" + + "\x06shared\x18\x04 \x01(\bR\x06shared\"\x97\x03\n" + "\fRunningShard\x12\x1d\n" + "\n" + "replica_id\x18\x01 \x01(\tR\treplicaId\x12\x19\n" + @@ -1675,52 +1777,54 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage (*ControlMessage)(nil), // 2: llmgw.v1.ControlMessage (*Register)(nil), // 3: llmgw.v1.Register - (*RunningShard)(nil), // 4: llmgw.v1.RunningShard - (*GPU)(nil), // 5: llmgw.v1.GPU - (*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment - (*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights - (*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat - (*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus - (*StartShard)(nil), // 10: llmgw.v1.StartShard - (*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile - (*StopShard)(nil), // 12: llmgw.v1.StopShard - (*DrainShard)(nil), // 13: llmgw.v1.DrainShard - (*Ack)(nil), // 14: llmgw.v1.Ack - nil, // 15: llmgw.v1.Register.LabelsEntry - nil, // 16: llmgw.v1.Register.EngineVersionsEntry - nil, // 17: llmgw.v1.EngineProfile.EnvEntry + (*Storage)(nil), // 4: llmgw.v1.Storage + (*RunningShard)(nil), // 5: llmgw.v1.RunningShard + (*GPU)(nil), // 6: llmgw.v1.GPU + (*FabricAttachment)(nil), // 7: llmgw.v1.FabricAttachment + (*LocalWeights)(nil), // 8: llmgw.v1.LocalWeights + (*Heartbeat)(nil), // 9: llmgw.v1.Heartbeat + (*ShardStatus)(nil), // 10: llmgw.v1.ShardStatus + (*StartShard)(nil), // 11: llmgw.v1.StartShard + (*EngineProfile)(nil), // 12: llmgw.v1.EngineProfile + (*StopShard)(nil), // 13: llmgw.v1.StopShard + (*DrainShard)(nil), // 14: llmgw.v1.DrainShard + (*Ack)(nil), // 15: llmgw.v1.Ack + nil, // 16: llmgw.v1.Register.LabelsEntry + nil, // 17: llmgw.v1.Register.EngineVersionsEntry + nil, // 18: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry - 5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU - 6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry - 4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights - 0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 18, // [18:19] is the sub-list for method output_type - 17, // [17:18] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 9, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 10, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 11, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 13, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 14, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 15, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 16, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 6, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU + 7, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment + 17, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 5, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard + 8, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 4, // 13: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage + 0, // 14: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 0, // 15: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 12, // 16: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 18, // 17: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 18: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 19: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 19, // [19:20] is the sub-list for method output_type + 18, // [18:19] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1745,7 +1849,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 17, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 39604c70..8fc6c2f7 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -90,6 +90,33 @@ message Register { // nothing — an agent with no model directories configured behaves exactly as // every agent did before this existed, and constrains nothing. repeated LocalWeights weights = 11; + + // storage is the disk behind those weights, one entry per filesystem. + // + // Reported so an operator can see a node filling up before a fetch dies at + // ninety percent, and so placement can refuse a node that cannot hold a + // model rather than discovering it during the download. + // + // Empty means the node was not asked to look, exactly as for weights, and + // constrains nothing. + repeated Storage storage = 12; +} + +// Storage is one filesystem a node keeps weights on. +// +// Per filesystem rather than per configured directory: free space belongs to +// the disk, and two model directories on one disk are one pool of bytes. +message Storage { + // path is one directory on this filesystem, so the operator recognises it. + string path = 1; + // free_bytes is what a non-root process can write — not the root reserve, + // which is space the agent cannot have. + int64 free_bytes = 2; + int64 total_bytes = 3; + // shared marks a filesystem the node did not fill. Free space on one is + // shared with every other node using it, so two nodes each reporting 500 GB + // free on the same filer have 500 GB between them and not a terabyte. + bool shared = 4; } // RunningShard describes an engine already running on a node. diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 812acbe3..275cc31d 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -51,6 +51,9 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e // holds has nothing to do with its cards, and a node with none is exactly // where a CPU profile will look for them. inv.Weights = ScanWeights(opts.ModelDirs) + // And the disk they sit on. A node can hold a model today and be unable to + // take the next one, which the file list alone never shows. + inv.Storage = ScanStorage(opts.ModelDirs) if !opts.SkipGPUs { switch { diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index cd94b5c1..4d475bf1 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -114,6 +114,9 @@ type Inventory struct { // Weights are the model files this node can see. Empty means it was not // asked to look. Weights []LocalWeights `json:"weights,omitempty"` + // Storage is the disk behind those weights, one entry per filesystem. + // Empty means the same thing it means for Weights: not asked. + Storage []Storage `json:"storage,omitempty"` } // Classes returns the distinct GPU classes present, sorted. diff --git a/internal/agent/discovery/storage.go b/internal/agent/discovery/storage.go new file mode 100644 index 00000000..fc8a9881 --- /dev/null +++ b/internal/agent/discovery/storage.go @@ -0,0 +1,90 @@ +package discovery + +import ( + "os" + "syscall" +) + +// Storage is one filesystem a node keeps weights on. +// +// One entry per filesystem rather than per configured directory, because free +// space belongs to the disk and not to the path. Two model directories on the +// same disk are one pool of bytes, and reporting them separately would let the +// control plane add 24 GB to 24 GB and place a 40 GB model on a node with 24. +type Storage struct { + // Path is one directory on this filesystem — the first one configured that + // landed here, so the operator recognises it. + Path string `json:"path"` + // FreeBytes is what a non-root process can still write. Deliberately not + // the root reserve: the agent does not run as root, so the reserve is space + // it cannot have. + FreeBytes int64 `json:"free_bytes"` + // TotalBytes is the filesystem's size, so a reading of "8 GB free" can be + // told apart on a 10 TB filer and on a 10 GB root partition. + TotalBytes int64 `json:"total_bytes"` + // Shared marks a filesystem the node did not have to fill. Free space on + // one is shared with every other node using it, so two nodes reporting + // 500 GB free on the same filer have 500 GB between them, not a terabyte. + Shared bool `json:"shared,omitempty"` +} + +// ScanStorage reports the filesystems behind the given model directories. +// +// Empty in, empty out, and a directory that cannot be read is skipped rather +// than reported as full: a mount that has not come back yet looks exactly like +// a missing directory, and claiming zero free bytes would take the node out of +// placement for a reason that is not true. Absent is not zero here, as +// everywhere else in this system. +func ScanStorage(dirs []string) []Storage { + var out []Storage + seen := map[uint64]int{} // filesystem id -> index in out + + for _, dir := range dirs { + if dir == "" { + continue + } + dev, ok := filesystemID(dir) + if !ok { + continue + } + if i, dup := seen[dev]; dup { + // Same disk, already counted. Keep the first path: it is the one + // the operator wrote first and so the one they will recognise. + _ = i + continue + } + + var st syscall.Statfs_t + if err := syscall.Statfs(dir, &st); err != nil { + continue + } + // Bsize is int32 on some platforms and int64 on others, which is why + // every field here is widened explicitly rather than assumed. + block := int64(st.Bsize) + seen[dev] = len(out) + out = append(out, Storage{ + Path: dir, + FreeBytes: int64(st.Bavail) * block, //nolint:gosec,unconvert // widened on purpose + TotalBytes: int64(st.Blocks) * block, //nolint:gosec,unconvert // widened on purpose + Shared: isNetworkFS(dir), + }) + } + return out +} + +// filesystemID identifies the disk a path is on. +// +// By device number rather than by the statfs filesystem id, because the field +// holding that id is spelled differently on Linux and on macOS and this has to +// compile on both — the agent runs on one and is developed on the other. +func filesystemID(path string) (uint64, bool) { + fi, err := os.Stat(path) + if err != nil { + return 0, false + } + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return uint64(st.Dev), true //nolint:gosec,unconvert // int32 on macOS, uint64 on Linux +} diff --git a/internal/agent/discovery/storage_test.go b/internal/agent/discovery/storage_test.go new file mode 100644 index 00000000..b8f7a090 --- /dev/null +++ b/internal/agent/discovery/storage_test.go @@ -0,0 +1,63 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func TestTwoDirectoriesOnOneDiskAreOnePoolOfBytes(t *testing.T) { + // The reason this matters: free space belongs to the disk, not to the path. + // Counting two directories separately lets the control plane add 24 GB to + // 24 GB and place a 40 GB model on a node that has 24. + base := t.TempDir() + a := filepath.Join(base, "models-a") + b := filepath.Join(base, "models-b") + for _, d := range []string{a, b} { + if err := os.Mkdir(d, 0o755); err != nil { + t.Fatal(err) + } + } + + got := ScanStorage([]string{a, b}) + if len(got) != 1 { + t.Fatalf("two directories on one filesystem reported as %d pools: %+v", len(got), got) + } + if got[0].Path != a { + t.Errorf("kept %q rather than the first directory configured", got[0].Path) + } +} + +func TestStorageReportsSomethingPlausible(t *testing.T) { + got := ScanStorage([]string{t.TempDir()}) + if len(got) != 1 { + t.Fatalf("a real directory reported %d filesystems", len(got)) + } + s := got[0] + if s.TotalBytes <= 0 { + t.Error("a filesystem with no size at all") + } + if s.FreeBytes < 0 || s.FreeBytes > s.TotalBytes { + t.Errorf("free %d of total %d", s.FreeBytes, s.TotalBytes) + } +} + +func TestAMissingDirectoryIsNotAFullDisk(t *testing.T) { + // A mount that has not come back yet looks exactly like a missing + // directory. Reporting it as zero free bytes would take the node out of + // placement for a reason that is not true; reporting nothing says only + // that nothing is known, which is the honest answer. + got := ScanStorage([]string{filepath.Join(t.TempDir(), "never-mounted")}) + if len(got) != 0 { + t.Errorf("a missing directory was reported as a filesystem: %+v", got) + } +} + +func TestNoDirectoriesConstrainNothing(t *testing.T) { + if got := ScanStorage(nil); len(got) != 0 { + t.Errorf("scanning nothing found %+v", got) + } + if got := ScanStorage([]string{""}); len(got) != 0 { + t.Errorf("an empty path found %+v", got) + } +} diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index cce20d30..b98c888b 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -149,10 +149,11 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%d\t%s\n", + fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%d\t%s\n", n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), + diskColumn(n.DiskFreeGB, n.DiskTotalGB), join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -221,6 +222,23 @@ func memColumn(gb int) string { return fmt.Sprintf("%dG", gb) } +// diskColumn renders free space against the size it is free out of. +// +// Both, because they answer different questions and the first is useless +// without the second: 8 GB free is comfortable on a 10 TB filer and nearly +// full on a 10 GB root partition, and it is the second case that stops a +// model from being placed. +// +// A dash for a node that reported nothing, which an older agent and a node +// with no model directories both do. It has to read as "not known" rather +// than as "no space", or every unconfigured node looks like it is full. +func diskColumn(freeGB, totalGB int) string { + if totalGB <= 0 { + return "-" + } + return fmt.Sprintf("%dG/%dG", freeGB, totalGB) +} + func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return &cobra.Command{ Use: "models", diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index b9f09c16..2b9792ac 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -62,9 +62,21 @@ type NodeView struct { MemoryGB int `json:"memory_gb,omitempty"` // Models are the weights this node can see. Absent means it was not asked // to look, which is not the same as holding none. - Models []string `json:"models_on_disk,omitempty"` - Classes []string `json:"gpu_classes,omitempty"` - Fabrics []string `json:"fabrics,omitempty"` + Models []string `json:"models_on_disk,omitempty"` + // DiskFreeGB and DiskTotalGB are the filesystems holding those weights, + // summed. Zero means the agent did not report any, which an older agent + // and a node with no model directories both do — so it reads as unknown + // rather than as a full disk. + // + // Summed because an operator asking "can this node take another model" + // wants one number; the per-filesystem detail is in the node's own record + // for anyone who needs it. Shared filesystems are left out of the sum: the + // space on a filer is not this node's to spend, and adding it would let a + // fleet of twenty nodes each believe it has the same free terabyte. + DiskFreeGB int `json:"disk_free_gb,omitempty"` + DiskTotalGB int `json:"disk_total_gb,omitempty"` + Classes []string `json:"gpu_classes,omitempty"` + Fabrics []string `json:"fabrics,omitempty"` // Unhealthy lists GPUs kept out of placement and why, which is usually the // answer when a node looks idle but nothing will schedule onto it. @@ -166,6 +178,10 @@ func (a *API) Nodes() []NodeView { Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Replicas: replicasOn[n.ID], } + if free, total := n.LocalDisk(); total > 0 { + v.DiskFreeGB = int(free / (1 << 30)) + v.DiskTotalGB = int(total / (1 << 30)) + } for _, w := range n.Weights { v.Models = append(v.Models, w.Path) } diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index e8d00089..4c23a663 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -30,6 +30,14 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(), }) } + for _, s := range r.GetStorage() { + n.Storage = append(n.Storage, discovery.Storage{ + Path: s.GetPath(), + FreeBytes: s.GetFreeBytes(), + TotalBytes: s.GetTotalBytes(), + Shared: s.GetShared(), + }) + } for _, g := range r.GetGpus() { n.GPUs = append(n.GPUs, discovery.GPU{ Index: int(g.GetIndex()), @@ -75,6 +83,14 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared, }) } + for _, s := range inv.Storage { + r.Storage = append(r.Storage, &llmgwv1.Storage{ + Path: s.Path, + FreeBytes: s.FreeBytes, + TotalBytes: s.TotalBytes, + Shared: s.Shared, + }) + } for _, g := range inv.GPUs { r.Gpus = append(r.Gpus, &llmgwv1.GPU{ Index: int32(g.Index), diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index a9b39bf3..b5b9c9ae 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -45,6 +45,8 @@ type Collector struct { nodeGPUsFree *prometheus.Desc nodeMemory *prometheus.Desc nodeSeen *prometheus.Desc + nodeDiskFree *prometheus.Desc + nodeDiskSize *prometheus.Desc gpuMemoryTotal *prometheus.Desc gpuMemoryUsed *prometheus.Desc @@ -76,6 +78,15 @@ func New(src Source) *Collector { "A node's total RAM, 0 when it does not report one.", node, nil), nodeSeen: prometheus.NewDesc("llmgw_node_last_seen_seconds", "Seconds since a node last reported. Rising means an agent has gone quiet.", node, nil), + // Only the disk a node fills itself: space on a shared filer is not + // this node's to spend, and summing it per node would have a fleet of + // twenty each report the same free terabyte. Absent rather than zero + // when nothing was reported, so an alert on "free is low" does not + // fire for every node that was never asked about its disk. + nodeDiskFree: prometheus.NewDesc("llmgw_node_disk_free_bytes", + "Free space on the local filesystems holding model weights. Absent when the node reports no storage.", node, nil), + nodeDiskSize: prometheus.NewDesc("llmgw_node_disk_total_bytes", + "Size of those filesystems, so free space can be read as a fraction.", node, nil), gpuMemoryTotal: prometheus.NewDesc("llmgw_gpu_memory_total_bytes", "A GPU's framebuffer size.", gpu, nil), @@ -111,6 +122,7 @@ func New(src Source) *Collector { func (c *Collector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, + c.nodeDiskFree, c.nodeDiskSize, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -143,6 +155,15 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { float64(n.MemoryTotalMB)*1024*1024, n.ID) ch <- prometheus.MustNewConstMetric(c.nodeSeen, prometheus.GaugeValue, now.Sub(n.LastSeen).Seconds(), n.ID) + // Nothing reported means nothing is known, so no sample at all. A zero + // here would read as a full disk and page someone about a node that + // was simply never asked. + if free, total := n.LocalDisk(); total > 0 { + ch <- prometheus.MustNewConstMetric(c.nodeDiskFree, prometheus.GaugeValue, + float64(free), n.ID) + ch <- prometheus.MustNewConstMetric(c.nodeDiskSize, prometheus.GaugeValue, + float64(total), n.ID) + } byClass := map[string]int{} for _, g := range n.GPUs { diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 24830798..a58a3e9f 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -84,6 +84,12 @@ type Node struct { // Empty means the node was not asked to look, which is not the same as // holding none — so it constrains nothing. Weights []discovery.LocalWeights `json:"weights,omitempty"` + // Storage is the disk those weights sit on, one entry per filesystem. + // + // Empty means the node was not asked, not that it has no disk — so an + // agent too old to report it, or one with no model directories, is left + // unconstrained rather than treated as full. + Storage []discovery.Storage `json:"storage,omitempty"` GPUs []discovery.GPU `json:"gpus"` // Fabrics is empty on a node without IB. Such a node is its own @@ -310,6 +316,28 @@ func (s *Store) FreeGPUs(nodeID string) []int { return out } +// LocalDisk sums the space on filesystems this node fills itself. +// +// Shared filesystems are left out on purpose. Space on a filer belongs to +// everyone mounting it, and counting it per node would let twenty nodes each +// believe they have the same free terabyte — then place twenty models into it. +// A node whose weights are all on a shared mount therefore reports no local +// disk, which is correct: it has none to spend. +// +// Both zero means nothing is known — an agent too old to report storage, or a +// node with no model directories — and callers must read that as unknown +// rather than as a full disk. +func (n Node) LocalDisk() (free, total int64) { + for _, s := range n.Storage { + if s.Shared { + continue + } + free += s.FreeBytes + total += s.TotalBytes + } + return free, total +} + // HasWeights reports whether this node can see the given source. // // An exact path match, deliberately. A source that is not a path — a model id diff --git a/internal/control/inventory/inventory_test.go b/internal/control/inventory/inventory_test.go index 41eeef93..f6e72dfc 100644 --- a/internal/control/inventory/inventory_test.go +++ b/internal/control/inventory/inventory_test.go @@ -248,3 +248,42 @@ func TestConcurrentAccess(t *testing.T) { } wg.Wait() } + +func TestSharedFilesystemsAreNotThisNodesSpaceToSpend(t *testing.T) { + // Twenty nodes mounting one filer have 500 GB between them, not ten + // terabytes. Counting a shared mount per node is how a scheduler places + // twenty models into space for one. + n := Node{Storage: []discovery.Storage{ + {Path: "/mnt/filer", FreeBytes: 500 << 30, TotalBytes: 900 << 30, Shared: true}, + {Path: "/var/lib/llmgw-models", FreeBytes: 20 << 30, TotalBytes: 60 << 30}, + }} + free, total := n.LocalDisk() + if free != 20<<30 || total != 60<<30 { + t.Errorf("shared space was counted as local: free=%d total=%d", free, total) + } +} + +func TestANodeThatReportedNoStorageIsUnknownNotFull(t *testing.T) { + // An agent too old to report storage, and a node with no model directories, + // both send nothing. Reading that as a full disk would take every such node + // out of placement for a reason that is not true. + free, total := Node{}.LocalDisk() + if free != 0 || total != 0 { + t.Errorf("a node with no storage reported %d/%d", free, total) + } + // The contract callers rely on: total of zero is the signal for "unknown", + // which is why nothing may read free alone. + if total != 0 { + t.Error("callers cannot distinguish unknown from full") + } +} + +func TestLocalDiskSumsSeveralDisks(t *testing.T) { + n := Node{Storage: []discovery.Storage{ + {Path: "/a", FreeBytes: 10, TotalBytes: 100}, + {Path: "/b", FreeBytes: 5, TotalBytes: 50}, + }} + if free, total := n.LocalDisk(); free != 15 || total != 150 { + t.Errorf("free=%d total=%d, want 15/150", free, total) + } +} diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 62038322..b2556f1b 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -98,12 +98,16 @@ requested whenever a fallback served it.

Nodes

{{if .Nodes}} - +
NodeStateGPUsRAMClassesFabricsModelsReplicasLast seen
{{range .Nodes}} + {{/* Free out of total, because the first is meaningless without the + second, and an em dash where nothing was reported: an agent that + was never asked about its disk must not read as a full one. */}} +
NodeStateGPUsRAMDiskClassesFabricsModelsReplicasLast seen
{{.ID}}{{if .MgmtAddr}} {{.MgmtAddr}}{{end}} {{.State}} {{.GPUFree}}/{{.GPUTotal}} {{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}} {{join .Classes}} {{join .Fabrics}} {{if .Models}}{{len .Models}}{{else}}—{{end}}