diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 45ebe9d..c38a95d 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0} } // AgentMessage is anything a node tells the control plane. @@ -344,6 +344,19 @@ type Register struct { // ready node looks like a candidate: a 14B model whose working set is 24 GB // would be placed just as happily on a node with 8 GB. MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"` + // weights are the model files this node can see, from the directories its + // configuration names. + // + // How they got there is the operator's business and deliberately outside + // llmgw's: a shared filesystem, an image exported over the fabric, rsync, or + // a copy made by hand all look identical here. What matters to placement is + // only whether they are present, because a node without them either cannot + // start the replica at all or must first move tens of gigabytes. + // + // Empty means the node was not asked to look, which is not the same as having + // nothing — an agent with no model directories configured behaves exactly as + // every agent did before this existed, and constrains nothing. + Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -448,6 +461,13 @@ func (x *Register) GetMemoryTotalMb() int64 { return 0 } +func (x *Register) GetWeights() []*LocalWeights { + if x != nil { + return x.Weights + } + return nil +} + // RunningShard describes an engine already running on a node. // // It carries enough for the control plane to rebuild the replica it belongs to, @@ -819,6 +839,72 @@ func (x *FabricAttachment) GetRateGbps() int32 { } // Heartbeat says the node is alive and reports anything that changed cheaply. +// LocalWeights is one model file or directory a node can see. +type LocalWeights struct { + state protoimpl.MessageState `protogen:"open.v1"` + // path is what an engine would be given, so it can be compared against a + // model's source without interpretation. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + SizeBytes int64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + // shared marks weights on a filesystem the node did not have to fill — + // reported so a fabric with a shared filer is not mistaken for one where + // every node happens to have fetched its own copy. + Shared bool `protobuf:"varint,3,opt,name=shared,proto3" json:"shared,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LocalWeights) Reset() { + *x = LocalWeights{} + mi := &file_llmgw_v1_agent_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LocalWeights) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LocalWeights) ProtoMessage() {} + +func (x *LocalWeights) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead. +func (*LocalWeights) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} +} + +func (x *LocalWeights) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *LocalWeights) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *LocalWeights) GetShared() bool { + if x != nil { + return x.Shared + } + return false +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -832,7 +918,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -844,7 +930,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[6] + mi := &file_llmgw_v1_agent_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -857,7 +943,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} } func (x *Heartbeat) GetNodeId() string { @@ -899,7 +985,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -911,7 +997,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[7] + mi := &file_llmgw_v1_agent_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -924,7 +1010,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} } func (x *ShardStatus) GetNodeId() string { @@ -1005,7 +1091,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1017,7 +1103,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[8] + mi := &file_llmgw_v1_agent_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1030,7 +1116,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} } func (x *StartShard) GetReplicaId() string { @@ -1150,7 +1236,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1162,7 +1248,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[9] + mi := &file_llmgw_v1_agent_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1175,7 +1261,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} } func (x *EngineProfile) GetName() string { @@ -1265,7 +1351,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1277,7 +1363,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[10] + mi := &file_llmgw_v1_agent_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1290,7 +1376,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} } func (x *StopShard) GetShardId() string { @@ -1310,7 +1396,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1322,7 +1408,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[11] + mi := &file_llmgw_v1_agent_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1335,7 +1421,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} } func (x *DrainShard) GetShardId() string { @@ -1357,7 +1443,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1369,7 +1455,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1382,7 +1468,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *Ack) GetMessageId() string { @@ -1421,7 +1507,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" + "\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" + "\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" + - "\x03msg\"\xc8\x04\n" + + "\x03msg\"\xfa\x04\n" + "\bRegister\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" + "\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" + @@ -1433,7 +1519,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" + "\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" + "\x0fmemory_total_mb\x18\n" + - " \x01(\x03R\rmemoryTotalMb\x1a9\n" + + " \x01(\x03R\rmemoryTotalMb\x120\n" + + "\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" + @@ -1481,7 +1568,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x10FabricAttachment\x12\x16\n" + "\x06fabric\x18\x01 \x01(\tR\x06fabric\x12\x12\n" + "\x04hcas\x18\x02 \x03(\tR\x04hcas\x12\x1b\n" + - "\trate_gbps\x18\x03 \x01(\x05R\brateGbps\"t\n" + + "\trate_gbps\x18\x03 \x01(\x05R\brateGbps\"Y\n" + + "\fLocalWeights\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" + + "\n" + + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" + + "\x06shared\x18\x03 \x01(\bR\x06shared\"t\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -1571,7 +1663,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -1580,41 +1672,43 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*RunningShard)(nil), // 4: llmgw.v1.RunningShard (*GPU)(nil), // 5: llmgw.v1.GPU (*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment - (*Heartbeat)(nil), // 7: llmgw.v1.Heartbeat - (*ShardStatus)(nil), // 8: llmgw.v1.ShardStatus - (*StartShard)(nil), // 9: llmgw.v1.StartShard - (*EngineProfile)(nil), // 10: llmgw.v1.EngineProfile - (*StopShard)(nil), // 11: llmgw.v1.StopShard - (*DrainShard)(nil), // 12: llmgw.v1.DrainShard - (*Ack)(nil), // 13: llmgw.v1.Ack - nil, // 14: llmgw.v1.Register.LabelsEntry - nil, // 15: llmgw.v1.Register.EngineVersionsEntry - nil, // 16: llmgw.v1.EngineProfile.EnvEntry + (*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights + (*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat + (*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus + (*StartShard)(nil), // 10: llmgw.v1.StartShard + (*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile + (*StopShard)(nil), // 12: llmgw.v1.StopShard + (*DrainShard)(nil), // 13: llmgw.v1.DrainShard + (*Ack)(nil), // 14: llmgw.v1.Ack + nil, // 15: llmgw.v1.Register.LabelsEntry + nil, // 16: llmgw.v1.Register.EngineVersionsEntry + nil, // 17: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 7, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 8, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 9, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 11, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 12, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 13, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 14, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 15, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard - 0, // 12: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 0, // 13: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 10, // 14: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 16, // 15: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 16: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 17: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 17, // [17:18] is the sub-list for method output_type - 16, // [16:17] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights + 0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State + 0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 18, // [18:19] is the sub-list for method output_type + 17, // [17:18] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -1639,7 +1733,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 16, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 2e20b61..4293c6d 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -76,6 +76,20 @@ message Register { // ready node looks like a candidate: a 14B model whose working set is 24 GB // would be placed just as happily on a node with 8 GB. int64 memory_total_mb = 10; + + // weights are the model files this node can see, from the directories its + // configuration names. + // + // How they got there is the operator's business and deliberately outside + // llmgw's: a shared filesystem, an image exported over the fabric, rsync, or + // a copy made by hand all look identical here. What matters to placement is + // only whether they are present, because a node without them either cannot + // start the replica at all or must first move tens of gigabytes. + // + // Empty means the node was not asked to look, which is not the same as having + // nothing — an agent with no model directories configured behaves exactly as + // every agent did before this existed, and constrains nothing. + repeated LocalWeights weights = 11; } // RunningShard describes an engine already running on a node. @@ -135,6 +149,18 @@ message FabricAttachment { } // Heartbeat says the node is alive and reports anything that changed cheaply. +// LocalWeights is one model file or directory a node can see. +message LocalWeights { + // path is what an engine would be given, so it can be compared against a + // model's source without interpretation. + string path = 1; + int64 size_bytes = 2; + // shared marks weights on a filesystem the node did not have to fill — + // reported so a fabric with a shared filer is not mistaken for one where + // every node happens to have fetched its own copy. + bool shared = 3; +} + message Heartbeat { string node_id = 1; // gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index a78f521..07f4ae1 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -57,6 +57,23 @@ discovery: prefer_nvml: true skip_gpus: false +# Where this node keeps model weights. +# +# Unset by default, and unset means the node is not asked to look — which is +# not the same as having nothing, and constrains nothing. Storage differs at +# every installation and llmgw deliberately does not own it: a Lustre mount, an +# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy +# made by hand all look identical from here. +# +# What it buys is placement preferring a node that already holds the model over +# one that must first move tens of gigabytes. That matters most where fabrics +# are isolated from one another, because weights in one fabric are not weights +# in another and the only path between them is the management network. +# +# model_dirs: +# - /var/lib/llmgw-models +# - /mnt/lustre/models + engines: vllm: bin: /opt/venvs/vllm/bin/vllm diff --git a/docs/install.md b/docs/install.md index f0f954d..1bbd667 100644 --- a/docs/install.md +++ b/docs/install.md @@ -282,6 +282,34 @@ sudo install -d -o root -g root /var/lib/llmgw-models For a first model, something small: a 3B at `q4_k_m` is about 2 GB, runs on CPU, and answers fast enough to prove the whole path works. +### Telling llmgw where they are + +Optional, and unset by default. Naming the directories in the agent's +`model_dirs` lets a node report which models it can see, and placement then +prefers a node that already holds one over a node that must first move tens of +gigabytes: + +```yaml +model_dirs: + - /var/lib/llmgw-models + - /mnt/lustre/models +``` + +llmgw does not manage that storage and does not care how the files arrived. A +Lustre or NFS mount, an image exported over the fabric with SRP or NVMe-oF, +local NVMe, or an `scp` all look identical to it. A node with nothing configured +reports nothing and is treated exactly as every node was before this existed. + +**It matters most when your fabrics are not connected to each other.** Weights +in `ib-a` are not weights in `ib-b`: the only path between them is the +management network, which is the slow one. `llmgw ctl status` shows what each +fabric holds, so a model seeded into one fabric does not quietly leave the rest +of the cluster unable to serve it. + +How much disk a node needs follows from the models rather than a rule of thumb. +`llmgw ctl model plan ` prints the sizes; as a starting point, a 70B at +bf16 is 141 GB and the same model at q4 is 43 GB. + ## 5. Adding the node Two halves, because they need different access: building the machine needs the diff --git a/internal/agent/agent.go b/internal/agent/agent.go index bb854f3..ecca23d 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -58,6 +58,16 @@ type Config struct { // keeps the plaintext behaviour, which only belongs on a trusted network. TLS tlsconf.Config `koanf:"tls"` + // ModelDirs are the directories this node keeps model weights in. + // + // Empty by default, and empty means the node is not asked to look — which + // is not the same as having nothing. Storage differs at every installation, + // so llmgw observes it rather than owning it: a Lustre mount, an image + // exported over the fabric, local NVMe and a copy made by hand all look + // identical from here, and placement only needs to know whether the file + // is present. + ModelDirs []string `koanf:"model_dirs"` + Discovery discovery.Options `koanf:"discovery"` Engines engines.Config `koanf:"engines"` Superv supervisor.Config `koanf:"supervisor"` @@ -368,7 +378,11 @@ func int32Slice(v []int) []int32 { // scan refreshes the node's hardware inventory. func (a *Agent) scan(ctx context.Context) error { - inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log) + // Carried from the agent's own configuration rather than duplicated into + // the discovery block, so an operator names the directories once. + opts := a.cfg.Discovery + opts.ModelDirs = a.cfg.ModelDirs + inv, err := discovery.Discover(ctx, opts, a.log) a.mu.Lock() a.inv = inv a.mu.Unlock() diff --git a/internal/agent/discovery/discover.go b/internal/agent/discovery/discover.go index 1225cf2..812acbe 100644 --- a/internal/agent/discovery/discover.go +++ b/internal/agent/discovery/discover.go @@ -26,6 +26,8 @@ type Options struct { PreferNVML bool `koanf:"prefer_nvml"` // SkipGPUs omits GPU discovery, for nodes that have none. SkipGPUs bool `koanf:"skip_gpus"` + // ModelDirs are scanned for weights. Empty means do not look. + ModelDirs []string `koanf:"-"` } // Discover builds the node's inventory. @@ -45,6 +47,10 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e // has nothing to do with cards goes with it. That is exactly the node whose // memory matters most, since a CPU profile is what it will be asked to run. inv.MemoryTotalMB = MemoryTotalMB() + // Also before the GPU paths, and for the same reason: which weights a node + // holds has nothing to do with its cards, and a node with none is exactly + // where a CPU profile will look for them. + inv.Weights = ScanWeights(opts.ModelDirs) if !opts.SkipGPUs { switch { diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index 6b4c71f..5780a94 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -103,6 +103,9 @@ type Inventory struct { // MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an // agent older than this field reports, and is treated as no constraint. MemoryTotalMB int `json:"memory_total_mb,omitempty"` + // Weights are the model files this node can see. Empty means it was not + // asked to look. + Weights []LocalWeights `json:"weights,omitempty"` } // Classes returns the distinct GPU classes present, sorted. diff --git a/internal/agent/discovery/weights.go b/internal/agent/discovery/weights.go new file mode 100644 index 0000000..9b640c8 --- /dev/null +++ b/internal/agent/discovery/weights.go @@ -0,0 +1,137 @@ +package discovery + +import ( + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "syscall" +) + +// LocalWeights is one model file or directory a node can see. +type LocalWeights struct { + Path string `json:"path"` + SizeBytes int64 `json:"size_bytes"` + // Shared marks weights on a filesystem this node did not have to fill. + // + // Worth distinguishing: twenty nodes each holding their own copy and twenty + // nodes seeing one filer are the same list of paths and completely + // different situations — one has twenty copies of a 141 GB model and the + // other has one. + Shared bool `json:"shared,omitempty"` +} + +// ScanWeights lists the model files under the given directories. +// +// Storage is the part of this system that will differ at every installation — +// a Lustre mount, an image exported over the fabric, local NVMe, an rsync from +// a laptop — so llmgw does not try to own it. It observes the result instead: +// whatever put the file there, the question placement needs answered is only +// whether it is present. +// +// Scanning nothing is the default and constrains nothing. A node with no +// directories configured behaves exactly as every node did before this existed, +// which is what makes the whole feature optional. +func ScanWeights(dirs []string) []LocalWeights { + var out []LocalWeights + seen := map[string]struct{}{} + + for _, dir := range dirs { + if dir == "" { + continue + } + shared := isNetworkFS(dir) + entries, err := os.ReadDir(dir) + if err != nil { + // A directory that is not there is not an error to report upward: + // a mount that has not come back yet looks exactly like this, and + // the node is simply one without those weights until it does. + continue + } + for _, e := range entries { + path := filepath.Join(dir, e.Name()) + if _, dup := seen[path]; dup { + continue + } + size, ok := weightSize(path, e) + if !ok { + continue + } + seen[path] = struct{}{} + out = append(out, LocalWeights{Path: path, SizeBytes: size, Shared: shared}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + +// weightSize measures a candidate and says whether it is one at all. +// +// A model is either a single file — a .gguf, usually — or a directory of +// shards and a config, which is what every HuggingFace checkout looks like. +// Anything else in the directory is somebody's notes and is not reported. +func weightSize(path string, e fs.DirEntry) (int64, bool) { + if e.IsDir() { + // A directory counts only if it actually holds weights. An empty one + // is a half-finished download, and reporting it as present would have + // placement choose a node whose engine then fails to load. + var total int64 + var found bool + _ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil //nolint:nilerr // an unreadable subtree is not fatal + } + if isWeightFile(p) { + found = true + } + if info, err := d.Info(); err == nil { + total += info.Size() + } + return nil + }) + return total, found + } + if !isWeightFile(path) { + return 0, false + } + info, err := e.Info() + if err != nil { + return 0, false + } + return info.Size(), true +} + +func isWeightFile(path string) bool { + switch strings.ToLower(filepath.Ext(path)) { + case ".gguf", ".safetensors", ".bin", ".pt", ".pth": + return true + } + return false +} + +// isNetworkFS reports whether a path is on a filesystem the node did not fill. +// +// By magic number rather than by parsing mounts, because the mount table says +// what was asked for and this says what the kernel actually has. The list is +// the network filesystems that turn up in front of GPUs: NFS, Lustre, GPFS, +// CephFS, GlusterFS and SMB. Anything unrecognised is treated as local, which +// is the conservative direction — a shared filer mistaken for local storage +// costs an unnecessary copy, while local storage mistaken for shared would have +// the cluster believe a model is everywhere when it is on one node. +func isNetworkFS(path string) bool { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return false + } + switch uint32(st.Type) { //nolint:unconvert // Type is int64 on some platforms + case 0x6969, // NFS + 0x0BD00BD0, // Lustre + 0x47504653, // GPFS + 0x00C36400, // CephFS + 0x65735546, // FUSE, which is how GlusterFS and many object mounts appear + 0xFF534D42: // SMB/CIFS + return true + } + return false +} diff --git a/internal/agent/discovery/weights_test.go b/internal/agent/discovery/weights_test.go new file mode 100644 index 0000000..21f0ae2 --- /dev/null +++ b/internal/agent/discovery/weights_test.go @@ -0,0 +1,114 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func write(t *testing.T, path string, size int) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestScanFindsFilesAndDirectoriesOfWeights(t *testing.T) { + dir := t.TempDir() + write(t, filepath.Join(dir, "qwen-3b-q4.gguf"), 1024) + // A HuggingFace checkout: a directory of shards rather than one file. + write(t, filepath.Join(dir, "Llama-3.3-70B", "model-00001.safetensors"), 2048) + write(t, filepath.Join(dir, "Llama-3.3-70B", "config.json"), 64) + + got := ScanWeights([]string{dir}) + if len(got) != 2 { + t.Fatalf("found %d entries: %+v", len(got), got) + } + byName := map[string]int64{} + for _, w := range got { + byName[filepath.Base(w.Path)] = w.SizeBytes + } + if byName["qwen-3b-q4.gguf"] != 1024 { + t.Errorf("gguf size = %d", byName["qwen-3b-q4.gguf"]) + } + // The directory's size is everything in it, config included: what matters + // is how much would have to be moved to put it somewhere else. + if byName["Llama-3.3-70B"] != 2048+64 { + t.Errorf("checkout size = %d, want %d", byName["Llama-3.3-70B"], 2048+64) + } +} + +func TestAnEmptyDirectoryIsNotWeights(t *testing.T) { + // A half-finished download looks exactly like this, and reporting it as + // present would have placement choose a node whose engine then fails to + // load — the failure this whole feature exists to avoid. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "downloading"), 0o755); err != nil { + t.Fatal(err) + } + write(t, filepath.Join(dir, "downloading", "README.md"), 10) + + if got := ScanWeights([]string{dir}); len(got) != 0 { + t.Errorf("a directory with no weight files was reported: %+v", got) + } +} + +func TestNotesAndScriptsAreNotReportedAsModels(t *testing.T) { + dir := t.TempDir() + write(t, filepath.Join(dir, "notes.txt"), 10) + write(t, filepath.Join(dir, "fetch.sh"), 10) + write(t, filepath.Join(dir, "real.gguf"), 10) + + got := ScanWeights([]string{dir}) + if len(got) != 1 || filepath.Base(got[0].Path) != "real.gguf" { + t.Errorf("scan returned %+v", got) + } +} + +func TestAMissingDirectoryIsNotAnError(t *testing.T) { + // A mount that has not come back yet looks exactly like a directory that is + // not there, and the node is simply one without those weights until it is. + got := ScanWeights([]string{filepath.Join(t.TempDir(), "never-existed")}) + if len(got) != 0 { + t.Errorf("got %+v", got) + } +} + +func TestScanningNothingIsTheDefaultAndConstrainsNothing(t *testing.T) { + // What makes the feature optional: an agent with no directories configured + // behaves exactly as every agent did before this existed. + if got := ScanWeights(nil); len(got) != 0 { + t.Errorf("got %+v", got) + } + if got := ScanWeights([]string{""}); len(got) != 0 { + t.Errorf("an empty directory name produced %+v", got) + } +} + +func TestTheSamePathIsReportedOnce(t *testing.T) { + // Two configured directories can overlap, or be the same mount reached two + // ways. Counting a model twice would overstate what a fabric holds. + dir := t.TempDir() + write(t, filepath.Join(dir, "m.gguf"), 10) + if got := ScanWeights([]string{dir, dir}); len(got) != 1 { + t.Errorf("the same directory twice produced %+v", got) + } +} + +func TestLocalStorageIsNeverMistakenForShared(t *testing.T) { + // The conservative direction. A shared filer read as local costs an + // unnecessary copy; local read as shared would have the cluster believe a + // model is everywhere when it is on one node. + dir := t.TempDir() + write(t, filepath.Join(dir, "m.gguf"), 10) + got := ScanWeights([]string{dir}) + if len(got) != 1 { + t.Fatalf("got %+v", got) + } + if got[0].Shared { + t.Error("a temporary directory on local disk was reported as shared") + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 9b3d6f4..82beee3 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -70,9 +70,10 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { // The free-block column is the one that decides whether a multi-node // replica can land: free cards on the wrong fabric are not capacity for // a model that spans nodes, because a replica may never cross one. - fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE") + fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE\tWEIGHTS") for _, f := range s.Fabrics { - fmt.Fprintf(w, " %s\t%d\t%d\n", f.Fabric, f.Nodes, f.LargestFreeBlock) + fmt.Fprintf(w, " %s\t%d\t%d\t%s\n", + f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f)) } _ = w.Flush() } @@ -114,6 +115,22 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) { } } +// weightsCell says what a fabric holds, and stays quiet when nobody asked. +// +// Fabrics are not connected to one another, so weights in one are not weights +// in another: a model seeded into a single fabric leaves the rest of the +// cluster unable to serve it without first moving tens of gigabytes over the +// management network. This is the column that shows that before someone +// discovers it during a scale-up. +func weightsCell(f adminapi.FabricUse) string { + if f.NodesWithWeights == 0 { + // Nobody on this fabric was asked to look, which is the default and is + // not the same as an empty fabric. + return "-" + } + return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes) +} + // bar draws usage so a column of classes can be read at a glance rather than // compared digit by digit. func bar(used, total int) string { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index d96bbc6..b9f09c1 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -59,9 +59,12 @@ type NodeView struct { // MemoryGB is the machine's RAM, 0 when the agent did not report it. It is // what decides whether a CPU profile can be placed here, so it belongs // where someone looks when a model will not schedule. - MemoryGB int `json:"memory_gb,omitempty"` - Classes []string `json:"gpu_classes,omitempty"` - Fabrics []string `json:"fabrics,omitempty"` + MemoryGB int `json:"memory_gb,omitempty"` + // Models are the weights this node can see. Absent means it was not asked + // to look, which is not the same as holding none. + Models []string `json:"models_on_disk,omitempty"` + Classes []string `json:"gpu_classes,omitempty"` + Fabrics []string `json:"fabrics,omitempty"` // Unhealthy lists GPUs kept out of placement and why, which is usually the // answer when a node looks idle but nothing will schedule onto it. @@ -163,6 +166,9 @@ func (a *API) Nodes() []NodeView { Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Replicas: replicasOn[n.ID], } + for _, w := range n.Weights { + v.Models = append(v.Models, w.Path) + } seen := map[string]struct{}{} for _, g := range n.GPUs { if _, dup := seen[g.Class()]; !dup { diff --git a/internal/control/adminapi/summary.go b/internal/control/adminapi/summary.go index fe2dfde..b3d23c1 100644 --- a/internal/control/adminapi/summary.go +++ b/internal/control/adminapi/summary.go @@ -56,6 +56,16 @@ func (c ClassUse) Free() int { return c.Total - c.Used - c.Unhealthy } type FabricUse struct { Fabric string `json:"fabric"` Nodes int `json:"nodes"` + // ModelsPresent is how many nodes here hold weights at all, and Models how + // many distinct models they hold between them. + // + // Per fabric because that is the unit at which the answer is actionable. + // Fabrics are not connected to one another, so weights in one are not + // weights in another: a model seeded into a single fabric leaves the rest + // of the cluster unable to serve it without first moving tens of gigabytes + // over the management network, and nothing said so. + NodesWithWeights int `json:"nodes_with_weights"` + Models int `json:"models_present"` // LargestFreeBlock is the most nodes on this fabric that are entirely free, // which is the number that decides whether a multi-node replica can land. LargestFreeBlock int `json:"largest_free_block"` @@ -134,6 +144,9 @@ func (a *API) Summary() Summary { byFabric[f] = &FabricUse{Fabric: f} } byFabric[f].Nodes++ + if len(n.Models) > 0 { + byFabric[f].NodesWithWeights++ + } // Entirely free, which is what a multi-node replica needs: a node // with one card in use cannot contribute a full-width shard. if n.State == "ready" && n.GPUFree == n.GPUTotal && n.GPUTotal > 0 { @@ -141,7 +154,22 @@ func (a *API) Summary() Summary { } } } - for _, f := range byFabric { + // Distinct models per fabric, counted across its nodes: one copy on a + // shared filer and twenty local copies are the same answer to "can this + // fabric serve it". + perFabricModels := map[string]map[string]struct{}{} + for _, n := range nodes { + for _, f := range n.Fabrics { + if perFabricModels[f] == nil { + perFabricModels[f] = map[string]struct{}{} + } + for _, m := range n.Models { + perFabricModels[f][m] = struct{}{} + } + } + } + for name, f := range byFabric { + f.Models = len(perFabricModels[name]) s.Fabrics = append(s.Fabrics, *f) } sort.Slice(s.Fabrics, func(i, j int) bool { return s.Fabrics[i].Fabric < s.Fabrics[j].Fabric }) diff --git a/internal/control/adminapi/summary_test.go b/internal/control/adminapi/summary_test.go index b843e9a..ecc7e8a 100644 --- a/internal/control/adminapi/summary_test.go +++ b/internal/control/adminapi/summary_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) @@ -109,3 +110,46 @@ func TestSummaryNamesModelsThatCannotBePlaced(t *testing.T) { t.Error("no model was counted short") } } + +func TestWeightsAreCountedPerFabric(t *testing.T) { + // Fabrics are not connected, so weights in one are not weights in another. + // A model seeded into a single fabric leaves the rest of the cluster unable + // to serve it without first moving tens of gigabytes over the management + // network, and nothing said so. + a1 := gpuNode("a1", 4, "ib-a") + a1.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}} + a2 := gpuNode("a2", 4, "ib-a") + a2.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}} + b1 := gpuNode("b1", 4, "ib-b") // the same fabric capacity, none of the weights + + api, _ := setup(t, testModels(), a1, a2, b1) + s := api.Summary() + + byName := map[string]FabricUse{} + for _, f := range s.Fabrics { + byName[f.Fabric] = f + } + if got := byName["ib-a"]; got.Models != 1 || got.NodesWithWeights != 2 { + t.Errorf("ib-a = %+v, want one model on two nodes", got) + } + if got := byName["ib-b"]; got.Models != 0 || got.NodesWithWeights != 0 { + t.Errorf("ib-b = %+v; it holds nothing and should say so", got) + } +} + +func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) { + // A filer mounted on every node and a local copy on every node are the same + // answer to "can this fabric serve it", which is the question being asked. + var nodes []inventory.Node + for _, id := range []string{"n1", "n2", "n3"} { + n := gpuNode(id, 4, "ib-a") + n.Weights = []discovery.LocalWeights{{Path: "/mnt/shared/llama.gguf", Shared: true}} + nodes = append(nodes, n) + } + api, _ := setup(t, testModels(), nodes...) + + f := api.Summary().Fabrics[0] + if f.Models != 1 { + t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models) + } +} diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index f124d4a..8236566 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -25,6 +25,11 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { EngineVersions: r.GetEngineVersions(), MemoryTotalMB: int(r.GetMemoryTotalMb()), } + for _, w := range r.GetWeights() { + n.Weights = append(n.Weights, discovery.LocalWeights{ + Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(), + }) + } for _, g := range r.GetGpus() { n.GPUs = append(n.GPUs, discovery.GPU{ Index: int(g.GetIndex()), @@ -64,6 +69,11 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, Labels: labels, AgentVersion: agentVersion, MemoryTotalMb: int64(inv.MemoryTotalMB), } + for _, w := range inv.Weights { + r.Weights = append(r.Weights, &llmgwv1.LocalWeights{ + Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared, + }) + } for _, g := range inv.GPUs { r.Gpus = append(r.Gpus, &llmgwv1.GPU{ Index: int32(g.Index), diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index e0bce3e..92aee86 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -322,6 +322,9 @@ func (m Model) PlacementProfiles() []placement.Profile { Priority: p.Priority, NodeSelector: p.NodeSelector, MinMemoryGB: p.MinMemoryGB, + // The model's source, so placement can prefer nodes that already + // hold the weights rather than one that must first move them. + Source: m.Source, }) } return out diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 03bad14..2483079 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -80,6 +80,11 @@ type Node struct { // this field existed. MemoryTotalMB int `json:"memory_total_mb,omitempty"` + // Weights are the model files this node can see, however they got there. + // Empty means the node was not asked to look, which is not the same as + // holding none — so it constrains nothing. + Weights []discovery.LocalWeights `json:"weights,omitempty"` + GPUs []discovery.GPU `json:"gpus"` // Fabrics is empty on a node without IB. Such a node is its own // single-node island and can never host a multi-node replica. @@ -305,6 +310,27 @@ func (s *Store) FreeGPUs(nodeID string) []int { return out } +// HasWeights reports whether this node can see the given source. +// +// An exact path match, deliberately. A source that is not a path — a model id +// an engine will resolve itself — cannot be checked from here, and guessing +// would be worse than the honest answer that nothing is known. +func (n Node) HasWeights(source string) bool { + for _, w := range n.Weights { + if w.Path == source { + return true + } + } + return false +} + +// KnowsItsWeights reports whether this node was asked to look at all. +// +// The distinction that keeps the feature optional: a node reporting no weights +// because nobody configured a directory must not be excluded from placement, +// while a node that looked and genuinely has none should be. +func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 } + // Fabrics lists every fabric any node is attached to, sorted. func (s *Store) Fabrics() []string { s.mu.RLock() diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 6a52f1b..85521c2 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -36,6 +36,9 @@ type Profile struct { NodeSelector map[string]string // MinMemoryGB is the host RAM this profile needs. Zero is unconstrained. MinMemoryGB int + // Source is the model's weights, as the engine would be given it. Used to + // prefer nodes that already hold them; empty asks for no preference. + Source string } // CPUOnly reports whether this profile needs no GPUs. The catalog only lets a @@ -193,6 +196,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re return len(s.inv.Allocations(candidates[i].node.ID)) < len(s.inv.Allocations(candidates[j].node.ID)) }) + preferNodesHoldingWeights(p.Source, candidates) return &Placement{ Profile: p, Pool: "none/cpu", @@ -200,6 +204,11 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re }, rejections } + // Last, so it takes precedence: moving tens of gigabytes costs more than an + // imperfectly packed node, and with isolated fabrics it may cost a + // cross-fabric copy over the management network. + preferNodesHoldingWeights(p.Source, candidates) + best := candidates[0] gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode) if gpus == nil { @@ -291,6 +300,46 @@ type candidate struct { // // fabric is empty for single-node placement, in which case attachment is not // checked at all. +// preferNodesHoldingWeights moves nodes that already have the model to the +// front, leaving everything else in the order it was given. +// +// A stable partition rather than a sort, so every other consideration — bin +// packing, spreading, the emptiest machine for a multi-node replica — survives +// intact among nodes that are equal on this one. +func preferNodesHoldingWeights(source string, candidates []candidate) { + if source == "" { + return + } + sort.SliceStable(candidates, func(i, j int) bool { + a, aKnown := hasWeights(source, candidates[i].node) + b, bKnown := hasWeights(source, candidates[j].node) + // A node that was never asked to look ranks with those that looked and + // have nothing: unknown is not a reason to prefer or to punish. + return (a && aKnown) && !(b && bKnown) + }) +} + +// hasWeights reports whether a node already holds this profile's model, and +// whether that is known at all. +// +// Never a filter, always a preference. Storage differs at every installation +// and llmgw does not own it: a node that was never asked to look reports +// nothing, and excluding those would break every cluster that has not opted +// in. Even a node that looked and does not have the weights stays eligible, +// because "fetch them" is a valid answer and refusing to place would turn a +// slow start into no start at all. +// +// What it changes is the ordering, and that is worth a lot on its own: with +// isolated fabrics the difference between placing where the weights are and +// placing where they are not is a cross-fabric copy of tens of gigabytes over +// the management network. +func hasWeights(source string, n inventory.Node) (present, known bool) { + if source == "" || !n.KnowsItsWeights() { + return false, false + } + return n.HasWeights(source), true +} + // unmatchedLabel reports the first selector entry a node does not satisfy. // // It returns the label rather than a bare false so the rejection can name it: diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 3b0d9ff..c66c432 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -582,3 +582,100 @@ func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) { t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain()) } } + +func nodeWithWeights(id string, gpus int, paths ...string) inventory.Node { + n := node(id, "h100-80g", gpus) + for _, p := range paths { + n.Weights = append(n.Weights, discovery.LocalWeights{Path: p, SizeBytes: 1 << 30}) + } + return n +} + +func TestPlacementPrefersANodeThatAlreadyHasTheWeights(t *testing.T) { + // With isolated fabrics the difference between placing where the weights + // are and placing where they are not is a cross-fabric copy of tens of + // gigabytes over the management network. + const src = "/models/llama.gguf" + without := nodeWithWeights("empty", 8, "/models/something-else.gguf") + with := nodeWithWeights("holds-it", 8, src) + + s := New(store(without, with)) + res := s.Place([]Profile{{ + Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src, + }}, nil) + if res.Placement == nil { + t.Fatalf("nothing was placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "holds-it" { + t.Errorf("placed on %q; the other node already has the weights", got) + } +} + +func TestANodeWithoutTheWeightsIsStillEligible(t *testing.T) { + // Fetching them is a valid answer. Refusing to place would turn a slow + // start into no start at all, which is a worse cluster than a slow one. + const src = "/models/llama.gguf" + only := nodeWithWeights("no-weights", 8, "/models/other.gguf") + + s := New(store(only)) + res := s.Place([]Profile{{ + Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src, + }}, nil) + if res.Placement == nil { + t.Errorf("a node without the weights was excluded:\n%s", res.Explain()) + } +} + +func TestANodeThatWasNeverAskedIsNotPunished(t *testing.T) { + // What keeps the whole feature optional. A node with no model directories + // configured reports nothing, and that must rank with "looked and has + // none" rather than below it — otherwise enabling the feature on one node + // would quietly drain every other. + const src = "/models/llama.gguf" + silent := node("never-asked", "h100-80g", 8) // reports no weights at all + looked := nodeWithWeights("looked", 8, "/models/other.gguf") + + s := New(store(silent, looked)) + for range 4 { + res := s.Place([]Profile{{ + Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src, + }}, nil) + if res.Placement == nil { + t.Fatalf("nothing placed:\n%s", res.Explain()) + } + } + // Both remain candidates; neither is preferred on weights, so the ordinary + // rules decide and nothing errors. +} + +func TestWeightsDoNotOverrideHardConstraints(t *testing.T) { + // A preference, never a filter. The node holding the weights here is the + // wrong GPU class, and choosing it would produce a replica that cannot run. + const src = "/models/llama.gguf" + wrongClass := node("wrong", "a100-40g", 8) + wrongClass.Weights = []discovery.LocalWeights{{Path: src}} + right := node("right", "h100-80g", 8) + + s := New(store(wrongClass, right)) + res := s.Place([]Profile{{ + Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src, + }}, nil) + if res.Placement == nil { + t.Fatalf("nothing placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "right" { + t.Errorf("placed on %q, which is the wrong GPU class", got) + } +} + +func TestAProfileWithNoSourceIsUnaffected(t *testing.T) { + // Every profile before this existed. + a := nodeWithWeights("a", 8, "/models/x.gguf") + b := node("b", "h100-80g", 8) + s := New(store(a, b)) + if res := s.Place([]Profile{{ + Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, + }}, nil); res.Placement == nil { + t.Fatalf("nothing placed:\n%s", res.Explain()) + } +}