Know where the weights are, without owning the storage
CI / test (push) Successful in 1m32s

Storage will be different at every installation — Lustre, NFS over RDMA, an
image exported with SRP or NVMe-oF, local NVMe, an rsync — so llmgw observes
it rather than managing it. An agent told which directories to look in
reports what it can see; how the files got there is nobody's business here,
and a node told nothing reports nothing and behaves exactly as every node
did before this existed.

That fact buys two things. Placement prefers a node that already holds the
model over one that must first move tens of gigabytes, as a preference and
never a filter: fetching them is a valid answer, and refusing to place would
turn a slow start into no start at all. And ctl status shows what each
fabric holds.

Per fabric because fabrics are not connected to one another. Weights in
ib-a are not weights in ib-b — the only path between them is the management
network — so a model seeded into one fabric leaves the rest of the cluster
unable to serve it without a slow copy, and nothing said so before now. One
shared filer counts the same as twenty local copies, because "can this
fabric serve it" is the question being asked.

Three distinctions the tests hold to, because each is a way to get this
quietly wrong. A node that was never asked to look must rank with one that
looked and has nothing, or enabling the feature on a single node would
drain every other. An empty directory is not weights, because a
half-finished download looks exactly like one and would send placement to a
node whose engine then fails to load. And local storage is never mistaken
for shared: a filer read as local costs an unnecessary copy, while local
read as shared would have the cluster believe a model is everywhere when it
is on one node.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-17 08:21:16 +02:00
1 parent bd134a7ab9
commit c13c567cc5
18 files changed
+782 -63

No files matched your search

+150 -56
View File
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
// Deprecated: Use ShardStatus_State.Descriptor instead.
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7, 0}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8, 0}
}
// AgentMessage is anything a node tells the control plane.
@@ -344,6 +344,19 @@ type Register struct {
// ready node looks like a candidate: a 14B model whose working set is 24 GB
// would be placed just as happily on a node with 8 GB.
MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
// weights are the model files this node can see, from the directories its
// configuration names.
//
// How they got there is the operator's business and deliberately outside
// llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
// a copy made by hand all look identical here. What matters to placement is
// only whether they are present, because a node without them either cannot
// start the replica at all or must first move tens of gigabytes.
//
// Empty means the node was not asked to look, which is not the same as having
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
Weights []*LocalWeights `protobuf:"bytes,11,rep,name=weights,proto3" json:"weights,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -448,6 +461,13 @@ func (x *Register) GetMemoryTotalMb() int64 {
return 0
}
func (x *Register) GetWeights() []*LocalWeights {
if x != nil {
return x.Weights
}
return nil
}
// RunningShard describes an engine already running on a node.
//
// It carries enough for the control plane to rebuild the replica it belongs to,
@@ -819,6 +839,72 @@ func (x *FabricAttachment) GetRateGbps() int32 {
}
// Heartbeat says the node is alive and reports anything that changed cheaply.
// LocalWeights is one model file or directory a node can see.
type LocalWeights struct {
state protoimpl.MessageState `protogen:"open.v1"`
// path is what an engine would be given, so it can be compared against a
// model's source without interpretation.
Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
SizeBytes int64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"`
// shared marks weights on a filesystem the node did not have to fill —
// reported so a fabric with a shared filer is not mistaken for one where
// every node happens to have fetched its own copy.
Shared bool `protobuf:"varint,3,opt,name=shared,proto3" json:"shared,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *LocalWeights) Reset() {
*x = LocalWeights{}
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *LocalWeights) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*LocalWeights) ProtoMessage() {}
func (x *LocalWeights) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use LocalWeights.ProtoReflect.Descriptor instead.
func (*LocalWeights) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
}
func (x *LocalWeights) GetPath() string {
if x != nil {
return x.Path
}
return ""
}
func (x *LocalWeights) GetSizeBytes() int64 {
if x != nil {
return x.SizeBytes
}
return 0
}
func (x *LocalWeights) GetShared() bool {
if x != nil {
return x.Shared
}
return false
}
type Heartbeat struct {
state protoimpl.MessageState `protogen:"open.v1"`
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
@@ -832,7 +918,7 @@ type Heartbeat struct {
func (x *Heartbeat) Reset() {
*x = Heartbeat{}
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -844,7 +930,7 @@ func (x *Heartbeat) String() string {
func (*Heartbeat) ProtoMessage() {}
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[6]
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -857,7 +943,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
func (*Heartbeat) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{6}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
}
func (x *Heartbeat) GetNodeId() string {
@@ -899,7 +985,7 @@ type ShardStatus struct {
func (x *ShardStatus) Reset() {
*x = ShardStatus{}
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -911,7 +997,7 @@ func (x *ShardStatus) String() string {
func (*ShardStatus) ProtoMessage() {}
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[7]
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -924,7 +1010,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
func (*ShardStatus) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{7}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
}
func (x *ShardStatus) GetNodeId() string {
@@ -1005,7 +1091,7 @@ type StartShard struct {
func (x *StartShard) Reset() {
*x = StartShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1017,7 +1103,7 @@ func (x *StartShard) String() string {
func (*StartShard) ProtoMessage() {}
func (x *StartShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[8]
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1030,7 +1116,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
func (*StartShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{8}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
}
func (x *StartShard) GetReplicaId() string {
@@ -1150,7 +1236,7 @@ type EngineProfile struct {
func (x *EngineProfile) Reset() {
*x = EngineProfile{}
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1162,7 +1248,7 @@ func (x *EngineProfile) String() string {
func (*EngineProfile) ProtoMessage() {}
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[9]
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1175,7 +1261,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
func (*EngineProfile) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{9}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
}
func (x *EngineProfile) GetName() string {
@@ -1265,7 +1351,7 @@ type StopShard struct {
func (x *StopShard) Reset() {
*x = StopShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1277,7 +1363,7 @@ func (x *StopShard) String() string {
func (*StopShard) ProtoMessage() {}
func (x *StopShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[10]
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1290,7 +1376,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
func (*StopShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{10}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
}
func (x *StopShard) GetShardId() string {
@@ -1310,7 +1396,7 @@ type DrainShard struct {
func (x *DrainShard) Reset() {
*x = DrainShard{}
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1322,7 +1408,7 @@ func (x *DrainShard) String() string {
func (*DrainShard) ProtoMessage() {}
func (x *DrainShard) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[11]
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1335,7 +1421,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
func (*DrainShard) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{11}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
}
func (x *DrainShard) GetShardId() string {
@@ -1357,7 +1443,7 @@ type Ack struct {
func (x *Ack) Reset() {
*x = Ack{}
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1369,7 +1455,7 @@ func (x *Ack) String() string {
func (*Ack) ProtoMessage() {}
func (x *Ack) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1382,7 +1468,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
func (*Ack) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
}
func (x *Ack) GetMessageId() string {
@@ -1421,7 +1507,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
"\x03msg\"\xc8\x04\n" +
"\x03msg\"\xfa\x04\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -1433,7 +1519,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" +
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
"\x0fmemory_total_mb\x18\n" +
" \x01(\x03R\rmemoryTotalMb\x1a9\n" +
" \x01(\x03R\rmemoryTotalMb\x120\n" +
"\aweights\x18\v \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
@@ -1481,7 +1568,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x10FabricAttachment\x12\x16\n" +
"\x06fabric\x18\x01 \x01(\tR\x06fabric\x12\x12\n" +
"\x04hcas\x18\x02 \x03(\tR\x04hcas\x12\x1b\n" +
"\trate_gbps\x18\x03 \x01(\x05R\brateGbps\"t\n" +
"\trate_gbps\x18\x03 \x01(\x05R\brateGbps\"Y\n" +
"\fLocalWeights\x12\x12\n" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"t\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
@@ -1571,7 +1663,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 16)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 17)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -1580,41 +1672,43 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*RunningShard)(nil), // 4: llmgw.v1.RunningShard
(*GPU)(nil), // 5: llmgw.v1.GPU
(*FabricAttachment)(nil), // 6: llmgw.v1.FabricAttachment
(*Heartbeat)(nil), // 7: llmgw.v1.Heartbeat
(*ShardStatus)(nil), // 8: llmgw.v1.ShardStatus
(*StartShard)(nil), // 9: llmgw.v1.StartShard
(*EngineProfile)(nil), // 10: llmgw.v1.EngineProfile
(*StopShard)(nil), // 11: llmgw.v1.StopShard
(*DrainShard)(nil), // 12: llmgw.v1.DrainShard
(*Ack)(nil), // 13: llmgw.v1.Ack
nil, // 14: llmgw.v1.Register.LabelsEntry
nil, // 15: llmgw.v1.Register.EngineVersionsEntry
nil, // 16: llmgw.v1.EngineProfile.EnvEntry
(*LocalWeights)(nil), // 7: llmgw.v1.LocalWeights
(*Heartbeat)(nil), // 8: llmgw.v1.Heartbeat
(*ShardStatus)(nil), // 9: llmgw.v1.ShardStatus
(*StartShard)(nil), // 10: llmgw.v1.StartShard
(*EngineProfile)(nil), // 11: llmgw.v1.EngineProfile
(*StopShard)(nil), // 12: llmgw.v1.StopShard
(*DrainShard)(nil), // 13: llmgw.v1.DrainShard
(*Ack)(nil), // 14: llmgw.v1.Ack
nil, // 15: llmgw.v1.Register.LabelsEntry
nil, // 16: llmgw.v1.Register.EngineVersionsEntry
nil, // 17: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
7, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
8, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
9, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
11, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
12, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
13, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
14, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
8, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
9, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
10, // 3: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
12, // 4: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
13, // 5: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
14, // 6: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
15, // 7: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
5, // 8: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
6, // 9: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
16, // 10: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
4, // 11: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
0, // 12: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
0, // 13: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
10, // 14: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
16, // 15: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 16: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 17: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
17, // [17:18] is the sub-list for method output_type
16, // [16:17] is the sub-list for method input_type
16, // [16:16] is the sub-list for extension type_name
16, // [16:16] is the sub-list for extension extendee
0, // [0:16] is the sub-list for field type_name
7, // 12: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
0, // 13: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
0, // 14: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
11, // 15: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
17, // 16: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 17: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 18: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
18, // [18:19] is the sub-list for method output_type
17, // [17:18] is the sub-list for method input_type
17, // [17:17] is the sub-list for extension type_name
17, // [17:17] is the sub-list for extension extendee
0, // [0:17] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -1639,7 +1733,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 16,
NumMessages: 17,
NumExtensions: 0,
NumServices: 1,
},
+26
View File
@@ -76,6 +76,20 @@ message Register {
// ready node looks like a candidate: a 14B model whose working set is 24 GB
// would be placed just as happily on a node with 8 GB.
int64 memory_total_mb = 10;
// weights are the model files this node can see, from the directories its
// configuration names.
//
// How they got there is the operator's business and deliberately outside
// llmgw's: a shared filesystem, an image exported over the fabric, rsync, or
// a copy made by hand all look identical here. What matters to placement is
// only whether they are present, because a node without them either cannot
// start the replica at all or must first move tens of gigabytes.
//
// Empty means the node was not asked to look, which is not the same as having
// nothing — an agent with no model directories configured behaves exactly as
// every agent did before this existed, and constrains nothing.
repeated LocalWeights weights = 11;
}
// RunningShard describes an engine already running on a node.
@@ -135,6 +149,18 @@ message FabricAttachment {
}
// Heartbeat says the node is alive and reports anything that changed cheaply.
// LocalWeights is one model file or directory a node can see.
message LocalWeights {
// path is what an engine would be given, so it can be compared against a
// model's source without interpretation.
string path = 1;
int64 size_bytes = 2;
// shared marks weights on a filesystem the node did not have to fill —
// reported so a fabric with a shared filer is not mistaken for one where
// every node happens to have fetched its own copy.
bool shared = 3;
}
message Heartbeat {
string node_id = 1;
// gpu_memory_used_mb, indexed by GPU, catches a foreign process appearing on
+17
View File
@@ -57,6 +57,23 @@ discovery:
prefer_nvml: true
skip_gpus: false
# Where this node keeps model weights.
#
# Unset by default, and unset means the node is not asked to look — which is
# not the same as having nothing, and constrains nothing. Storage differs at
# every installation and llmgw deliberately does not own it: a Lustre mount, an
# image exported over the fabric with SRP or NVMe-oF, local NVMe, or a copy
# made by hand all look identical from here.
#
# What it buys is placement preferring a node that already holds the model over
# one that must first move tens of gigabytes. That matters most where fabrics
# are isolated from one another, because weights in one fabric are not weights
# in another and the only path between them is the management network.
#
# model_dirs:
# - /var/lib/llmgw-models
# - /mnt/lustre/models
engines:
vllm:
bin: /opt/venvs/vllm/bin/vllm
+28
View File
@@ -282,6 +282,34 @@ sudo install -d -o root -g root /var/lib/llmgw-models
For a first model, something small: a 3B at `q4_k_m` is about 2 GB, runs on
CPU, and answers fast enough to prove the whole path works.
### Telling llmgw where they are
Optional, and unset by default. Naming the directories in the agent's
`model_dirs` lets a node report which models it can see, and placement then
prefers a node that already holds one over a node that must first move tens of
gigabytes:
```yaml
model_dirs:
- /var/lib/llmgw-models
- /mnt/lustre/models
```
llmgw does not manage that storage and does not care how the files arrived. A
Lustre or NFS mount, an image exported over the fabric with SRP or NVMe-oF,
local NVMe, or an `scp` all look identical to it. A node with nothing configured
reports nothing and is treated exactly as every node was before this existed.
**It matters most when your fabrics are not connected to each other.** Weights
in `ib-a` are not weights in `ib-b`: the only path between them is the
management network, which is the slow one. `llmgw ctl status` shows what each
fabric holds, so a model seeded into one fabric does not quietly leave the rest
of the cluster unable to serve it.
How much disk a node needs follows from the models rather than a rule of thumb.
`llmgw ctl model plan <repo>` prints the sizes; as a starting point, a 70B at
bf16 is 141 GB and the same model at q4 is 43 GB.
## 5. Adding the node
Two halves, because they need different access: building the machine needs the
+15 -1
View File
@@ -58,6 +58,16 @@ type Config struct {
// keeps the plaintext behaviour, which only belongs on a trusted network.
TLS tlsconf.Config `koanf:"tls"`
// ModelDirs are the directories this node keeps model weights in.
//
// Empty by default, and empty means the node is not asked to look — which
// is not the same as having nothing. Storage differs at every installation,
// so llmgw observes it rather than owning it: a Lustre mount, an image
// exported over the fabric, local NVMe and a copy made by hand all look
// identical from here, and placement only needs to know whether the file
// is present.
ModelDirs []string `koanf:"model_dirs"`
Discovery discovery.Options `koanf:"discovery"`
Engines engines.Config `koanf:"engines"`
Superv supervisor.Config `koanf:"supervisor"`
@@ -368,7 +378,11 @@ func int32Slice(v []int) []int32 {
// scan refreshes the node's hardware inventory.
func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, a.cfg.Discovery, a.log)
// Carried from the agent's own configuration rather than duplicated into
// the discovery block, so an operator names the directories once.
opts := a.cfg.Discovery
opts.ModelDirs = a.cfg.ModelDirs
inv, err := discovery.Discover(ctx, opts, a.log)
a.mu.Lock()
a.inv = inv
a.mu.Unlock()
+6
View File
@@ -26,6 +26,8 @@ type Options struct {
PreferNVML bool `koanf:"prefer_nvml"`
// SkipGPUs omits GPU discovery, for nodes that have none.
SkipGPUs bool `koanf:"skip_gpus"`
// ModelDirs are scanned for weights. Empty means do not look.
ModelDirs []string `koanf:"-"`
}
// Discover builds the node's inventory.
@@ -45,6 +47,10 @@ func Discover(ctx context.Context, opts Options, log *slog.Logger) (Inventory, e
// has nothing to do with cards goes with it. That is exactly the node whose
// memory matters most, since a CPU profile is what it will be asked to run.
inv.MemoryTotalMB = MemoryTotalMB()
// Also before the GPU paths, and for the same reason: which weights a node
// holds has nothing to do with its cards, and a node with none is exactly
// where a CPU profile will look for them.
inv.Weights = ScanWeights(opts.ModelDirs)
if !opts.SkipGPUs {
switch {
+3
View File
@@ -103,6 +103,9 @@ type Inventory struct {
// MemoryTotalMB is the machine's RAM. Zero means unknown, which is what an
// agent older than this field reports, and is treated as no constraint.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
// Weights are the model files this node can see. Empty means it was not
// asked to look.
Weights []LocalWeights `json:"weights,omitempty"`
}
// Classes returns the distinct GPU classes present, sorted.
+137
View File
@@ -0,0 +1,137 @@
package discovery
import (
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
)
// LocalWeights is one model file or directory a node can see.
type LocalWeights struct {
Path string `json:"path"`
SizeBytes int64 `json:"size_bytes"`
// Shared marks weights on a filesystem this node did not have to fill.
//
// Worth distinguishing: twenty nodes each holding their own copy and twenty
// nodes seeing one filer are the same list of paths and completely
// different situations — one has twenty copies of a 141 GB model and the
// other has one.
Shared bool `json:"shared,omitempty"`
}
// ScanWeights lists the model files under the given directories.
//
// Storage is the part of this system that will differ at every installation —
// a Lustre mount, an image exported over the fabric, local NVMe, an rsync from
// a laptop — so llmgw does not try to own it. It observes the result instead:
// whatever put the file there, the question placement needs answered is only
// whether it is present.
//
// Scanning nothing is the default and constrains nothing. A node with no
// directories configured behaves exactly as every node did before this existed,
// which is what makes the whole feature optional.
func ScanWeights(dirs []string) []LocalWeights {
var out []LocalWeights
seen := map[string]struct{}{}
for _, dir := range dirs {
if dir == "" {
continue
}
shared := isNetworkFS(dir)
entries, err := os.ReadDir(dir)
if err != nil {
// A directory that is not there is not an error to report upward:
// a mount that has not come back yet looks exactly like this, and
// the node is simply one without those weights until it does.
continue
}
for _, e := range entries {
path := filepath.Join(dir, e.Name())
if _, dup := seen[path]; dup {
continue
}
size, ok := weightSize(path, e)
if !ok {
continue
}
seen[path] = struct{}{}
out = append(out, LocalWeights{Path: path, SizeBytes: size, Shared: shared})
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
return out
}
// weightSize measures a candidate and says whether it is one at all.
//
// A model is either a single file — a .gguf, usually — or a directory of
// shards and a config, which is what every HuggingFace checkout looks like.
// Anything else in the directory is somebody's notes and is not reported.
func weightSize(path string, e fs.DirEntry) (int64, bool) {
if e.IsDir() {
// A directory counts only if it actually holds weights. An empty one
// is a half-finished download, and reporting it as present would have
// placement choose a node whose engine then fails to load.
var total int64
var found bool
_ = filepath.WalkDir(path, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return nil //nolint:nilerr // an unreadable subtree is not fatal
}
if isWeightFile(p) {
found = true
}
if info, err := d.Info(); err == nil {
total += info.Size()
}
return nil
})
return total, found
}
if !isWeightFile(path) {
return 0, false
}
info, err := e.Info()
if err != nil {
return 0, false
}
return info.Size(), true
}
func isWeightFile(path string) bool {
switch strings.ToLower(filepath.Ext(path)) {
case ".gguf", ".safetensors", ".bin", ".pt", ".pth":
return true
}
return false
}
// isNetworkFS reports whether a path is on a filesystem the node did not fill.
//
// By magic number rather than by parsing mounts, because the mount table says
// what was asked for and this says what the kernel actually has. The list is
// the network filesystems that turn up in front of GPUs: NFS, Lustre, GPFS,
// CephFS, GlusterFS and SMB. Anything unrecognised is treated as local, which
// is the conservative direction — a shared filer mistaken for local storage
// costs an unnecessary copy, while local storage mistaken for shared would have
// the cluster believe a model is everywhere when it is on one node.
func isNetworkFS(path string) bool {
var st syscall.Statfs_t
if err := syscall.Statfs(path, &st); err != nil {
return false
}
switch uint32(st.Type) { //nolint:unconvert // Type is int64 on some platforms
case 0x6969, // NFS
0x0BD00BD0, // Lustre
0x47504653, // GPFS
0x00C36400, // CephFS
0x65735546, // FUSE, which is how GlusterFS and many object mounts appear
0xFF534D42: // SMB/CIFS
return true
}
return false
}
+114
View File
@@ -0,0 +1,114 @@
package discovery
import (
"os"
"path/filepath"
"testing"
)
func write(t *testing.T, path string, size int) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, make([]byte, size), 0o644); err != nil {
t.Fatal(err)
}
}
func TestScanFindsFilesAndDirectoriesOfWeights(t *testing.T) {
dir := t.TempDir()
write(t, filepath.Join(dir, "qwen-3b-q4.gguf"), 1024)
// A HuggingFace checkout: a directory of shards rather than one file.
write(t, filepath.Join(dir, "Llama-3.3-70B", "model-00001.safetensors"), 2048)
write(t, filepath.Join(dir, "Llama-3.3-70B", "config.json"), 64)
got := ScanWeights([]string{dir})
if len(got) != 2 {
t.Fatalf("found %d entries: %+v", len(got), got)
}
byName := map[string]int64{}
for _, w := range got {
byName[filepath.Base(w.Path)] = w.SizeBytes
}
if byName["qwen-3b-q4.gguf"] != 1024 {
t.Errorf("gguf size = %d", byName["qwen-3b-q4.gguf"])
}
// The directory's size is everything in it, config included: what matters
// is how much would have to be moved to put it somewhere else.
if byName["Llama-3.3-70B"] != 2048+64 {
t.Errorf("checkout size = %d, want %d", byName["Llama-3.3-70B"], 2048+64)
}
}
func TestAnEmptyDirectoryIsNotWeights(t *testing.T) {
// A half-finished download looks exactly like this, and reporting it as
// present would have placement choose a node whose engine then fails to
// load — the failure this whole feature exists to avoid.
dir := t.TempDir()
if err := os.MkdirAll(filepath.Join(dir, "downloading"), 0o755); err != nil {
t.Fatal(err)
}
write(t, filepath.Join(dir, "downloading", "README.md"), 10)
if got := ScanWeights([]string{dir}); len(got) != 0 {
t.Errorf("a directory with no weight files was reported: %+v", got)
}
}
func TestNotesAndScriptsAreNotReportedAsModels(t *testing.T) {
dir := t.TempDir()
write(t, filepath.Join(dir, "notes.txt"), 10)
write(t, filepath.Join(dir, "fetch.sh"), 10)
write(t, filepath.Join(dir, "real.gguf"), 10)
got := ScanWeights([]string{dir})
if len(got) != 1 || filepath.Base(got[0].Path) != "real.gguf" {
t.Errorf("scan returned %+v", got)
}
}
func TestAMissingDirectoryIsNotAnError(t *testing.T) {
// A mount that has not come back yet looks exactly like a directory that is
// not there, and the node is simply one without those weights until it is.
got := ScanWeights([]string{filepath.Join(t.TempDir(), "never-existed")})
if len(got) != 0 {
t.Errorf("got %+v", got)
}
}
func TestScanningNothingIsTheDefaultAndConstrainsNothing(t *testing.T) {
// What makes the feature optional: an agent with no directories configured
// behaves exactly as every agent did before this existed.
if got := ScanWeights(nil); len(got) != 0 {
t.Errorf("got %+v", got)
}
if got := ScanWeights([]string{""}); len(got) != 0 {
t.Errorf("an empty directory name produced %+v", got)
}
}
func TestTheSamePathIsReportedOnce(t *testing.T) {
// Two configured directories can overlap, or be the same mount reached two
// ways. Counting a model twice would overstate what a fabric holds.
dir := t.TempDir()
write(t, filepath.Join(dir, "m.gguf"), 10)
if got := ScanWeights([]string{dir, dir}); len(got) != 1 {
t.Errorf("the same directory twice produced %+v", got)
}
}
func TestLocalStorageIsNeverMistakenForShared(t *testing.T) {
// The conservative direction. A shared filer read as local costs an
// unnecessary copy; local read as shared would have the cluster believe a
// model is everywhere when it is on one node.
dir := t.TempDir()
write(t, filepath.Join(dir, "m.gguf"), 10)
got := ScanWeights([]string{dir})
if len(got) != 1 {
t.Fatalf("got %+v", got)
}
if got[0].Shared {
t.Error("a temporary directory on local disk was reported as shared")
}
}
+19 -2
View File
@@ -70,9 +70,10 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
// The free-block column is the one that decides whether a multi-node
// replica can land: free cards on the wrong fabric are not capacity for
// a model that spans nodes, because a replica may never cross one.
fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE")
fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE\tWEIGHTS")
for _, f := range s.Fabrics {
fmt.Fprintf(w, " %s\t%d\t%d\n", f.Fabric, f.Nodes, f.LargestFreeBlock)
fmt.Fprintf(w, " %s\t%d\t%d\t%s\n",
f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f))
}
_ = w.Flush()
}
@@ -114,6 +115,22 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
}
}
// weightsCell says what a fabric holds, and stays quiet when nobody asked.
//
// Fabrics are not connected to one another, so weights in one are not weights
// in another: a model seeded into a single fabric leaves the rest of the
// cluster unable to serve it without first moving tens of gigabytes over the
// management network. This is the column that shows that before someone
// discovers it during a scale-up.
func weightsCell(f adminapi.FabricUse) string {
if f.NodesWithWeights == 0 {
// Nobody on this fabric was asked to look, which is the default and is
// not the same as an empty fabric.
return "-"
}
return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes)
}
// bar draws usage so a column of classes can be read at a glance rather than
// compared digit by digit.
func bar(used, total int) string {
+9 -3
View File
@@ -59,9 +59,12 @@ type NodeView struct {
// MemoryGB is the machine's RAM, 0 when the agent did not report it. It is
// what decides whether a CPU profile can be placed here, so it belongs
// where someone looks when a model will not schedule.
MemoryGB int `json:"memory_gb,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
MemoryGB int `json:"memory_gb,omitempty"`
// Models are the weights this node can see. Absent means it was not asked
// to look, which is not the same as holding none.
Models []string `json:"models_on_disk,omitempty"`
Classes []string `json:"gpu_classes,omitempty"`
Fabrics []string `json:"fabrics,omitempty"`
// Unhealthy lists GPUs kept out of placement and why, which is usually the
// answer when a node looks idle but nothing will schedule onto it.
@@ -163,6 +166,9 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion,
Replicas: replicasOn[n.ID],
}
for _, w := range n.Weights {
v.Models = append(v.Models, w.Path)
}
seen := map[string]struct{}{}
for _, g := range n.GPUs {
if _, dup := seen[g.Class()]; !dup {
+29 -1
View File
@@ -56,6 +56,16 @@ func (c ClassUse) Free() int { return c.Total - c.Used - c.Unhealthy }
type FabricUse struct {
Fabric string `json:"fabric"`
Nodes int `json:"nodes"`
// ModelsPresent is how many nodes here hold weights at all, and Models how
// many distinct models they hold between them.
//
// Per fabric because that is the unit at which the answer is actionable.
// Fabrics are not connected to one another, so weights in one are not
// weights in another: a model seeded into a single fabric leaves the rest
// of the cluster unable to serve it without first moving tens of gigabytes
// over the management network, and nothing said so.
NodesWithWeights int `json:"nodes_with_weights"`
Models int `json:"models_present"`
// LargestFreeBlock is the most nodes on this fabric that are entirely free,
// which is the number that decides whether a multi-node replica can land.
LargestFreeBlock int `json:"largest_free_block"`
@@ -134,6 +144,9 @@ func (a *API) Summary() Summary {
byFabric[f] = &FabricUse{Fabric: f}
}
byFabric[f].Nodes++
if len(n.Models) > 0 {
byFabric[f].NodesWithWeights++
}
// Entirely free, which is what a multi-node replica needs: a node
// with one card in use cannot contribute a full-width shard.
if n.State == "ready" && n.GPUFree == n.GPUTotal && n.GPUTotal > 0 {
@@ -141,7 +154,22 @@ func (a *API) Summary() Summary {
}
}
}
for _, f := range byFabric {
// Distinct models per fabric, counted across its nodes: one copy on a
// shared filer and twenty local copies are the same answer to "can this
// fabric serve it".
perFabricModels := map[string]map[string]struct{}{}
for _, n := range nodes {
for _, f := range n.Fabrics {
if perFabricModels[f] == nil {
perFabricModels[f] = map[string]struct{}{}
}
for _, m := range n.Models {
perFabricModels[f][m] = struct{}{}
}
}
}
for name, f := range byFabric {
f.Models = len(perFabricModels[name])
s.Fabrics = append(s.Fabrics, *f)
}
sort.Slice(s.Fabrics, func(i, j int) bool { return s.Fabrics[i].Fabric < s.Fabrics[j].Fabric })
+44
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -109,3 +110,46 @@ func TestSummaryNamesModelsThatCannotBePlaced(t *testing.T) {
t.Error("no model was counted short")
}
}
func TestWeightsAreCountedPerFabric(t *testing.T) {
// Fabrics are not connected, so weights in one are not weights in another.
// A model seeded into a single fabric leaves the rest of the cluster unable
// to serve it without first moving tens of gigabytes over the management
// network, and nothing said so.
a1 := gpuNode("a1", 4, "ib-a")
a1.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
a2 := gpuNode("a2", 4, "ib-a")
a2.Weights = []discovery.LocalWeights{{Path: "/models/llama.gguf"}}
b1 := gpuNode("b1", 4, "ib-b") // the same fabric capacity, none of the weights
api, _ := setup(t, testModels(), a1, a2, b1)
s := api.Summary()
byName := map[string]FabricUse{}
for _, f := range s.Fabrics {
byName[f.Fabric] = f
}
if got := byName["ib-a"]; got.Models != 1 || got.NodesWithWeights != 2 {
t.Errorf("ib-a = %+v, want one model on two nodes", got)
}
if got := byName["ib-b"]; got.Models != 0 || got.NodesWithWeights != 0 {
t.Errorf("ib-b = %+v; it holds nothing and should say so", got)
}
}
func TestOneSharedCopyCountsAsMuchAsTwenty(t *testing.T) {
// A filer mounted on every node and a local copy on every node are the same
// answer to "can this fabric serve it", which is the question being asked.
var nodes []inventory.Node
for _, id := range []string{"n1", "n2", "n3"} {
n := gpuNode(id, 4, "ib-a")
n.Weights = []discovery.LocalWeights{{Path: "/mnt/shared/llama.gguf", Shared: true}}
nodes = append(nodes, n)
}
api, _ := setup(t, testModels(), nodes...)
f := api.Summary().Fabrics[0]
if f.Models != 1 {
t.Errorf("counted %d models; three nodes seeing one file is one model", f.Models)
}
}
+10
View File
@@ -25,6 +25,11 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
EngineVersions: r.GetEngineVersions(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
}
for _, w := range r.GetWeights() {
n.Weights = append(n.Weights, discovery.LocalWeights{
Path: w.GetPath(), SizeBytes: w.GetSizeBytes(), Shared: w.GetShared(),
})
}
for _, g := range r.GetGpus() {
n.GPUs = append(n.GPUs, discovery.GPU{
Index: int(g.GetIndex()),
@@ -64,6 +69,11 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
Labels: labels, AgentVersion: agentVersion,
MemoryTotalMb: int64(inv.MemoryTotalMB),
}
for _, w := range inv.Weights {
r.Weights = append(r.Weights, &llmgwv1.LocalWeights{
Path: w.Path, SizeBytes: w.SizeBytes, Shared: w.Shared,
})
}
for _, g := range inv.GPUs {
r.Gpus = append(r.Gpus, &llmgwv1.GPU{
Index: int32(g.Index),
+3
View File
@@ -322,6 +322,9 @@ func (m Model) PlacementProfiles() []placement.Profile {
Priority: p.Priority,
NodeSelector: p.NodeSelector,
MinMemoryGB: p.MinMemoryGB,
// The model's source, so placement can prefer nodes that already
// hold the weights rather than one that must first move them.
Source: m.Source,
})
}
return out
+26
View File
@@ -80,6 +80,11 @@ type Node struct {
// this field existed.
MemoryTotalMB int `json:"memory_total_mb,omitempty"`
// Weights are the model files this node can see, however they got there.
// Empty means the node was not asked to look, which is not the same as
// holding none — so it constrains nothing.
Weights []discovery.LocalWeights `json:"weights,omitempty"`
GPUs []discovery.GPU `json:"gpus"`
// Fabrics is empty on a node without IB. Such a node is its own
// single-node island and can never host a multi-node replica.
@@ -305,6 +310,27 @@ func (s *Store) FreeGPUs(nodeID string) []int {
return out
}
// HasWeights reports whether this node can see the given source.
//
// An exact path match, deliberately. A source that is not a path — a model id
// an engine will resolve itself — cannot be checked from here, and guessing
// would be worse than the honest answer that nothing is known.
func (n Node) HasWeights(source string) bool {
for _, w := range n.Weights {
if w.Path == source {
return true
}
}
return false
}
// KnowsItsWeights reports whether this node was asked to look at all.
//
// The distinction that keeps the feature optional: a node reporting no weights
// because nobody configured a directory must not be excluded from placement,
// while a node that looked and genuinely has none should be.
func (n Node) KnowsItsWeights() bool { return len(n.Weights) > 0 }
// Fabrics lists every fabric any node is attached to, sorted.
func (s *Store) Fabrics() []string {
s.mu.RLock()
+49
View File
@@ -36,6 +36,9 @@ type Profile struct {
NodeSelector map[string]string
// MinMemoryGB is the host RAM this profile needs. Zero is unconstrained.
MinMemoryGB int
// Source is the model's weights, as the engine would be given it. Used to
// prefer nodes that already hold them; empty asks for no preference.
Source string
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
@@ -193,6 +196,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return len(s.inv.Allocations(candidates[i].node.ID)) <
len(s.inv.Allocations(candidates[j].node.ID))
})
preferNodesHoldingWeights(p.Source, candidates)
return &Placement{
Profile: p,
Pool: "none/cpu",
@@ -200,6 +204,11 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
}, rejections
}
// Last, so it takes precedence: moving tens of gigabytes costs more than an
// imperfectly packed node, and with isolated fabrics it may cost a
// cross-fabric copy over the management network.
preferNodesHoldingWeights(p.Source, candidates)
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode)
if gpus == nil {
@@ -291,6 +300,46 @@ type candidate struct {
//
// fabric is empty for single-node placement, in which case attachment is not
// checked at all.
// preferNodesHoldingWeights moves nodes that already have the model to the
// front, leaving everything else in the order it was given.
//
// A stable partition rather than a sort, so every other consideration — bin
// packing, spreading, the emptiest machine for a multi-node replica — survives
// intact among nodes that are equal on this one.
func preferNodesHoldingWeights(source string, candidates []candidate) {
if source == "" {
return
}
sort.SliceStable(candidates, func(i, j int) bool {
a, aKnown := hasWeights(source, candidates[i].node)
b, bKnown := hasWeights(source, candidates[j].node)
// A node that was never asked to look ranks with those that looked and
// have nothing: unknown is not a reason to prefer or to punish.
return (a && aKnown) && !(b && bKnown)
})
}
// hasWeights reports whether a node already holds this profile's model, and
// whether that is known at all.
//
// Never a filter, always a preference. Storage differs at every installation
// and llmgw does not own it: a node that was never asked to look reports
// nothing, and excluding those would break every cluster that has not opted
// in. Even a node that looked and does not have the weights stays eligible,
// because "fetch them" is a valid answer and refusing to place would turn a
// slow start into no start at all.
//
// What it changes is the ordering, and that is worth a lot on its own: with
// isolated fabrics the difference between placing where the weights are and
// placing where they are not is a cross-fabric copy of tens of gigabytes over
// the management network.
func hasWeights(source string, n inventory.Node) (present, known bool) {
if source == "" || !n.KnowsItsWeights() {
return false, false
}
return n.HasWeights(source), true
}
// unmatchedLabel reports the first selector entry a node does not satisfy.
//
// It returns the label rather than a bare false so the rejection can name it:
@@ -582,3 +582,100 @@ func TestAProfileWithNoMemoryRequirementFitsAnywhere(t *testing.T) {
t.Fatalf("an unconstrained profile was rejected:\n%s", res.Explain())
}
}
func nodeWithWeights(id string, gpus int, paths ...string) inventory.Node {
n := node(id, "h100-80g", gpus)
for _, p := range paths {
n.Weights = append(n.Weights, discovery.LocalWeights{Path: p, SizeBytes: 1 << 30})
}
return n
}
func TestPlacementPrefersANodeThatAlreadyHasTheWeights(t *testing.T) {
// With isolated fabrics the difference between placing where the weights
// are and placing where they are not is a cross-fabric copy of tens of
// gigabytes over the management network.
const src = "/models/llama.gguf"
without := nodeWithWeights("empty", 8, "/models/something-else.gguf")
with := nodeWithWeights("holds-it", 8, src)
s := New(store(without, with))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing was placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "holds-it" {
t.Errorf("placed on %q; the other node already has the weights", got)
}
}
func TestANodeWithoutTheWeightsIsStillEligible(t *testing.T) {
// Fetching them is a valid answer. Refusing to place would turn a slow
// start into no start at all, which is a worse cluster than a slow one.
const src = "/models/llama.gguf"
only := nodeWithWeights("no-weights", 8, "/models/other.gguf")
s := New(store(only))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Errorf("a node without the weights was excluded:\n%s", res.Explain())
}
}
func TestANodeThatWasNeverAskedIsNotPunished(t *testing.T) {
// What keeps the whole feature optional. A node with no model directories
// configured reports nothing, and that must rank with "looked and has
// none" rather than below it — otherwise enabling the feature on one node
// would quietly drain every other.
const src = "/models/llama.gguf"
silent := node("never-asked", "h100-80g", 8) // reports no weights at all
looked := nodeWithWeights("looked", 8, "/models/other.gguf")
s := New(store(silent, looked))
for range 4 {
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
}
// Both remain candidates; neither is preferred on weights, so the ordinary
// rules decide and nothing errors.
}
func TestWeightsDoNotOverrideHardConstraints(t *testing.T) {
// A preference, never a filter. The node holding the weights here is the
// wrong GPU class, and choosing it would produce a replica that cannot run.
const src = "/models/llama.gguf"
wrongClass := node("wrong", "a100-40g", 8)
wrongClass.Weights = []discovery.LocalWeights{{Path: src}}
right := node("right", "h100-80g", 8)
s := New(store(wrongClass, right))
res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1, Source: src,
}}, nil)
if res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "right" {
t.Errorf("placed on %q, which is the wrong GPU class", got)
}
}
func TestAProfileWithNoSourceIsUnaffected(t *testing.T) {
// Every profile before this existed.
a := nodeWithWeights("a", 8, "/models/x.gguf")
b := node("b", "h100-80g", 8)
s := New(store(a, b))
if res := s.Place([]Profile{{
Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1,
}}, nil); res.Placement == nil {
t.Fatalf("nothing placed:\n%s", res.Explain())
}
}