Place on nodes that can actually hold the model
Placement had no idea how much memory a node has or a model needs. A CPU profile has no card to be free, so eligibleNodes accepted every ready node: Qwen2.5-Coder 14B, whose working set is about 24 GB, would have been placed just as happily on one of the 8 GB nodes, which reports itself perfectly healthy right up until the OOM killer arrives. node_selector was yesterday's stopgap and it is the operator restating by hand something the system can know. Agents now report MemTotal, profiles declare min_memory_gb, and placement filters on it, naming what was short — "has 8 GB of memory, needs 24" is actionable where "no candidates were considered" is not. The label selector stays, because "this node is for coding models" is a real thing to want to say that memory does not express. Two deliberate non-exclusions. A profile with no requirement fits anywhere, which is every profile that exists today. And a node that reports no memory is still usable: an agent older than the field says nothing, and letting a control plane upgrade quietly empty half a fleet is a worse failure than the one this prevents. ctl nodes grows a MEM column showing "-" rather than 0 for a node that did not say, because an old agent is not a small machine. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e9fe3ad379
commit
3081c6f69b
13 files changed
+233
-8
No files matched your search
@@ -336,6 +336,14 @@ type Register struct {
|
||||
// — so the new engines fail to allocate, with an error naming the wrong
|
||||
// process (plan §6.5).
|
||||
RunningShards []*RunningShard `protobuf:"bytes,9,rep,name=running_shards,json=runningShards,proto3" json:"running_shards,omitempty"`
|
||||
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||||
// which is what every agent older than this field reports, and is treated as
|
||||
// no constraint rather than as a node with no memory.
|
||||
//
|
||||
// Placement needs it because a CPU profile has no card to be free, so every
|
||||
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||||
// would be placed just as happily on a node with 8 GB.
|
||||
MemoryTotalMb int64 `protobuf:"varint,10,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -433,6 +441,13 @@ func (x *Register) GetRunningShards() []*RunningShard {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Register) GetMemoryTotalMb() int64 {
|
||||
if x != nil {
|
||||
return x.MemoryTotalMb
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// RunningShard describes an engine already running on a node.
|
||||
//
|
||||
// It carries enough for the control plane to rebuild the replica it belongs to,
|
||||
@@ -1406,7 +1421,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
"\x05drain\x18\x03 \x01(\v2\x14.llmgw.v1.DrainShardH\x00R\x05drain\x12!\n" +
|
||||
"\x03ack\x18\x04 \x01(\v2\r.llmgw.v1.AckH\x00R\x03ackB\x05\n" +
|
||||
"\x03msg\"\xa0\x04\n" +
|
||||
"\x03msg\"\xc8\x04\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
|
||||
@@ -1416,7 +1431,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\afabrics\x18\x06 \x03(\v2\x1a.llmgw.v1.FabricAttachmentR\afabrics\x12#\n" +
|
||||
"\ragent_version\x18\a \x01(\tR\fagentVersion\x12O\n" +
|
||||
"\x0fengine_versions\x18\b \x03(\v2&.llmgw.v1.Register.EngineVersionsEntryR\x0eengineVersions\x12=\n" +
|
||||
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x1a9\n" +
|
||||
"\x0erunning_shards\x18\t \x03(\v2\x16.llmgw.v1.RunningShardR\rrunningShards\x12&\n" +
|
||||
"\x0fmemory_total_mb\x18\n" +
|
||||
" \x01(\x03R\rmemoryTotalMb\x1a9\n" +
|
||||
"\vLabelsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
|
||||
|
||||
@@ -67,6 +67,15 @@ message Register {
|
||||
// — so the new engines fail to allocate, with an error naming the wrong
|
||||
// process (plan §6.5).
|
||||
repeated RunningShard running_shards = 9;
|
||||
|
||||
// memory_total_mb is the machine's RAM, and 0 means the agent did not say —
|
||||
// which is what every agent older than this field reports, and is treated as
|
||||
// no constraint rather than as a node with no memory.
|
||||
//
|
||||
// Placement needs it because a CPU profile has no card to be free, so every
|
||||
// ready node looks like a candidate: a 14B model whose working set is 24 GB
|
||||
// would be placed just as happily on a node with 8 GB.
|
||||
int64 memory_total_mb = 10;
|
||||
}
|
||||
|
||||
// RunningShard describes an engine already running on a node.
|
||||
|
||||
Reference in new issue
Block a user