Let a node describe itself instead of being described by hand

Every agent file on the cluster named its own node, its own address, the
path to llama-server and two weight directories — different on every node,
kept true by hand, and not true: node05 carried "ram: 32g" beside a measured
31 GB, looked for weights in an empty directory while fetching them into
another, and the other four never looked in theirs at all, so four nodes
reported no disk. Growing a node's RAM would have gone unnoticed, because
memory was read once, at registration.

The agent now finds these out. Its id is the hostname, which is also the
certificate's common name. Its address is the one on the route to the
control plane, which the kernel picks for a UDP dial without sending
anything. llama-server is looked for on PATH and then where release archives
are unpacked. A missing nvidia-smi is a node without NVIDIA cards rather
than a failed discovery — which also cut discovery short before InfiniBand
on every CPU node not told to skip GPUs. Fetched weights land in
/var/lib/llmgw-weights by default, and wherever they land is always looked
in.

Memory is read on every heartbeat, and installed engines are probed again
each minute and sent when they change, so a VM given more RAM or a node
given vLLM becomes a candidate without anyone restarting its agent. The
control plane logs both changes, since "the node now qualifies" is the
answer to why a model placed an hour after it was asked for.

What is written still wins. The agent file shrinks to the control plane's
addresses and the TLS files, the same on every node; onboarding writes only
the overrides it was given. What cannot be found stays configuration: which
fabric an adapter is on, because OpenSM's default prefix is the same on all
of them.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 10:13:42 +02:00
1 parent 2cae92d113
commit 20deb22dfa
14 files changed
+552 -134

No files matched your search

+55 -16
View File
@@ -1060,6 +1060,16 @@ type Heartbeat struct {
// report", and the first would quietly empty a node's inventory.
WeightsChanged bool `protobuf:"varint,5,opt,name=weights_changed,json=weightsChanged,proto3" json:"weights_changed,omitempty"`
Weights []*LocalWeights `protobuf:"bytes,6,rep,name=weights,proto3" json:"weights,omitempty"`
// memory_total_mb for the same reason again: RAM is added to a virtual
// machine while it runs, or across a reboot the agent does not notice, and a
// figure read once at registration kept a node that had grown to 32 GB
// looking like the 7 GB machine it used to be. 0 means not read.
MemoryTotalMb int64 `protobuf:"varint,7,opt,name=memory_total_mb,json=memoryTotalMb,proto3" json:"memory_total_mb,omitempty"`
// engine_versions only when they changed, like weights. An engine installed
// after the agent started is otherwise invisible to placement until
// somebody restarts the agent — which is a step nobody knows to take.
EnginesChanged bool `protobuf:"varint,8,opt,name=engines_changed,json=enginesChanged,proto3" json:"engines_changed,omitempty"`
EngineVersions map[string]string `protobuf:"bytes,9,rep,name=engine_versions,json=engineVersions,proto3" json:"engine_versions,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1136,6 +1146,27 @@ func (x *Heartbeat) GetWeights() []*LocalWeights {
return nil
}
func (x *Heartbeat) GetMemoryTotalMb() int64 {
if x != nil {
return x.MemoryTotalMb
}
return 0
}
func (x *Heartbeat) GetEnginesChanged() bool {
if x != nil {
return x.EnginesChanged
}
return false
}
func (x *Heartbeat) GetEngineVersions() map[string]string {
if x != nil {
return x.EngineVersions
}
return nil
}
// ShardStatus reports a shard's lifecycle transition.
type ShardStatus struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -1894,14 +1925,20 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
"\n" +
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xfc\x01\n" +
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xe2\x03\n" +
"\tHeartbeat\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
"\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" +
"\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" +
"\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" +
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\xbb\x03\n" +
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\x12&\n" +
"\x0fmemory_total_mb\x18\a \x01(\x03R\rmemoryTotalMb\x12'\n" +
"\x0fengines_changed\x18\b \x01(\bR\x0eenginesChanged\x12P\n" +
"\x0fengine_versions\x18\t \x03(\v2'.llmgw.v1.Heartbeat.EngineVersionsEntryR\x0eengineVersions\x1aA\n" +
"\x13EngineVersionsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbb\x03\n" +
"\vShardStatus\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" +
"\n" +
@@ -1998,7 +2035,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 20)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 21)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -2020,7 +2057,8 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*Ack)(nil), // 17: llmgw.v1.Ack
nil, // 18: llmgw.v1.Register.LabelsEntry
nil, // 19: llmgw.v1.Register.EngineVersionsEntry
nil, // 20: llmgw.v1.EngineProfile.EnvEntry
nil, // 20: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 21: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -2041,17 +2079,18 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
11, // 19: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
13, // 20: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
20, // 21: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 22: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 23: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
23, // [23:24] is the sub-list for method output_type
22, // [22:23] is the sub-list for method input_type
22, // [22:22] is the sub-list for extension type_name
22, // [22:22] is the sub-list for extension extendee
0, // [0:22] is the sub-list for field type_name
20, // 18: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
0, // 19: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
11, // 20: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
13, // 21: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
21, // 22: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 23: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 24: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
24, // [24:25] is the sub-list for method output_type
23, // [23:24] is the sub-list for method input_type
23, // [23:23] is the sub-list for extension type_name
23, // [23:23] is the sub-list for extension extendee
0, // [0:23] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -2077,7 +2116,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 20,
NumMessages: 21,
NumExtensions: 0,
NumServices: 1,
},
+12
View File
@@ -222,6 +222,18 @@ message Heartbeat {
// report", and the first would quietly empty a node's inventory.
bool weights_changed = 5;
repeated LocalWeights weights = 6;
// memory_total_mb for the same reason again: RAM is added to a virtual
// machine while it runs, or across a reboot the agent does not notice, and a
// figure read once at registration kept a node that had grown to 32 GB
// looking like the 7 GB machine it used to be. 0 means not read.
int64 memory_total_mb = 7;
// engine_versions only when they changed, like weights. An engine installed
// after the agent started is otherwise invisible to placement until
// somebody restarts the agent — which is a step nobody knows to take.
bool engines_changed = 8;
map<string, string> engine_versions = 9;
}
// ShardStatus reports a shard's lifecycle transition.