Refresh disk and weights while the agent runs, not once when it starts
CI / test (push) Successful in 1m46s
CI / test (push) Successful in 1m46s
Found by answering "where can we see this live". `ctl weights` was offering a file for deletion that I had deleted two days earlier, and the node's disk figure was the one it had when its agent started. Discover() is called at registration and nowhere else, so both were frozen. Disk is a fact about a moment, not about a machine. Sampled once, the figure is wrong on exactly the node worth watching — the one busy filling up — and it stays wrong until that agent restarts. The heartbeat already refreshes GPU memory for the same reason, with a comment about noticing a foreign process "after registration"; disk belonged beside it and I did not put it there. The stale list cut both ways. A model fetched after registration was invisible until a restart, so placement could not prefer the node that had just spent ten minutes downloading it, and the disk check could not see the space it used. Storage goes in every heartbeat, being a few numbers per filesystem. The file list goes only when a fingerprint of paths and sizes changes, with a flag saying the list is meant — without that, an empty list would be ambiguous between "this node now holds nothing" and "nothing to report", and the second reading would empty a node's inventory five seconds after it registered. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cf20570ff1
commit
45e913bd10
5 files changed
+261
-15
No files matched your search
@@ -1033,8 +1033,22 @@ type Heartbeat struct {
|
||||
// a card after registration.
|
||||
GpuMemoryUsedMb []int32 `protobuf:"varint,2,rep,packed,name=gpu_memory_used_mb,json=gpuMemoryUsedMb,proto3" json:"gpu_memory_used_mb,omitempty"`
|
||||
NewProblems []string `protobuf:"bytes,3,rep,name=new_problems,json=newProblems,proto3" json:"new_problems,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// storage for the same reason the GPU memory is here: it changes while the
|
||||
// agent runs, and a figure sampled once at registration is a figure that is
|
||||
// wrong on exactly the node worth watching — the one busy filling up.
|
||||
//
|
||||
// Always sent, because it is a few numbers per filesystem.
|
||||
Storage []*Storage `protobuf:"bytes,4,rep,name=storage,proto3" json:"storage,omitempty"`
|
||||
// weights only when they changed, because the list is long enough that
|
||||
// sending it every few seconds would be paying for nothing most of the time.
|
||||
//
|
||||
// weights_changed is what says the list is meant: without it an empty list
|
||||
// would be ambiguous between "this node now holds nothing" and "nothing to
|
||||
// report", and the first would quietly empty a node's inventory.
|
||||
WeightsChanged bool `protobuf:"varint,5,opt,name=weights_changed,json=weightsChanged,proto3" json:"weights_changed,omitempty"`
|
||||
Weights []*LocalWeights `protobuf:"bytes,6,rep,name=weights,proto3" json:"weights,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Heartbeat) Reset() {
|
||||
@@ -1088,6 +1102,27 @@ func (x *Heartbeat) GetNewProblems() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetStorage() []*Storage {
|
||||
if x != nil {
|
||||
return x.Storage
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetWeightsChanged() bool {
|
||||
if x != nil {
|
||||
return x.WeightsChanged
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetWeights() []*LocalWeights {
|
||||
if x != nil {
|
||||
return x.Weights
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
type ShardStatus struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -1755,11 +1790,14 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04path\x18\x01 \x01(\tR\x04path\x12\x1d\n" +
|
||||
"\n" +
|
||||
"size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x16\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"t\n" +
|
||||
"\x06shared\x18\x03 \x01(\bR\x06shared\"\xfc\x01\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
"\fnew_problems\x18\x03 \x03(\tR\vnewProblems\"\x8c\x03\n" +
|
||||
"\fnew_problems\x18\x03 \x03(\tR\vnewProblems\x12+\n" +
|
||||
"\astorage\x18\x04 \x03(\v2\x11.llmgw.v1.StorageR\astorage\x12'\n" +
|
||||
"\x0fweights_changed\x18\x05 \x01(\bR\x0eweightsChanged\x120\n" +
|
||||
"\aweights\x18\x06 \x03(\v2\x16.llmgw.v1.LocalWeightsR\aweights\"\x8c\x03\n" +
|
||||
"\vShardStatus\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -1887,16 +1925,18 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
8, // 13: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
4, // 14: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
0, // 15: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
0, // 16: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
12, // 17: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
19, // 18: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 19: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 20: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
20, // [20:21] is the sub-list for method output_type
|
||||
19, // [19:20] is the sub-list for method input_type
|
||||
19, // [19:19] is the sub-list for extension type_name
|
||||
19, // [19:19] is the sub-list for extension extendee
|
||||
0, // [0:19] is the sub-list for field type_name
|
||||
4, // 16: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
8, // 17: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
0, // 18: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
12, // 19: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
19, // 20: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 21: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 22: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
22, // [22:23] is the sub-list for method output_type
|
||||
21, // [21:22] is the sub-list for method input_type
|
||||
21, // [21:21] is the sub-list for extension type_name
|
||||
21, // [21:21] is the sub-list for extension extendee
|
||||
0, // [0:21] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
|
||||
@@ -199,6 +199,22 @@ message Heartbeat {
|
||||
// a card after registration.
|
||||
repeated int32 gpu_memory_used_mb = 2;
|
||||
repeated string new_problems = 3;
|
||||
|
||||
// storage for the same reason the GPU memory is here: it changes while the
|
||||
// agent runs, and a figure sampled once at registration is a figure that is
|
||||
// wrong on exactly the node worth watching — the one busy filling up.
|
||||
//
|
||||
// Always sent, because it is a few numbers per filesystem.
|
||||
repeated Storage storage = 4;
|
||||
|
||||
// weights only when they changed, because the list is long enough that
|
||||
// sending it every few seconds would be paying for nothing most of the time.
|
||||
//
|
||||
// weights_changed is what says the list is meant: without it an empty list
|
||||
// would be ambiguous between "this node now holds nothing" and "nothing to
|
||||
// report", and the first would quietly empty a node's inventory.
|
||||
bool weights_changed = 5;
|
||||
repeated LocalWeights weights = 6;
|
||||
}
|
||||
|
||||
// ShardStatus reports a shard's lifecycle transition.
|
||||
|
||||
Reference in new issue
Block a user