Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -272,7 +272,13 @@ type ControlMessage struct {
|
||||
// *ControlMessage_FabricTest
|
||||
// *ControlMessage_Upgrade
|
||||
// *ControlMessage_ControlPlanes
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
// agent does for it — fetching weights, launching the engine, waiting for
|
||||
// it to answer — lands in the same trace. Empty when the control plane is
|
||||
// not tracing, and ignored by an agent that is not.
|
||||
Trace map[string]string `protobuf:"bytes,12,rep,name=trace,proto3" json:"trace,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -413,6 +419,13 @@ func (x *ControlMessage) GetControlPlanes() *ControlPlanes {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetTrace() map[string]string {
|
||||
if x != nil {
|
||||
return x.Trace
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isControlMessage_Msg interface {
|
||||
isControlMessage_Msg()
|
||||
}
|
||||
@@ -4128,7 +4141,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgradeB\x05\n" +
|
||||
"\x03msg\"\xdb\x04\n" +
|
||||
"\x03msg\"\xd0\x05\n" +
|
||||
"\x0eControlMessage\x12,\n" +
|
||||
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
@@ -4143,7 +4156,12 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\n" +
|
||||
" \x01(\v2\x16.llmgw.v1.UpgradeAgentH\x00R\aupgrade\x12@\n" +
|
||||
"\x0econtrol_planes\x18\v \x01(\v2\x17.llmgw.v1.ControlPlanesH\x00R\rcontrolPlanesB\x05\n" +
|
||||
"\x0econtrol_planes\x18\v \x01(\v2\x17.llmgw.v1.ControlPlanesH\x00R\rcontrolPlanes\x129\n" +
|
||||
"\x05trace\x18\f \x03(\v2#.llmgw.v1.ControlMessage.TraceEntryR\x05trace\x1a8\n" +
|
||||
"\n" +
|
||||
"TraceEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x05\n" +
|
||||
"\x03msg\")\n" +
|
||||
"\rControlPlanes\x12\x18\n" +
|
||||
"\aservers\x18\x01 \x03(\tR\aservers\"\x80\a\n" +
|
||||
@@ -4532,7 +4550,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 44)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 45)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -4573,12 +4591,13 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*UpgradeAgent)(nil), // 36: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 37: llmgw.v1.UpgradeReply
|
||||
(*LoraAdapter)(nil), // 38: llmgw.v1.LoraAdapter
|
||||
nil, // 39: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 40: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 41: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 42: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 43: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 44: llmgw.v1.EngineProfile.EnvEntry
|
||||
nil, // 39: llmgw.v1.ControlMessage.TraceEntry
|
||||
nil, // 40: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 41: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 42: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 43: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 44: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 45: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
4, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -4599,43 +4618,44 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
34, // 15: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
36, // 16: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
3, // 17: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes
|
||||
39, // 18: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
8, // 19: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
9, // 20: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
40, // 21: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
7, // 22: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
10, // 23: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
6, // 24: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
5, // 25: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
16, // 26: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 27: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
41, // 28: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
13, // 29: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
12, // 30: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
42, // 31: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
15, // 32: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
6, // 33: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
10, // 34: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
43, // 35: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
21, // 36: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
11, // 37: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
14, // 38: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
9, // 39: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
17, // 40: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
16, // 41: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
20, // 42: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 43: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
23, // 44: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
25, // 45: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
44, // 46: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
38, // 47: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 48: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 49: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
49, // [49:50] is the sub-list for method output_type
|
||||
48, // [48:49] is the sub-list for method input_type
|
||||
48, // [48:48] is the sub-list for extension type_name
|
||||
48, // [48:48] is the sub-list for extension extendee
|
||||
0, // [0:48] is the sub-list for field type_name
|
||||
39, // 18: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry
|
||||
40, // 19: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
8, // 20: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
9, // 21: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
41, // 22: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
7, // 23: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
10, // 24: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
6, // 25: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
5, // 26: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
16, // 27: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 28: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
42, // 29: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
13, // 30: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
12, // 31: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
43, // 32: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
15, // 33: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
6, // 34: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
10, // 35: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
44, // 36: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
21, // 37: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
11, // 38: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
14, // 39: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
9, // 40: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
17, // 41: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
16, // 42: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
20, // 43: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 44: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
23, // 45: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
25, // 46: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
45, // 47: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
38, // 48: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 49: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 50: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
50, // [50:51] is the sub-list for method output_type
|
||||
49, // [49:50] is the sub-list for method input_type
|
||||
49, // [49:49] is the sub-list for extension type_name
|
||||
49, // [49:49] is the sub-list for extension extendee
|
||||
0, // [0:49] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -4671,7 +4691,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 44,
|
||||
NumMessages: 45,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -45,6 +45,12 @@ message ControlMessage {
|
||||
UpgradeAgent upgrade = 10;
|
||||
ControlPlanes control_planes = 11;
|
||||
}
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
// agent does for it — fetching weights, launching the engine, waiting for
|
||||
// it to answer — lands in the same trace. Empty when the control plane is
|
||||
// not tracing, and ignored by an agent that is not.
|
||||
map<string, string> trace = 12;
|
||||
}
|
||||
|
||||
// ControlPlanes is every control plane an agent may dial, sent by the leader
|
||||
|
||||
Reference in new issue
Block a user