Measure two nodes' fabric rail by rail: llmgw ctl fabric-test

doctor checks a node alone, and every check can pass while two nodes still
cannot reach each other. ctl fabric-test A B asks B's agent for an
ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs
them, one rail at a time, and grades each against its link's rate. It goes
through the agents' streams, so there is no SSH between nodes; the agent
installs perftest the first time, and names the RoCE v2 GID itself.

The result is read from perftest's table by its heading, not a column
number. Agents now list the requests they answer beyond the usual
(Register.features), and the control plane refuses a test on one too old to
answer instead of waiting minutes for it. The forwarding dispatcher carries
the new capability, and its test says so: a missing forward would have made
the command quietly unavailable in production.

Not yet run over real InfiniBand or RoCE; this cluster has neither.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 21:29:23 +02:00
1 parent af6f8e023e
commit 7bb360fc78
21 files changed
+1265 -62

No files matched your search

+313 -60
View File
@@ -103,6 +103,7 @@ type AgentMessage struct {
// *AgentMessage_Shard
// *AgentMessage_Logs
// *AgentMessage_FabricScan
// *AgentMessage_FabricTest
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -190,6 +191,15 @@ func (x *AgentMessage) GetFabricScan() *FabricScanReply {
return nil
}
func (x *AgentMessage) GetFabricTest() *FabricTestReply {
if x != nil {
if x, ok := x.Msg.(*AgentMessage_FabricTest); ok {
return x.FabricTest
}
}
return nil
}
type isAgentMessage_Msg interface {
isAgentMessage_Msg()
}
@@ -214,6 +224,10 @@ type AgentMessage_FabricScan struct {
FabricScan *FabricScanReply `protobuf:"bytes,5,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
}
type AgentMessage_FabricTest struct {
FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
}
func (*AgentMessage_Register) isAgentMessage_Msg() {}
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
@@ -224,6 +238,8 @@ func (*AgentMessage_Logs) isAgentMessage_Msg() {}
func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
func (*AgentMessage_FabricTest) isAgentMessage_Msg() {}
// ControlMessage is anything the control plane tells a node.
type ControlMessage struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -237,6 +253,7 @@ type ControlMessage struct {
// *ControlMessage_Provision
// *ControlMessage_Logs
// *ControlMessage_FabricScan
// *ControlMessage_FabricTest
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -351,6 +368,15 @@ func (x *ControlMessage) GetFabricScan() *FabricScanRequest {
return nil
}
func (x *ControlMessage) GetFabricTest() *FabricTestRequest {
if x != nil {
if x, ok := x.Msg.(*ControlMessage_FabricTest); ok {
return x.FabricTest
}
}
return nil
}
type isControlMessage_Msg interface {
isControlMessage_Msg()
}
@@ -387,6 +413,10 @@ type ControlMessage_FabricScan struct {
FabricScan *FabricScanRequest `protobuf:"bytes,8,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
}
type ControlMessage_FabricTest struct {
FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
}
func (*ControlMessage_Start) isControlMessage_Msg() {}
func (*ControlMessage_Stop) isControlMessage_Msg() {}
@@ -403,6 +433,8 @@ func (*ControlMessage_Logs) isControlMessage_Msg() {}
func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
func (*ControlMessage_FabricTest) isControlMessage_Msg() {}
// Register is the first message on every stream. It carries the node's full
// hardware inventory, so the control plane needs no separate discovery call and
// a reconnecting agent automatically refreshes what the scheduler knows.
@@ -478,8 +510,13 @@ type Register struct {
// vLLM, it answered "unsupported", and every node of a cluster half way
// through an upgrade showed as failing.
InstallableEngines []string `protobuf:"bytes,16,rep,name=installable_engines,json=installableEngines,proto3" json:"installable_engines,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// features are the requests this agent answers beyond the ones every agent
// does: "fabric-test". An agent that does not know a request ignores it, so
// without this the control plane would wait minutes for an answer that is
// never coming.
Features []string `protobuf:"bytes,17,rep,name=features,proto3" json:"features,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *Register) Reset() {
@@ -624,6 +661,13 @@ func (x *Register) GetInstallableEngines() []string {
return nil
}
func (x *Register) GetFeatures() []string {
if x != nil {
return x.Features
}
return nil
}
// Host describes the operating system and processor of a node.
type Host struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -3223,19 +3267,203 @@ func (x *FabricScanReply) GetTookMs() int64 {
return 0
}
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
// between two nodes: ib_write_bw, from perftest, over one adapter. The
// server side is asked first and answers when the test is over; the client
// dials it. What it proves is what a replica spanning nodes needs and nothing
// on one node can show: that these two adapters reach each other through the
// fabric, and how fast.
type FabricTestRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
// role is "server" or "client".
Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"`
Hca string `protobuf:"bytes,3,opt,name=hca,proto3" json:"hca,omitempty"`
Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"`
// peer is the server's management address, for the client. perftest
// exchanges its connection details over TCP before any RDMA.
Peer string `protobuf:"bytes,5,opt,name=peer,proto3" json:"peer,omitempty"`
// tcp_port is that exchange's port, the same on both sides.
TcpPort int32 `protobuf:"varint,6,opt,name=tcp_port,json=tcpPort,proto3" json:"tcp_port,omitempty"`
Seconds int32 `protobuf:"varint,7,opt,name=seconds,proto3" json:"seconds,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *FabricTestRequest) Reset() {
*x = FabricTestRequest{}
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *FabricTestRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FabricTestRequest) ProtoMessage() {}
func (x *FabricTestRequest) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead.
func (*FabricTestRequest) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31}
}
func (x *FabricTestRequest) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *FabricTestRequest) GetRole() string {
if x != nil {
return x.Role
}
return ""
}
func (x *FabricTestRequest) GetHca() string {
if x != nil {
return x.Hca
}
return ""
}
func (x *FabricTestRequest) GetPort() int32 {
if x != nil {
return x.Port
}
return 0
}
func (x *FabricTestRequest) GetPeer() string {
if x != nil {
return x.Peer
}
return ""
}
func (x *FabricTestRequest) GetTcpPort() int32 {
if x != nil {
return x.TcpPort
}
return 0
}
func (x *FabricTestRequest) GetSeconds() int32 {
if x != nil {
return x.Seconds
}
return 0
}
// FabricTestReply is what one side measured.
type FabricTestReply struct {
state protoimpl.MessageState `protogen:"open.v1"`
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
// gbps is the average bandwidth, in gigabits a second.
Gbps float64 `protobuf:"fixed64,2,opt,name=gbps,proto3" json:"gbps,omitempty"`
// output is the end of the tool's output, for when the number is not
// enough.
Output string `protobuf:"bytes,3,opt,name=output,proto3" json:"output,omitempty"`
Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
TookMs int64 `protobuf:"varint,5,opt,name=took_ms,json=tookMs,proto3" json:"took_ms,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *FabricTestReply) Reset() {
*x = FabricTestReply{}
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *FabricTestReply) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*FabricTestReply) ProtoMessage() {}
func (x *FabricTestReply) ProtoReflect() protoreflect.Message {
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead.
func (*FabricTestReply) Descriptor() ([]byte, []int) {
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32}
}
func (x *FabricTestReply) GetRequestId() string {
if x != nil {
return x.RequestId
}
return ""
}
func (x *FabricTestReply) GetGbps() float64 {
if x != nil {
return x.Gbps
}
return 0
}
func (x *FabricTestReply) GetOutput() string {
if x != nil {
return x.Output
}
return ""
}
func (x *FabricTestReply) GetError() string {
if x != nil {
return x.Error
}
return ""
}
func (x *FabricTestReply) GetTookMs() int64 {
if x != nil {
return x.TookMs
}
return 0
}
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
const file_llmgw_v1_agent_proto_rawDesc = "" +
"\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x94\x02\n" +
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" +
"\fAgentMessage\x120\n" +
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
"\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shard\x12)\n" +
"\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logs\x12<\n" +
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
"fabricScanB\x05\n" +
"\x03msg\"\xa5\x03\n" +
"fabricScan\x12<\n" +
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
"fabricTestB\x05\n" +
"\x03msg\"\xe5\x03\n" +
"\x0eControlMessage\x12,\n" +
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
@@ -3245,8 +3473,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovision\x12+\n" +
"\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" +
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
"fabricScanB\x05\n" +
"\x03msg\"\xc7\x06\n" +
"fabricScan\x12>\n" +
"\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" +
"fabricTestB\x05\n" +
"\x03msg\"\xe3\x06\n" +
"\bRegister\x12\x17\n" +
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
@@ -3264,7 +3494,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" +
"\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" +
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x1a9\n" +
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" +
"\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" +
"\vLabelsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
@@ -3536,7 +3767,23 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" +
"\x06output\x18\x02 \x01(\tR\x06output\x12\x14\n" +
"\x05error\x18\x03 \x01(\tR\x05error\x12\x17\n" +
"\atook_ms\x18\x04 \x01(\x03R\x06tookMs2O\n" +
"\atook_ms\x18\x04 \x01(\x03R\x06tookMs\"\xb5\x01\n" +
"\x11FabricTestRequest\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x12\n" +
"\x04role\x18\x02 \x01(\tR\x04role\x12\x10\n" +
"\x03hca\x18\x03 \x01(\tR\x03hca\x12\x12\n" +
"\x04port\x18\x04 \x01(\x05R\x04port\x12\x12\n" +
"\x04peer\x18\x05 \x01(\tR\x04peer\x12\x19\n" +
"\btcp_port\x18\x06 \x01(\x05R\atcpPort\x12\x18\n" +
"\aseconds\x18\a \x01(\x05R\aseconds\"\x8b\x01\n" +
"\x0fFabricTestReply\x12\x1d\n" +
"\n" +
"request_id\x18\x01 \x01(\tR\trequestId\x12\x12\n" +
"\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" +
"\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" +
"\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" +
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" +
"\fAgentService\x12?\n" +
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
"\fcom.llmgw.v1B\n" +
@@ -3555,7 +3802,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
}
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 37)
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
var file_llmgw_v1_agent_proto_goTypes = []any{
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
@@ -3589,12 +3836,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
(*LogsReply)(nil), // 29: llmgw.v1.LogsReply
(*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
nil, // 32: llmgw.v1.Register.LabelsEntry
nil, // 33: llmgw.v1.Register.EngineVersionsEntry
nil, // 34: llmgw.v1.GPU.NicPathsEntry
nil, // 35: llmgw.v1.IBPort.CountersEntry
nil, // 36: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 37: llmgw.v1.EngineProfile.EnvEntry
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
nil, // 34: llmgw.v1.Register.LabelsEntry
nil, // 35: llmgw.v1.Register.EngineVersionsEntry
nil, // 36: llmgw.v1.GPU.NicPathsEntry
nil, // 37: llmgw.v1.IBPort.CountersEntry
nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry
nil, // 39: llmgw.v1.EngineProfile.EnvEntry
}
var file_llmgw_v1_agent_proto_depIdxs = []int32{
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -3602,49 +3851,51 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
22, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
25, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
26, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
27, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
24, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
17, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
28, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
30, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
32, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
33, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
14, // 21: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 22: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
34, // 23: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 24: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
35, // 25: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 26: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 27: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 28: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
36, // 29: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
19, // 30: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 31: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 32: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 33: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 34: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
14, // 35: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
18, // 36: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 37: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
21, // 38: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
23, // 39: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
37, // 40: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 41: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 42: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
42, // [42:43] is the sub-list for method output_type
41, // [41:42] is the sub-list for method input_type
41, // [41:41] is the sub-list for extension type_name
41, // [41:41] is the sub-list for extension extendee
0, // [0:41] is the sub-list for field type_name
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
44, // [44:45] is the sub-list for method output_type
43, // [43:44] is the sub-list for method input_type
43, // [43:43] is the sub-list for extension type_name
43, // [43:43] is the sub-list for extension extendee
0, // [0:43] is the sub-list for field type_name
}
func init() { file_llmgw_v1_agent_proto_init() }
@@ -3658,6 +3909,7 @@ func file_llmgw_v1_agent_proto_init() {
(*AgentMessage_Shard)(nil),
(*AgentMessage_Logs)(nil),
(*AgentMessage_FabricScan)(nil),
(*AgentMessage_FabricTest)(nil),
}
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
(*ControlMessage_Start)(nil),
@@ -3668,6 +3920,7 @@ func file_llmgw_v1_agent_proto_init() {
(*ControlMessage_Provision)(nil),
(*ControlMessage_Logs)(nil),
(*ControlMessage_FabricScan)(nil),
(*ControlMessage_FabricTest)(nil),
}
type x struct{}
out := protoimpl.TypeBuilder{
@@ -3675,7 +3928,7 @@ func file_llmgw_v1_agent_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
NumEnums: 1,
NumMessages: 37,
NumMessages: 39,
NumExtensions: 0,
NumServices: 1,
},
+40
View File
@@ -25,6 +25,7 @@ message AgentMessage {
ShardStatus shard = 3;
LogsReply logs = 4;
FabricScanReply fabric_scan = 5;
FabricTestReply fabric_test = 6;
}
}
@@ -39,6 +40,7 @@ message ControlMessage {
Provision provision = 6;
LogsRequest logs = 7;
FabricScanRequest fabric_scan = 8;
FabricTestRequest fabric_test = 9;
}
}
@@ -125,6 +127,12 @@ message Register {
// vLLM, it answered "unsupported", and every node of a cluster half way
// through an upgrade showed as failing.
repeated string installable_engines = 16;
// features are the requests this agent answers beyond the ones every agent
// does: "fabric-test". An agent that does not know a request ignores it, so
// without this the control plane would wait minutes for an answer that is
// never coming.
repeated string features = 17;
}
// Host describes the operating system and processor of a node.
@@ -605,3 +613,35 @@ message FabricScanReply {
string error = 3;
int64 took_ms = 4;
}
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
// between two nodes: ib_write_bw, from perftest, over one adapter. The
// server side is asked first and answers when the test is over; the client
// dials it. What it proves is what a replica spanning nodes needs and nothing
// on one node can show: that these two adapters reach each other through the
// fabric, and how fast.
message FabricTestRequest {
string request_id = 1;
// role is "server" or "client".
string role = 2;
string hca = 3;
int32 port = 4;
// peer is the server's management address, for the client. perftest
// exchanges its connection details over TCP before any RDMA.
string peer = 5;
// tcp_port is that exchange's port, the same on both sides.
int32 tcp_port = 6;
int32 seconds = 7;
}
// FabricTestReply is what one side measured.
message FabricTestReply {
string request_id = 1;
// gbps is the average bandwidth, in gigabits a second.
double gbps = 2;
// output is the end of the tool's output, for when the number is not
// enough.
string output = 3;
string error = 4;
int64 took_ms = 5;
}