Measure two nodes' fabric rail by rail: llmgw ctl fabric-test
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
af6f8e023e
commit
7bb360fc78
21 files changed
+1265
-62
No files matched your search
+313
-60
@@ -103,6 +103,7 @@ type AgentMessage struct {
|
||||
// *AgentMessage_Shard
|
||||
// *AgentMessage_Logs
|
||||
// *AgentMessage_FabricScan
|
||||
// *AgentMessage_FabricTest
|
||||
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -190,6 +191,15 @@ func (x *AgentMessage) GetFabricScan() *FabricScanReply {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AgentMessage) GetFabricTest() *FabricTestReply {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*AgentMessage_FabricTest); ok {
|
||||
return x.FabricTest
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isAgentMessage_Msg interface {
|
||||
isAgentMessage_Msg()
|
||||
}
|
||||
@@ -214,6 +224,10 @@ type AgentMessage_FabricScan struct {
|
||||
FabricScan *FabricScanReply `protobuf:"bytes,5,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AgentMessage_FabricTest struct {
|
||||
FabricTest *FabricTestReply `protobuf:"bytes,6,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*AgentMessage_Register) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
|
||||
@@ -224,6 +238,8 @@ func (*AgentMessage_Logs) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_FabricScan) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_FabricTest) isAgentMessage_Msg() {}
|
||||
|
||||
// ControlMessage is anything the control plane tells a node.
|
||||
type ControlMessage struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -237,6 +253,7 @@ type ControlMessage struct {
|
||||
// *ControlMessage_Provision
|
||||
// *ControlMessage_Logs
|
||||
// *ControlMessage_FabricScan
|
||||
// *ControlMessage_FabricTest
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -351,6 +368,15 @@ func (x *ControlMessage) GetFabricScan() *FabricScanRequest {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetFabricTest() *FabricTestRequest {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*ControlMessage_FabricTest); ok {
|
||||
return x.FabricTest
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isControlMessage_Msg interface {
|
||||
isControlMessage_Msg()
|
||||
}
|
||||
@@ -387,6 +413,10 @@ type ControlMessage_FabricScan struct {
|
||||
FabricScan *FabricScanRequest `protobuf:"bytes,8,opt,name=fabric_scan,json=fabricScan,proto3,oneof"`
|
||||
}
|
||||
|
||||
type ControlMessage_FabricTest struct {
|
||||
FabricTest *FabricTestRequest `protobuf:"bytes,9,opt,name=fabric_test,json=fabricTest,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*ControlMessage_Start) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Stop) isControlMessage_Msg() {}
|
||||
@@ -403,6 +433,8 @@ func (*ControlMessage_Logs) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_FabricScan) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_FabricTest) isControlMessage_Msg() {}
|
||||
|
||||
// Register is the first message on every stream. It carries the node's full
|
||||
// hardware inventory, so the control plane needs no separate discovery call and
|
||||
// a reconnecting agent automatically refreshes what the scheduler knows.
|
||||
@@ -478,8 +510,13 @@ type Register struct {
|
||||
// vLLM, it answered "unsupported", and every node of a cluster half way
|
||||
// through an upgrade showed as failing.
|
||||
InstallableEngines []string `protobuf:"bytes,16,rep,name=installable_engines,json=installableEngines,proto3" json:"installable_engines,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
// features are the requests this agent answers beyond the ones every agent
|
||||
// does: "fabric-test". An agent that does not know a request ignores it, so
|
||||
// without this the control plane would wait minutes for an answer that is
|
||||
// never coming.
|
||||
Features []string `protobuf:"bytes,17,rep,name=features,proto3" json:"features,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Register) Reset() {
|
||||
@@ -624,6 +661,13 @@ func (x *Register) GetInstallableEngines() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Register) GetFeatures() []string {
|
||||
if x != nil {
|
||||
return x.Features
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Host describes the operating system and processor of a node.
|
||||
type Host struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -3223,19 +3267,203 @@ func (x *FabricScanReply) GetTookMs() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
|
||||
// between two nodes: ib_write_bw, from perftest, over one adapter. The
|
||||
// server side is asked first and answers when the test is over; the client
|
||||
// dials it. What it proves is what a replica spanning nodes needs and nothing
|
||||
// on one node can show: that these two adapters reach each other through the
|
||||
// fabric, and how fast.
|
||||
type FabricTestRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
// role is "server" or "client".
|
||||
Role string `protobuf:"bytes,2,opt,name=role,proto3" json:"role,omitempty"`
|
||||
Hca string `protobuf:"bytes,3,opt,name=hca,proto3" json:"hca,omitempty"`
|
||||
Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"`
|
||||
// peer is the server's management address, for the client. perftest
|
||||
// exchanges its connection details over TCP before any RDMA.
|
||||
Peer string `protobuf:"bytes,5,opt,name=peer,proto3" json:"peer,omitempty"`
|
||||
// tcp_port is that exchange's port, the same on both sides.
|
||||
TcpPort int32 `protobuf:"varint,6,opt,name=tcp_port,json=tcpPort,proto3" json:"tcp_port,omitempty"`
|
||||
Seconds int32 `protobuf:"varint,7,opt,name=seconds,proto3" json:"seconds,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) Reset() {
|
||||
*x = FabricTestRequest{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*FabricTestRequest) ProtoMessage() {}
|
||||
|
||||
func (x *FabricTestRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[31]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use FabricTestRequest.ProtoReflect.Descriptor instead.
|
||||
func (*FabricTestRequest) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{31}
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetRole() string {
|
||||
if x != nil {
|
||||
return x.Role
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetHca() string {
|
||||
if x != nil {
|
||||
return x.Hca
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetPort() int32 {
|
||||
if x != nil {
|
||||
return x.Port
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetPeer() string {
|
||||
if x != nil {
|
||||
return x.Peer
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetTcpPort() int32 {
|
||||
if x != nil {
|
||||
return x.TcpPort
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FabricTestRequest) GetSeconds() int32 {
|
||||
if x != nil {
|
||||
return x.Seconds
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// FabricTestReply is what one side measured.
|
||||
type FabricTestReply struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
// gbps is the average bandwidth, in gigabits a second.
|
||||
Gbps float64 `protobuf:"fixed64,2,opt,name=gbps,proto3" json:"gbps,omitempty"`
|
||||
// output is the end of the tool's output, for when the number is not
|
||||
// enough.
|
||||
Output string `protobuf:"bytes,3,opt,name=output,proto3" json:"output,omitempty"`
|
||||
Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
|
||||
TookMs int64 `protobuf:"varint,5,opt,name=took_ms,json=tookMs,proto3" json:"took_ms,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) Reset() {
|
||||
*x = FabricTestReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*FabricTestReply) ProtoMessage() {}
|
||||
|
||||
func (x *FabricTestReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[32]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use FabricTestReply.ProtoReflect.Descriptor instead.
|
||||
func (*FabricTestReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{32}
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) GetGbps() float64 {
|
||||
if x != nil {
|
||||
return x.Gbps
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) GetOutput() string {
|
||||
if x != nil {
|
||||
return x.Output
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) GetError() string {
|
||||
if x != nil {
|
||||
return x.Error
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *FabricTestReply) GetTookMs() int64 {
|
||||
if x != nil {
|
||||
return x.TookMs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\x94\x02\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xd2\x02\n" +
|
||||
"\fAgentMessage\x120\n" +
|
||||
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
|
||||
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
|
||||
"\x05shard\x18\x03 \x01(\v2\x15.llmgw.v1.ShardStatusH\x00R\x05shard\x12)\n" +
|
||||
"\x04logs\x18\x04 \x01(\v2\x13.llmgw.v1.LogsReplyH\x00R\x04logs\x12<\n" +
|
||||
"\vfabric_scan\x18\x05 \x01(\v2\x19.llmgw.v1.FabricScanReplyH\x00R\n" +
|
||||
"fabricScanB\x05\n" +
|
||||
"\x03msg\"\xa5\x03\n" +
|
||||
"fabricScan\x12<\n" +
|
||||
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"\x03msg\"\xe5\x03\n" +
|
||||
"\x0eControlMessage\x12,\n" +
|
||||
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
@@ -3245,8 +3473,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\tprovision\x18\x06 \x01(\v2\x13.llmgw.v1.ProvisionH\x00R\tprovision\x12+\n" +
|
||||
"\x04logs\x18\a \x01(\v2\x15.llmgw.v1.LogsRequestH\x00R\x04logs\x12>\n" +
|
||||
"\vfabric_scan\x18\b \x01(\v2\x1b.llmgw.v1.FabricScanRequestH\x00R\n" +
|
||||
"fabricScanB\x05\n" +
|
||||
"\x03msg\"\xc7\x06\n" +
|
||||
"fabricScan\x12>\n" +
|
||||
"\vfabric_test\x18\t \x01(\v2\x1b.llmgw.v1.FabricTestRequestH\x00R\n" +
|
||||
"fabricTestB\x05\n" +
|
||||
"\x03msg\"\xe3\x06\n" +
|
||||
"\bRegister\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x1a\n" +
|
||||
"\bhostname\x18\x02 \x01(\tR\bhostname\x12\x1b\n" +
|
||||
@@ -3264,7 +3494,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04host\x18\r \x01(\v2\x0e.llmgw.v1.HostR\x04host\x12%\n" +
|
||||
"\x0eengines_probed\x18\x0e \x01(\bR\renginesProbed\x12\"\n" +
|
||||
"\x04imex\x18\x0f \x01(\v2\x0e.llmgw.v1.IMEXR\x04imex\x12/\n" +
|
||||
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x1a9\n" +
|
||||
"\x13installable_engines\x18\x10 \x03(\tR\x12installableEngines\x12\x1a\n" +
|
||||
"\bfeatures\x18\x11 \x03(\tR\bfeatures\x1a9\n" +
|
||||
"\vLabelsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1aA\n" +
|
||||
@@ -3536,7 +3767,23 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x16\n" +
|
||||
"\x06output\x18\x02 \x01(\tR\x06output\x12\x14\n" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error\x12\x17\n" +
|
||||
"\atook_ms\x18\x04 \x01(\x03R\x06tookMs2O\n" +
|
||||
"\atook_ms\x18\x04 \x01(\x03R\x06tookMs\"\xb5\x01\n" +
|
||||
"\x11FabricTestRequest\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x12\n" +
|
||||
"\x04role\x18\x02 \x01(\tR\x04role\x12\x10\n" +
|
||||
"\x03hca\x18\x03 \x01(\tR\x03hca\x12\x12\n" +
|
||||
"\x04port\x18\x04 \x01(\x05R\x04port\x12\x12\n" +
|
||||
"\x04peer\x18\x05 \x01(\tR\x04peer\x12\x19\n" +
|
||||
"\btcp_port\x18\x06 \x01(\x05R\atcpPort\x12\x18\n" +
|
||||
"\aseconds\x18\a \x01(\x05R\aseconds\"\x8b\x01\n" +
|
||||
"\x0fFabricTestReply\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x01 \x01(\tR\trequestId\x12\x12\n" +
|
||||
"\x04gbps\x18\x02 \x01(\x01R\x04gbps\x12\x16\n" +
|
||||
"\x06output\x18\x03 \x01(\tR\x06output\x12\x14\n" +
|
||||
"\x05error\x18\x04 \x01(\tR\x05error\x12\x17\n" +
|
||||
"\atook_ms\x18\x05 \x01(\x03R\x06tookMs2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -3555,7 +3802,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 37)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 39)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -3589,12 +3836,14 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*LogsReply)(nil), // 29: llmgw.v1.LogsReply
|
||||
(*FabricScanRequest)(nil), // 30: llmgw.v1.FabricScanRequest
|
||||
(*FabricScanReply)(nil), // 31: llmgw.v1.FabricScanReply
|
||||
nil, // 32: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 33: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 34: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 35: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 36: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 37: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*FabricTestRequest)(nil), // 32: llmgw.v1.FabricTestRequest
|
||||
(*FabricTestReply)(nil), // 33: llmgw.v1.FabricTestReply
|
||||
nil, // 34: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 35: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 36: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 37: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 38: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 39: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -3602,49 +3851,51 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
20, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
29, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
31, // 4: llmgw.v1.AgentMessage.fabric_scan:type_name -> llmgw.v1.FabricScanReply
|
||||
22, // 5: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 6: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 7: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 8: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 9: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 10: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 11: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 12: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 13: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 14: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 15: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
33, // 16: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 17: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 18: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 19: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 20: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 21: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 22: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
34, // 23: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 24: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
35, // 25: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 26: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 27: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 28: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
36, // 29: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 30: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 31: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 32: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 33: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 34: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 35: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 36: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 37: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 38: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 39: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
37, // 40: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 41: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 42: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
42, // [42:43] is the sub-list for method output_type
|
||||
41, // [41:42] is the sub-list for method input_type
|
||||
41, // [41:41] is the sub-list for extension type_name
|
||||
41, // [41:41] is the sub-list for extension extendee
|
||||
0, // [0:41] is the sub-list for field type_name
|
||||
33, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
|
||||
22, // 6: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
25, // 7: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
26, // 8: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
27, // 9: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
24, // 10: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
17, // 11: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
28, // 12: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
30, // 13: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
32, // 14: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
34, // 15: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 16: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 17: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
35, // 18: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 19: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 20: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 21: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 22: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
14, // 23: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 24: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
36, // 25: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 26: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
37, // 27: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 28: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 29: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 30: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
38, // 31: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
19, // 32: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 33: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 34: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 35: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 36: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
14, // 37: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
18, // 38: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 39: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
21, // 40: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
23, // 41: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
39, // 42: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 43: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 44: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
44, // [44:45] is the sub-list for method output_type
|
||||
43, // [43:44] is the sub-list for method input_type
|
||||
43, // [43:43] is the sub-list for extension type_name
|
||||
43, // [43:43] is the sub-list for extension extendee
|
||||
0, // [0:43] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -3658,6 +3909,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*AgentMessage_Shard)(nil),
|
||||
(*AgentMessage_Logs)(nil),
|
||||
(*AgentMessage_FabricScan)(nil),
|
||||
(*AgentMessage_FabricTest)(nil),
|
||||
}
|
||||
file_llmgw_v1_agent_proto_msgTypes[1].OneofWrappers = []any{
|
||||
(*ControlMessage_Start)(nil),
|
||||
@@ -3668,6 +3920,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*ControlMessage_Provision)(nil),
|
||||
(*ControlMessage_Logs)(nil),
|
||||
(*ControlMessage_FabricScan)(nil),
|
||||
(*ControlMessage_FabricTest)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -3675,7 +3928,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 37,
|
||||
NumMessages: 39,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -25,6 +25,7 @@ message AgentMessage {
|
||||
ShardStatus shard = 3;
|
||||
LogsReply logs = 4;
|
||||
FabricScanReply fabric_scan = 5;
|
||||
FabricTestReply fabric_test = 6;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +40,7 @@ message ControlMessage {
|
||||
Provision provision = 6;
|
||||
LogsRequest logs = 7;
|
||||
FabricScanRequest fabric_scan = 8;
|
||||
FabricTestRequest fabric_test = 9;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +127,12 @@ message Register {
|
||||
// vLLM, it answered "unsupported", and every node of a cluster half way
|
||||
// through an upgrade showed as failing.
|
||||
repeated string installable_engines = 16;
|
||||
|
||||
// features are the requests this agent answers beyond the ones every agent
|
||||
// does: "fabric-test". An agent that does not know a request ignores it, so
|
||||
// without this the control plane would wait minutes for an answer that is
|
||||
// never coming.
|
||||
repeated string features = 17;
|
||||
}
|
||||
|
||||
// Host describes the operating system and processor of a node.
|
||||
@@ -605,3 +613,35 @@ message FabricScanReply {
|
||||
string error = 3;
|
||||
int64 took_ms = 4;
|
||||
}
|
||||
|
||||
// FabricTestRequest asks a node for one side of an RDMA bandwidth test
|
||||
// between two nodes: ib_write_bw, from perftest, over one adapter. The
|
||||
// server side is asked first and answers when the test is over; the client
|
||||
// dials it. What it proves is what a replica spanning nodes needs and nothing
|
||||
// on one node can show: that these two adapters reach each other through the
|
||||
// fabric, and how fast.
|
||||
message FabricTestRequest {
|
||||
string request_id = 1;
|
||||
// role is "server" or "client".
|
||||
string role = 2;
|
||||
string hca = 3;
|
||||
int32 port = 4;
|
||||
// peer is the server's management address, for the client. perftest
|
||||
// exchanges its connection details over TCP before any RDMA.
|
||||
string peer = 5;
|
||||
// tcp_port is that exchange's port, the same on both sides.
|
||||
int32 tcp_port = 6;
|
||||
int32 seconds = 7;
|
||||
}
|
||||
|
||||
// FabricTestReply is what one side measured.
|
||||
message FabricTestReply {
|
||||
string request_id = 1;
|
||||
// gbps is the average bandwidth, in gigabits a second.
|
||||
double gbps = 2;
|
||||
// output is the end of the tool's output, for when the number is not
|
||||
// enough.
|
||||
string output = 3;
|
||||
string error = 4;
|
||||
int64 took_ms = 5;
|
||||
}
|
||||
Reference in new issue
Block a user