Tell independent fabrics apart by their subnet manager, not their name
A fabric was what agent.yaml called it. With several independent InfiniBand fabrics at a site, two given one name would have had a replica spread across both, starting and then hanging on its first collective. Each active InfiniBand port now reports the master subnet manager it sees (sminfo, asked once a minute), which every port on one fabric shares and ports on separate fabrics cannot; a RoCE port reports the IP network of its RoCE v2 address. Ports also carry their node and port GUIDs. Placement divides a declared fabric by that physical identity and forms a replica on one part only; a node whose adapters for one name are on two fabrics is refused, and a node that cannot say is kept apart from those that can. /api/v1/fabrics and the cluster page list each physical fabric with its subnet manager, and say when one name covers two fabrics, two names one fabric, an adapter is up but undeclared, or a node cannot be checked. doctor reports each port's subnet manager, and an agent on a node with adapters installs infiniband-diags when sminfo is missing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
51f1606b53
commit
77fe801892
21 files changed
+1130
-58
No files matched your search
@@ -1398,7 +1398,25 @@ type IBPort struct {
|
||||
// link_layer is InfiniBand or Ethernet (RoCE).
|
||||
LinkLayer string `protobuf:"bytes,6,opt,name=link_layer,json=linkLayer,proto3" json:"link_layer,omitempty"`
|
||||
// counters are cumulative, by sysfs name; data counters in bytes.
|
||||
Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
|
||||
Counters map[string]uint64 `protobuf:"bytes,7,rep,name=counters,proto3" json:"counters,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"`
|
||||
// Identity: the adapter's node GUID and the port's own, as the fabric
|
||||
// knows them.
|
||||
NodeGuid string `protobuf:"bytes,8,opt,name=node_guid,json=nodeGuid,proto3" json:"node_guid,omitempty"`
|
||||
PortGuid string `protobuf:"bytes,9,opt,name=port_guid,json=portGuid,proto3" json:"port_guid,omitempty"`
|
||||
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
||||
// one physical fabric sees the same one, which is what tells separate
|
||||
// fabrics apart when their names are only what somebody typed.
|
||||
SmGuid string `protobuf:"bytes,10,opt,name=sm_guid,json=smGuid,proto3" json:"sm_guid,omitempty"`
|
||||
SmLid int32 `protobuf:"varint,11,opt,name=sm_lid,json=smLid,proto3" json:"sm_lid,omitempty"`
|
||||
SmPriority int32 `protobuf:"varint,12,opt,name=sm_priority,json=smPriority,proto3" json:"sm_priority,omitempty"`
|
||||
SmState string `protobuf:"bytes,13,opt,name=sm_state,json=smState,proto3" json:"sm_state,omitempty"`
|
||||
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
||||
// infiniband-diags missing, or no subnet manager answering.
|
||||
SmError string `protobuf:"bytes,14,opt,name=sm_error,json=smError,proto3" json:"sm_error,omitempty"`
|
||||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"`
|
||||
GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1482,6 +1500,69 @@ func (x *IBPort) GetCounters() map[string]uint64 {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *IBPort) GetNodeGuid() string {
|
||||
if x != nil {
|
||||
return x.NodeGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetPortGuid() string {
|
||||
if x != nil {
|
||||
return x.PortGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmGuid() string {
|
||||
if x != nil {
|
||||
return x.SmGuid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmLid() int32 {
|
||||
if x != nil {
|
||||
return x.SmLid
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmPriority() int32 {
|
||||
if x != nil {
|
||||
return x.SmPriority
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmState() string {
|
||||
if x != nil {
|
||||
return x.SmState
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSmError() string {
|
||||
if x != nil {
|
||||
return x.SmError
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetNetwork() string {
|
||||
if x != nil {
|
||||
return x.Network
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetGidPrefix() string {
|
||||
if x != nil {
|
||||
return x.GidPrefix
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
@@ -2787,7 +2868,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x98\x02\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x92\x04\n" +
|
||||
"\x06IBPort\x12\x10\n" +
|
||||
"\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" +
|
||||
"\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" +
|
||||
@@ -2797,7 +2878,19 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\trate_gbps\x18\x05 \x01(\x05R\brateGbps\x12\x1d\n" +
|
||||
"\n" +
|
||||
"link_layer\x18\x06 \x01(\tR\tlinkLayer\x12:\n" +
|
||||
"\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x1a;\n" +
|
||||
"\bcounters\x18\a \x03(\v2\x1e.llmgw.v1.IBPort.CountersEntryR\bcounters\x12\x1b\n" +
|
||||
"\tnode_guid\x18\b \x01(\tR\bnodeGuid\x12\x1b\n" +
|
||||
"\tport_guid\x18\t \x01(\tR\bportGuid\x12\x17\n" +
|
||||
"\asm_guid\x18\n" +
|
||||
" \x01(\tR\x06smGuid\x12\x15\n" +
|
||||
"\x06sm_lid\x18\v \x01(\x05R\x05smLid\x12\x1f\n" +
|
||||
"\vsm_priority\x18\f \x01(\x05R\n" +
|
||||
"smPriority\x12\x19\n" +
|
||||
"\bsm_state\x18\r \x01(\tR\asmState\x12\x19\n" +
|
||||
"\bsm_error\x18\x0e \x01(\tR\asmError\x12\x18\n" +
|
||||
"\anetwork\x18\x0f \x01(\tR\anetwork\x12\x1d\n" +
|
||||
"\n" +
|
||||
"gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x1a;\n" +
|
||||
"\rCountersEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" +
|
||||
|
||||
@@ -277,6 +277,25 @@ message IBPort {
|
||||
string link_layer = 6;
|
||||
// counters are cumulative, by sysfs name; data counters in bytes.
|
||||
map<string, uint64> counters = 7;
|
||||
|
||||
// Identity: the adapter's node GUID and the port's own, as the fabric
|
||||
// knows them.
|
||||
string node_guid = 8;
|
||||
string port_guid = 9;
|
||||
// The subnet manager this port sees as master, on InfiniBand. Every port on
|
||||
// one physical fabric sees the same one, which is what tells separate
|
||||
// fabrics apart when their names are only what somebody typed.
|
||||
string sm_guid = 10;
|
||||
int32 sm_lid = 11;
|
||||
int32 sm_priority = 12;
|
||||
string sm_state = 13;
|
||||
// sm_error says why sm_guid is empty on an active InfiniBand port:
|
||||
// infiniband-diags missing, or no subnet manager answering.
|
||||
string sm_error = 14;
|
||||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
string network = 15;
|
||||
string gid_prefix = 16;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
|
||||
Reference in new issue
Block a user