From 23648fa19ae7fb345afb6a327847db89ac8e284b Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 07:47:03 +0200 Subject: [PATCH] Watch each fabric's subnet managers, and keep a fabric one through a failover A fabric's identity was its master subnet manager, which changes on a failover: for the minute its ports took to agree, one fabric read as two, and a name covering it as two separate fabrics. Each agent now also lists every subnet manager on a fabric (saquery SMIR, through one port per master, once a minute); managers listed together are one fabric, named by the lowest GUID among them, so the identity survives the master changing. The same list says what a fabric can survive. The cluster page's new subnet manager table, /api/v1/fabrics, doctor and metrics (llmgw_fabric_master_sm, llmgw_fabric_subnet_managers, llmgw_fabric_problems) show each fabric's master, priority and standbys, and say when a fabric has no standby, its ports disagree on the master, ports are up with no subnet manager (INIT), or the master changed in the last hour. Every change of master a port reports is kept, and a fabric's last day of them is listed. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/gen/llmgw/v1/agent.pb.go | 309 ++++++++++++------ api/proto/llmgw/v1/agent.proto | 15 + internal/agent/agent.go | 43 ++- internal/agent/agent_test.go | 33 ++ internal/agent/discovery/ib_test.go | 28 ++ internal/agent/discovery/sminfo.go | 73 +++++ internal/control/adminapi/api.go | 5 +- internal/control/agentsvc/service.go | 21 +- internal/control/fabrics/fabrics.go | 148 ++++++++- internal/control/fabrics/fabrics_test.go | 73 ++++- internal/control/fleetmetrics/fleetmetrics.go | 41 ++- internal/control/inventory/inventory.go | 103 +++++- internal/control/placement/placement.go | 7 +- internal/control/placement/placement_test.go | 12 + internal/doctor/doctor.go | 54 ++- internal/doctor/doctor_test.go | 2 + internal/web/admin_mirror.go | 4 +- internal/web/dashboard.go | 2 +- internal/web/templates/dashboard.html | 12 + internal/web/web_test.go | 8 +- 20 files changed, 857 insertions(+), 136 deletions(-) diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index df937a8c..2d1ce99c 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber { // Deprecated: Use ShardStatus_State.Descriptor instead. func (ShardStatus_State) EnumDescriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16, 0} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17, 0} } // AgentMessage is anything a node tells the control plane. @@ -1415,10 +1415,16 @@ type IBPort struct { SmError string `protobuf:"bytes,14,opt,name=sm_error,json=smError,proto3" json:"sm_error,omitempty"` // network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no // subnet manager, and the network is what its fabric is. - Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"` - GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"` + GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"` + // subnet_managers is every subnet manager the fabric's subnet + // administrator lists — master and standbys — asked through one port per + // fabric, so it is on that port only. It is what keeps a fabric one fabric + // through a failover: the master changes, the list does not. + SubnetManagers []*SubnetManager `protobuf:"bytes,17,rep,name=subnet_managers,json=subnetManagers,proto3" json:"subnet_managers,omitempty"` + SubnetManagersError string `protobuf:"bytes,18,opt,name=subnet_managers_error,json=subnetManagersError,proto3" json:"subnet_managers_error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *IBPort) Reset() { @@ -1563,6 +1569,90 @@ func (x *IBPort) GetGidPrefix() string { return "" } +func (x *IBPort) GetSubnetManagers() []*SubnetManager { + if x != nil { + return x.SubnetManagers + } + return nil +} + +func (x *IBPort) GetSubnetManagersError() string { + if x != nil { + return x.SubnetManagersError + } + return "" +} + +// SubnetManager is one subnet manager on a fabric. +type SubnetManager struct { + state protoimpl.MessageState `protogen:"open.v1"` + Guid string `protobuf:"bytes,1,opt,name=guid,proto3" json:"guid,omitempty"` + Lid int32 `protobuf:"varint,2,opt,name=lid,proto3" json:"lid,omitempty"` + Priority int32 `protobuf:"varint,3,opt,name=priority,proto3" json:"priority,omitempty"` + // state is master, standby, discovering or not active. + State string `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubnetManager) Reset() { + *x = SubnetManager{} + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubnetManager) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubnetManager) ProtoMessage() {} + +func (x *SubnetManager) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead. +func (*SubnetManager) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *SubnetManager) GetGuid() string { + if x != nil { + return x.Guid + } + return "" +} + +func (x *SubnetManager) GetLid() int32 { + if x != nil { + return x.Lid + } + return 0 +} + +func (x *SubnetManager) GetPriority() int32 { + if x != nil { + return x.Priority + } + return 0 +} + +func (x *SubnetManager) GetState() string { + if x != nil { + return x.State + } + return "" +} + type Heartbeat struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -1618,7 +1708,7 @@ type Heartbeat struct { func (x *Heartbeat) Reset() { *x = Heartbeat{} - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1630,7 +1720,7 @@ func (x *Heartbeat) String() string { func (*Heartbeat) ProtoMessage() {} func (x *Heartbeat) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[12] + mi := &file_llmgw_v1_agent_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1643,7 +1733,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message { // Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. func (*Heartbeat) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} } func (x *Heartbeat) GetNodeId() string { @@ -1770,7 +1860,7 @@ type Provision struct { func (x *Provision) Reset() { *x = Provision{} - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1782,7 +1872,7 @@ func (x *Provision) String() string { func (*Provision) ProtoMessage() {} func (x *Provision) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[13] + mi := &file_llmgw_v1_agent_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1795,7 +1885,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message { // Deprecated: Use Provision.ProtoReflect.Descriptor instead. func (*Provision) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} } func (x *Provision) GetEngines() []*EngineWant { @@ -1816,7 +1906,7 @@ type EngineWant struct { func (x *EngineWant) Reset() { *x = EngineWant{} - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1828,7 +1918,7 @@ func (x *EngineWant) String() string { func (*EngineWant) ProtoMessage() {} func (x *EngineWant) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[14] + mi := &file_llmgw_v1_agent_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1841,7 +1931,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineWant.ProtoReflect.Descriptor instead. func (*EngineWant) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} } func (x *EngineWant) GetEngine() string { @@ -1874,7 +1964,7 @@ type ProvisionStatus struct { func (x *ProvisionStatus) Reset() { *x = ProvisionStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1886,7 +1976,7 @@ func (x *ProvisionStatus) String() string { func (*ProvisionStatus) ProtoMessage() {} func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[15] + mi := &file_llmgw_v1_agent_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1899,7 +1989,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead. func (*ProvisionStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} } func (x *ProvisionStatus) GetEngine() string { @@ -1963,7 +2053,7 @@ type ShardStatus struct { func (x *ShardStatus) Reset() { *x = ShardStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1975,7 +2065,7 @@ func (x *ShardStatus) String() string { func (*ShardStatus) ProtoMessage() {} func (x *ShardStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[16] + mi := &file_llmgw_v1_agent_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1988,7 +2078,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead. func (*ShardStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} } func (x *ShardStatus) GetNodeId() string { @@ -2062,7 +2152,7 @@ type FetchProgress struct { func (x *FetchProgress) Reset() { *x = FetchProgress{} - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2074,7 +2164,7 @@ func (x *FetchProgress) String() string { func (*FetchProgress) ProtoMessage() {} func (x *FetchProgress) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[17] + mi := &file_llmgw_v1_agent_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2087,7 +2177,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message { // Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead. func (*FetchProgress) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} } func (x *FetchProgress) GetSource() string { @@ -2143,7 +2233,7 @@ type StartShard struct { func (x *StartShard) Reset() { *x = StartShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2155,7 +2245,7 @@ func (x *StartShard) String() string { func (*StartShard) ProtoMessage() {} func (x *StartShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[18] + mi := &file_llmgw_v1_agent_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2168,7 +2258,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StartShard.ProtoReflect.Descriptor instead. func (*StartShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} } func (x *StartShard) GetReplicaId() string { @@ -2295,7 +2385,7 @@ type EngineProfile struct { func (x *EngineProfile) Reset() { *x = EngineProfile{} - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2307,7 +2397,7 @@ func (x *EngineProfile) String() string { func (*EngineProfile) ProtoMessage() {} func (x *EngineProfile) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[19] + mi := &file_llmgw_v1_agent_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2320,7 +2410,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message { // Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead. func (*EngineProfile) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} } func (x *EngineProfile) GetName() string { @@ -2416,7 +2506,7 @@ type RemoveWeights struct { func (x *RemoveWeights) Reset() { *x = RemoveWeights{} - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2428,7 +2518,7 @@ func (x *RemoveWeights) String() string { func (*RemoveWeights) ProtoMessage() {} func (x *RemoveWeights) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[20] + mi := &file_llmgw_v1_agent_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2441,7 +2531,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead. func (*RemoveWeights) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} } func (x *RemoveWeights) GetPaths() []string { @@ -2461,7 +2551,7 @@ type StopShard struct { func (x *StopShard) Reset() { *x = StopShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2473,7 +2563,7 @@ func (x *StopShard) String() string { func (*StopShard) ProtoMessage() {} func (x *StopShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[21] + mi := &file_llmgw_v1_agent_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2486,7 +2576,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message { // Deprecated: Use StopShard.ProtoReflect.Descriptor instead. func (*StopShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} } func (x *StopShard) GetShardId() string { @@ -2506,7 +2596,7 @@ type DrainShard struct { func (x *DrainShard) Reset() { *x = DrainShard{} - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2518,7 +2608,7 @@ func (x *DrainShard) String() string { func (*DrainShard) ProtoMessage() {} func (x *DrainShard) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[22] + mi := &file_llmgw_v1_agent_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2531,7 +2621,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainShard.ProtoReflect.Descriptor instead. func (*DrainShard) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} } func (x *DrainShard) GetShardId() string { @@ -2553,7 +2643,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2565,7 +2655,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[23] + mi := &file_llmgw_v1_agent_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2578,7 +2668,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} } func (x *Ack) GetMessageId() string { @@ -2622,7 +2712,7 @@ type LogsRequest struct { func (x *LogsRequest) Reset() { *x = LogsRequest{} - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2634,7 +2724,7 @@ func (x *LogsRequest) String() string { func (*LogsRequest) ProtoMessage() {} func (x *LogsRequest) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[24] + mi := &file_llmgw_v1_agent_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2647,7 +2737,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead. func (*LogsRequest) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} } func (x *LogsRequest) GetRequestId() string { @@ -2687,7 +2777,7 @@ type LogsReply struct { func (x *LogsReply) Reset() { *x = LogsReply{} - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2699,7 +2789,7 @@ func (x *LogsReply) String() string { func (*LogsReply) ProtoMessage() {} func (x *LogsReply) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[25] + mi := &file_llmgw_v1_agent_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2712,7 +2802,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message { // Deprecated: Use LogsReply.ProtoReflect.Descriptor instead. func (*LogsReply) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26} } func (x *LogsReply) GetRequestId() string { @@ -2868,7 +2958,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\tShardLoad\x12\x19\n" + "\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" + "\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" + - "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x92\x04\n" + + "\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x88\x05\n" + "\x06IBPort\x12\x10\n" + "\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" + "\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" + @@ -2890,10 +2980,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\bsm_error\x18\x0e \x01(\tR\asmError\x12\x18\n" + "\anetwork\x18\x0f \x01(\tR\anetwork\x12\x1d\n" + "\n" + - "gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x1a;\n" + + "gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x12@\n" + + "\x0fsubnet_managers\x18\x11 \x03(\v2\x17.llmgw.v1.SubnetManagerR\x0esubnetManagers\x122\n" + + "\x15subnet_managers_error\x18\x12 \x01(\tR\x13subnetManagersError\x1a;\n" + "\rCountersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" + + "\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"g\n" + + "\rSubnetManager\x12\x12\n" + + "\x04guid\x18\x01 \x01(\tR\x04guid\x12\x10\n" + + "\x03lid\x18\x02 \x01(\x05R\x03lid\x12\x1a\n" + + "\bpriority\x18\x03 \x01(\x05R\bpriority\x12\x14\n" + + "\x05state\x18\x04 \x01(\tR\x05state\"\xfc\x05\n" + "\tHeartbeat\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" + "\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" + @@ -3033,7 +3130,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 32) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 33) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -3048,70 +3145,72 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad (*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad (*IBPort)(nil), // 12: llmgw.v1.IBPort - (*Heartbeat)(nil), // 13: llmgw.v1.Heartbeat - (*Provision)(nil), // 14: llmgw.v1.Provision - (*EngineWant)(nil), // 15: llmgw.v1.EngineWant - (*ProvisionStatus)(nil), // 16: llmgw.v1.ProvisionStatus - (*ShardStatus)(nil), // 17: llmgw.v1.ShardStatus - (*FetchProgress)(nil), // 18: llmgw.v1.FetchProgress - (*StartShard)(nil), // 19: llmgw.v1.StartShard - (*EngineProfile)(nil), // 20: llmgw.v1.EngineProfile - (*RemoveWeights)(nil), // 21: llmgw.v1.RemoveWeights - (*StopShard)(nil), // 22: llmgw.v1.StopShard - (*DrainShard)(nil), // 23: llmgw.v1.DrainShard - (*Ack)(nil), // 24: llmgw.v1.Ack - (*LogsRequest)(nil), // 25: llmgw.v1.LogsRequest - (*LogsReply)(nil), // 26: llmgw.v1.LogsReply - nil, // 27: llmgw.v1.Register.LabelsEntry - nil, // 28: llmgw.v1.Register.EngineVersionsEntry - nil, // 29: llmgw.v1.GPU.NicPathsEntry - nil, // 30: llmgw.v1.IBPort.CountersEntry - nil, // 31: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 32: llmgw.v1.EngineProfile.EnvEntry + (*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager + (*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat + (*Provision)(nil), // 15: llmgw.v1.Provision + (*EngineWant)(nil), // 16: llmgw.v1.EngineWant + (*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus + (*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus + (*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress + (*StartShard)(nil), // 20: llmgw.v1.StartShard + (*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile + (*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights + (*StopShard)(nil), // 23: llmgw.v1.StopShard + (*DrainShard)(nil), // 24: llmgw.v1.DrainShard + (*Ack)(nil), // 25: llmgw.v1.Ack + (*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest + (*LogsReply)(nil), // 27: llmgw.v1.LogsReply + nil, // 28: llmgw.v1.Register.LabelsEntry + nil, // 29: llmgw.v1.Register.EngineVersionsEntry + nil, // 30: llmgw.v1.GPU.NicPathsEntry + nil, // 31: llmgw.v1.IBPort.CountersEntry + nil, // 32: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 33: llmgw.v1.EngineProfile.EnvEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register - 13, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat - 17, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus - 26, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply - 19, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard - 22, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard - 23, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard - 24, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack - 21, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights - 14, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision - 25, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest - 27, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat + 18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus + 27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply + 20, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard + 23, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard + 24, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard + 25, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack + 22, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights + 15, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision + 26, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest + 28, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 28, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 29, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage 4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host 0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State - 29, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 30, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad - 30, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry - 5, // 23: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage - 9, // 24: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 31, // 25: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry - 16, // 26: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus - 10, // 27: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad - 12, // 28: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort - 8, // 29: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment - 15, // 30: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 0, // 31: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State - 18, // 32: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress - 20, // 33: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 32, // 34: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry - 1, // 35: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 2, // 36: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 36, // [36:37] is the sub-list for method output_type - 35, // [35:36] is the sub-list for method input_type - 35, // [35:35] is the sub-list for extension type_name - 35, // [35:35] is the sub-list for extension extendee - 0, // [0:35] is the sub-list for field type_name + 31, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 13, // 23: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager + 5, // 24: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage + 9, // 25: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights + 32, // 26: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 17, // 27: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus + 10, // 28: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad + 12, // 29: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort + 8, // 30: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment + 16, // 31: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant + 0, // 32: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State + 19, // 33: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress + 21, // 34: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile + 33, // 35: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 1, // 36: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 2, // 37: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 37, // [37:38] is the sub-list for method output_type + 36, // [36:37] is the sub-list for method input_type + 36, // [36:36] is the sub-list for extension type_name + 36, // [36:36] is the sub-list for extension extendee + 0, // [0:36] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -3140,7 +3239,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 32, + NumMessages: 33, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index d9a7874d..0bee445d 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -296,6 +296,21 @@ message IBPort { // subnet manager, and the network is what its fabric is. string network = 15; string gid_prefix = 16; + // subnet_managers is every subnet manager the fabric's subnet + // administrator lists — master and standbys — asked through one port per + // fabric, so it is on that port only. It is what keeps a fabric one fabric + // through a failover: the master changes, the list does not. + repeated SubnetManager subnet_managers = 17; + string subnet_managers_error = 18; +} + +// SubnetManager is one subnet manager on a fabric. +message SubnetManager { + string guid = 1; + int32 lid = 2; + int32 priority = 3; + // state is master, standby, discovering or not active. + string state = 4; } message Heartbeat { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index f659e5ea..405b6b9c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -19,6 +19,7 @@ import ( "os/exec" "path/filepath" "slices" + "strconv" "strings" "sync" "time" @@ -364,6 +365,10 @@ type Agent struct { sm map[string]*discovery.SMInfo smErr map[string]string smAsked time.Time + // sms is every subnet manager on a fabric, asked through one port per + // master, keyed by that port; smsErr says why a port could not ask. + sms map[string][]discovery.SMRecord + smsErr map[string]string // load measures the machine between heartbeats. load *discovery.LoadSampler @@ -663,13 +668,18 @@ func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) { RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters, NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix, } + a.mu.Lock() if p.SM != nil { ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State } else { - a.mu.Lock() ip.SmError = a.smErr[smKey(h.Name, p.Num)] - a.mu.Unlock() } + for _, r := range a.sms[smKey(h.Name, p.Num)] { + ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{ + Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State}) + } + ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)] + a.mu.Unlock() hb.IbPorts = append(hb.IbPorts, ip) } } @@ -1629,6 +1639,17 @@ func ResolveManagement(c Config) (addr, iface string) { func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) } +// querySMs is discovery.QuerySMs, replaced in tests. +var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) { + return discovery.QuerySMs(context.Background(), hca, port) +} + +func splitSMKey(k string) (string, int) { + i := strings.LastIndexByte(k, '/') + port, _ := strconv.Atoi(k[i+1:]) + return k[:i], port +} + // querySM is discovery.QuerySM, replaced in tests. var querySM = func(hca string, port int) (*discovery.SMInfo, error) { return discovery.QuerySM(context.Background(), hca, port) @@ -1651,6 +1672,23 @@ func (a *Agent) attachSM(hcas []discovery.HCA) { sm[smKey(hca, port)] = got return got, nil }) + // Every subnet manager on each fabric, through one port per master: + // the same list through every port of one fabric is one query too + // many for each. + sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{} + asked := map[string]bool{} + for k, v := range sm { + if asked[v.GUID] { + continue + } + asked[v.GUID] = true + hca, port := splitSMKey(k) + if recs, err := querySMs(hca, port); err != nil { + smsErr[k] = err.Error() + } else { + sms[k] = recs + } + } a.mu.Lock() for k, v := range sm { if old := a.sm[k]; old != nil && old.GUID != v.GUID { @@ -1658,6 +1696,7 @@ func (a *Agent) attachSM(hcas []discovery.HCA) { } } a.sm, a.smErr, a.smAsked = sm, errs, time.Now() + a.sms, a.smsErr = sms, smsErr a.mu.Unlock() return } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index e3bcdd48..47bf4149 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -828,3 +828,36 @@ func TestTheManagementInterfaceIsFoundNotAssumed(t *testing.T) { t.Errorf("interface = %q, want the loopback interface holding 127.0.0.1", cfg.MgmtInterface) } } + +func TestEachFabricsSubnetManagersAreAskedOnce(t *testing.T) { + // Two adapters on one fabric see one master. Its managers are listed + // through one of them — the same list twice is a query too many — and + // every port says which master it sees. + oldSM, oldSMs := querySM, querySMs + t.Cleanup(func() { querySM, querySMs = oldSM, oldSMs }) + querySM = func(string, int) (*discovery.SMInfo, error) { + return &discovery.SMInfo{GUID: "0x01", LID: 1, State: "SMINFO_MASTER"}, nil + } + lists := 0 + querySMs = func(string, int) ([]discovery.SMRecord, error) { + lists++ + return []discovery.SMRecord{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}}, nil + } + h := start(t, func(c *Config) { + c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1") + c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}} + }) + hb := h.ag.heartbeat().GetHeartbeat() + withList := 0 + for _, p := range hb.GetIbPorts() { + if p.GetSmGuid() != "0x01" { + t.Errorf("%s sees master %q", p.GetHca(), p.GetSmGuid()) + } + if len(p.GetSubnetManagers()) > 0 { + withList++ + } + } + if lists != 1 || withList != 1 { + t.Errorf("listed %d times, carried on %d ports; want once, on one", lists, withList) + } +} diff --git a/internal/agent/discovery/ib_test.go b/internal/agent/discovery/ib_test.go index c8640afb..4f2a0de6 100644 --- a/internal/agent/discovery/ib_test.go +++ b/internal/agent/discovery/ib_test.go @@ -323,3 +323,31 @@ func TestARoCEPortsNetworkIsItsFabric(t *testing.T) { t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey()) } } + +// smirDump is `saquery SMIR` on a fabric with a master and a standby, in the +// layout infiniband-diags prints. +const smirDump = "SMInfoRecord dump:\n\t\tRID\n\t\tLID...................1\n\t\tSMInfo dump:\n" + + "\t\tGUID..................0x248a07030030f0c0\n\t\tSM_Key................0x0000000000000000\n" + + "\t\tActCount..............386773\n\t\tPriority..............15\n\t\tSMState...............3\n" + + "SMInfoRecord dump:\n\t\tRID\n\t\tLID...................12\n\t\tSMInfo dump:\n" + + "\t\tGUID..................0x248a07030030f1d4\n\t\tSM_Key................0x0000000000000000\n" + + "\t\tActCount..............12\n\t\tPriority..............14\n\t\tSMState...............2\n" + +func TestEverySubnetManagerOnAFabricIsListed(t *testing.T) { + recs, err := ParseSMIR(smirDump) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 { + t.Fatalf("records = %+v", recs) + } + if recs[0].GUID != "0x248a07030030f0c0" || recs[0].State != "master" || recs[0].Priority != 15 || recs[0].LID != 1 { + t.Errorf("master = %+v", recs[0]) + } + if recs[1].State != "standby" || recs[1].LID != 12 { + t.Errorf("standby = %+v", recs[1]) + } + if _, err := ParseSMIR("ibwarn: can't open UMAD port"); err == nil { + t.Error("an error should not parse") + } +} diff --git a/internal/agent/discovery/sminfo.go b/internal/agent/discovery/sminfo.go index 0d0a170d..ff22b5a9 100644 --- a/internal/agent/discovery/sminfo.go +++ b/internal/agent/discovery/sminfo.go @@ -71,3 +71,76 @@ func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) err } return first } + +// SMRecord is one subnet manager on a fabric, as the subnet administrator +// lists them: master and standbys alike. +type SMRecord struct { + GUID string `json:"guid"` + LID int `json:"lid"` + Priority int `json:"priority"` + // State is master, standby, discovering or not active. + State string `json:"state"` +} + +// smStates are SMInfo's state numbers. +var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"} + +// smirField is one line of saquery's record dump: "LID.........1". +var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`) + +// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet +// administrator knows of, one "SMInfoRecord dump:" block each. +func ParseSMIR(out string) ([]SMRecord, error) { + var recs []SMRecord + var cur *SMRecord + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, "SMInfoRecord dump") { + recs = append(recs, SMRecord{}) + cur = &recs[len(recs)-1] + continue + } + m := smirField.FindStringSubmatch(line) + if m == nil || cur == nil { + continue + } + switch strings.ToLower(m[1]) { + case "lid": + cur.LID, _ = strconv.Atoi(m[2]) + case "guid": + cur.GUID = normalizeGUID(m[2]) + case "priority": + cur.Priority, _ = strconv.Atoi(m[2]) + case "smstate": + cur.State = smStates[m[2]] + if cur.State == "" { + cur.State = m[2] + } + } + } + out2 := recs[:0] + for _, r := range recs { + if r.GUID != "" { + out2 = append(out2, r) + } + } + if len(out2) == 0 { + return nil, fmt.Errorf("no subnet manager records in saquery output") + } + return out2, nil +} + +// QuerySMs asks the subnet administrator, through one port, for every subnet +// manager on the fabric. +func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) { + bin, err := exec.LookPath("saquery") + if err != nil { + return nil, ErrNoDiags + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput() + if err != nil { + return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out))) + } + return ParseSMIR(string(out)) +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 331c22d0..8d0763f5 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -10,7 +10,6 @@ import ( "context" "encoding/json" "fmt" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "io" "net/http" "slices" @@ -20,6 +19,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/bench" "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -836,5 +836,6 @@ func (a *API) profileChanged(model, profile, was string) bool { // Fabrics is the physical fabrics the adapters are on, against the names // the nodes gave them. func (a *API) Fabrics() fabrics.Report { - return fabrics.Analyze(a.ctl.Inventory().Nodes()) + inv := a.ctl.Inventory() + return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now()) } diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 02b73438..17973c4f 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -405,9 +405,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) { n.Load = loadFrom(l, time.Now()) } if ports := hb.GetIbPorts(); len(ports) > 0 { - n.IBPorts = n.IBPorts[:0] + before := map[string]string{} + for _, p := range n.IBPorts { + before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID + } + n.IBPorts = nil for _, p := range ports { + // A different master than last time is a failover, or a port + // moved: the fabric's own event, kept for the page to show. + where := fmt.Sprintf("%s/%d", p.GetHca(), p.GetPort()) + if old := before[where]; old != "" && p.GetSmGuid() != "" && old != p.GetSmGuid() { + s.log.Warn("a port sees a different master subnet manager", "node", nodeID, + "port", where, "from", old, "to", p.GetSmGuid()) + s.inv.RecordSMChange(inventory.SMChange{At: time.Now(), Node: nodeID, Port: where, + From: old, To: p.GetSmGuid()}) + } + var sms []inventory.SubnetManager + for _, r := range p.GetSubnetManagers() { + sms = append(sms, inventory.SubnetManager{GUID: r.GetGuid(), LID: int(r.GetLid()), + Priority: int(r.GetPriority()), State: r.GetState()}) + } n.IBPorts = append(n.IBPorts, inventory.IBPort{ + SMs: sms, SMsError: p.GetSubnetManagersError(), HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(), PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()), LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()), diff --git a/internal/control/fabrics/fabrics.go b/internal/control/fabrics/fabrics.go index 4af1ca23..6430c6af 100644 --- a/internal/control/fabrics/fabrics.go +++ b/internal/control/fabrics/fabrics.go @@ -13,6 +13,7 @@ import ( "slices" "sort" "strings" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) @@ -41,8 +42,17 @@ type Physical struct { Kind string `json:"kind"` // Label is the key as a person reads it. Label string `json:"label"` - // SM is the master subnet manager, on InfiniBand. + // SM is the master subnet manager, on InfiniBand: the one most of its + // ports see. SM *SM `json:"sm,omitempty"` + // SMs is every subnet manager the fabric lists, master first; empty when + // no port could ask. Standbys counts those that could take over. + SMs []SM `json:"sms,omitempty"` + Standbys int `json:"standbys"` + // Masters is how many ports see each master, when they do not all agree. + Masters map[string]int `json:"masters,omitempty"` + // Changes are the master changes seen in the last day, newest first. + Changes []Change `json:"changes,omitempty"` // Declared are the names nodes gave this fabric; one, when all is well. Declared []string `json:"declared"` Members []Member `json:"members"` @@ -58,6 +68,14 @@ type SM struct { State string `json:"state"` } +// Change is a fabric's master subnet manager changing, as its ports saw it. +type Change struct { + At time.Time `json:"at"` + From string `json:"from"` + To string `json:"to"` + Ports int `json:"ports"` +} + // Member is one node's adapters on a physical fabric. type Member struct { Node string `json:"node"` @@ -76,9 +94,15 @@ type Report struct { Unverified map[string][]string `json:"unverified,omitempty"` } -// Analyze reads every node's ports. -func Analyze(nodes []inventory.Node) Report { +// Analyze reads every node's ports, and the changes of master subnet manager +// the control plane has recorded. +func Analyze(nodes []inventory.Node, changes []inventory.SMChange, now time.Time) Report { r := Report{Unverified: map[string][]string{}} + canon := inventory.CanonicalSMs(nodes) + masters := map[string]map[string]int{} // key -> master GUID -> ports + smLists := map[string]map[string]inventory.SubnetManager{} + listErr := map[string]string{} + noSM := map[string][]string{} // declared -> node/port up with no SM byKey := map[string]*Physical{} declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes @@ -94,8 +118,24 @@ func Analyze(nodes []inventory.Node) Report { members := map[string]*Member{} // key -> this node's membership for _, p := range n.IBPorts { - key := p.FabricKey() + key := p.FabricKeyIn(canon) decl := declOf[p.HCA] + if decl != "" && !p.RoCE() && strings.EqualFold(p.State, "INIT") && strings.EqualFold(p.PhysState, "LinkUp") { + noSM[decl] = appendOnce(noSM[decl], fmt.Sprintf("%s %s/%d", n.ID, p.HCA, p.Port)) + } + if key != "" && !p.RoCE() { + if masters[key] == nil { + masters[key] = map[string]int{} + smLists[key] = map[string]inventory.SubnetManager{} + } + masters[key][p.SMGUID]++ + for _, sm := range p.SMs { + smLists[key][sm.GUID] = sm + } + if p.SMsError != "" { + listErr[key] = p.SMsError + } + } if key == "" { if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" { r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError) @@ -109,7 +149,7 @@ func Analyze(nodes []inventory.Node) Report { ph.Kind, ph.Label = "roce", "RoCE "+p.Network } else { ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState} - ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID) + ph.Label = "fabric of subnet manager " + strings.TrimPrefix(key, "sm:") } byKey[key] = ph } @@ -162,7 +202,10 @@ func Analyze(nodes []inventory.Node) Report { "%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name", decl, len(keys), strings.Join(parts, "; "))}) } - for _, ph := range byKey { + for key, ph := range byKey { + if ph.Kind == "infiniband" { + analyzeSMs(&r, ph, masters[key], smLists[key], listErr[key], nodes, changes, now) + } sort.Strings(ph.Declared) sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node }) if len(ph.Declared) > 1 { @@ -195,6 +238,11 @@ func Analyze(nodes []inventory.Node) Report { len(labels), prefix)}) } } + for decl, ports := range noSM { + r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( + "%s: %d port(s) have a link but no subnet manager has configured them (INIT): %s", + decl, len(ports), strings.Join(ports, ", "))}) + } for decl, why := range r.Unverified { r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf( "%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))}) @@ -209,6 +257,94 @@ func Analyze(nodes []inventory.Node) Report { return r } +// analyzeSMs fills in a fabric's subnet managers and what is wrong with them. +func analyzeSMs(r *Report, ph *Physical, masters map[string]int, list map[string]inventory.SubnetManager, + listErr string, nodes []inventory.Node, changes []inventory.SMChange, now time.Time, +) { + name := first(ph.Declared) + if name == "~" { + name = ph.Label + } + // The master is the one most ports see. + best, most := "", -1 + for g, n := range masters { + if n > most || (n == most && g < best) { + best, most = g, n + } + } + if best != "" && ph.SM != nil && ph.SM.GUID != best { + ph.SM = &SM{GUID: best} + } + if len(masters) > 1 { + ph.Masters = masters + var parts []string + for _, g := range sortedKeys(masters) { + parts = append(parts, fmt.Sprintf("%s by %d", g, masters[g])) + } + r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( + "%s: its ports do not agree on the master subnet manager (%s) — a failover in progress, or a fabric split in two", + name, strings.Join(parts, ", "))}) + } + + for _, sm := range list { + v := SM{GUID: sm.GUID, LID: sm.LID, Priority: sm.Priority, State: sm.State} + ph.SMs = append(ph.SMs, v) + if sm.State == "standby" { + ph.Standbys++ + } + if ph.SM != nil && sm.GUID == ph.SM.GUID { + ph.SM.LID, ph.SM.Priority, ph.SM.State = sm.LID, sm.Priority, sm.State + } + } + sort.Slice(ph.SMs, func(i, j int) bool { + if (ph.SMs[i].State == "master") != (ph.SMs[j].State == "master") { + return ph.SMs[i].State == "master" + } + return ph.SMs[i].Priority > ph.SMs[j].Priority + }) + switch { + case len(list) == 0 && listErr != "": + r.Problems = append(r.Problems, Problem{Level: Info, Fabric: name, Message: fmt.Sprintf( + "%s: its standby subnet managers cannot be listed: %s", name, listErr)}) + case len(list) > 0 && ph.Standbys == 0: + r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( + "%s has one subnet manager and no standby: if it stops, the fabric keeps its routes but cannot recover from any change — a cable, a switch, a node rebooting", + name)}) + } + + // Its recent failovers: the recorded changes of ports on this fabric, + // one per minute of changes rather than one per port. + onFabric := map[string]bool{} + for _, m := range ph.Members { + for _, h := range m.HCAs { + onFabric[m.Node+" "+h] = true + } + } + day := now.Add(-24 * time.Hour) + for _, c := range changes { + hca := c.Port + if i := strings.LastIndexByte(hca, '/'); i >= 0 { + hca = hca[:i] + } + if c.At.Before(day) || !onFabric[c.Node+" "+hca] { + continue + } + if n := len(ph.Changes); n > 0 && ph.Changes[n-1].To == c.To && c.At.Sub(ph.Changes[n-1].At) < 2*time.Minute { + ph.Changes[n-1].Ports++ + continue + } + ph.Changes = append(ph.Changes, Change{At: c.At, From: c.From, To: c.To, Ports: 1}) + } + slices.Reverse(ph.Changes) + if len(ph.Changes) > 0 && now.Sub(ph.Changes[0].At) < time.Hour { + c := ph.Changes[0] + r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( + "%s: the master subnet manager changed %s ago, from %s to %s", + name, now.Sub(c.At).Round(time.Minute), c.From, c.To)}) + } + _ = nodes +} + func rank(l Level) int { switch l { case Fail: diff --git a/internal/control/fabrics/fabrics_test.go b/internal/control/fabrics/fabrics_test.go index ac9a81df..d2a50d8a 100644 --- a/internal/control/fabrics/fabrics_test.go +++ b/internal/control/fabrics/fabrics_test.go @@ -3,6 +3,7 @@ package fabrics import ( "strings" "testing" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) @@ -30,7 +31,7 @@ func problem(t *testing.T, r Report, l Level, fragment string) { } func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) { - r := Analyze([]inventory.Node{ + r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x02", "mlx5_0"), }) @@ -50,14 +51,14 @@ func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) { } func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) { - r := Analyze([]inventory.Node{ + r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"), }) problem(t, r, Fail, "ib-a is 2 separate fabrics") } func TestTwoNamesOnOneFabricAreSaid(t *testing.T) { - r := Analyze([]inventory.Node{ + r := analyze([]inventory.Node{ ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"), }) problem(t, r, Warn, "ib-a and ib-b are one fabric") @@ -66,26 +67,84 @@ func TestTwoNamesOnOneFabricAreSaid(t *testing.T) { func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) { n := ibNode("a1", "ib-a", "0x01", "mlx5_0") n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"}) - problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09") + problem(t, analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on fabric of subnet manager 0x09") } func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) { n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1") n.IBPorts[1].SMGUID = "0x02" - problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics") + problem(t, analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics") } func TestANodeThatCannotSayIsReported(t *testing.T) { n := ibNode("x", "ib-a", "", "mlx5_0") n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags" - problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags") + problem(t, analyze([]inventory.Node{n}), Info, "install infiniband-diags") } func TestRoCEIsItsNetwork(t *testing.T) { n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}}, IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}} - r := Analyze([]inventory.Node{n}) + r := analyze([]inventory.Node{n}) if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" { t.Errorf("physical = %+v", r.Physical) } } + +func analyze(nodes []inventory.Node) Report { return Analyze(nodes, nil, time.Now()) } + +func withSMs(n inventory.Node, sms ...inventory.SubnetManager) inventory.Node { + n.IBPorts[0].SMs = sms + return n +} + +func TestAFailoverKeepsAFabricOneFabric(t *testing.T) { + // Mid-failover some ports see the old master, some the new. Keyed by the + // master alone, one fabric read as two for a minute — and a declared name + // covering it as "two separate fabrics". The list of managers is the same + // before and after, which is what keeps it one. + sms := []inventory.SubnetManager{{GUID: "0x01", State: "master", Priority: 15}, {GUID: "0x02", State: "standby", Priority: 14}} + r := analyze([]inventory.Node{ + withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), sms...), + ibNode("a2", "ib-a", "0x02", "mlx5_0"), + }) + if len(r.Physical) != 1 { + t.Fatalf("one fabric read as %d: %+v", len(r.Physical), r.Physical) + } + for _, p := range r.Problems { + if p.Level == Fail { + t.Errorf("a failover was reported as a failure: %s", p.Message) + } + } + problem(t, r, Warn, "do not agree on the master subnet manager") + if r.Physical[0].Standbys != 1 || len(r.Physical[0].SMs) != 2 || r.Physical[0].SMs[0].State != "master" { + t.Errorf("managers = %+v", r.Physical[0].SMs) + } +} + +func TestAFabricWithNoStandbyIsSaid(t *testing.T) { + r := analyze([]inventory.Node{withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), + inventory.SubnetManager{GUID: "0x01", State: "master"})}) + problem(t, r, Warn, "one subnet manager and no standby") +} + +func TestPortsUpWithoutASubnetManagerAreAFailure(t *testing.T) { + n := ibNode("a1", "ib-a", "", "mlx5_0") + n.IBPorts[0].State, n.IBPorts[0].PhysState = "INIT", "LinkUp" + problem(t, analyze([]inventory.Node{n}), Fail, "no subnet manager has configured them") +} + +func TestARecentFailoverIsShown(t *testing.T) { + now := time.Now() + nodes := []inventory.Node{withSMs(ibNode("a1", "ib-a", "0x02", "mlx5_0"), + inventory.SubnetManager{GUID: "0x01", State: "standby"}, inventory.SubnetManager{GUID: "0x02", State: "master"})} + changes := []inventory.SMChange{ + {At: now.Add(-20 * time.Minute), Node: "a1", Port: "mlx5_0/1", From: "0x01", To: "0x02"}, + {At: now.Add(-48 * time.Hour), Node: "a1", Port: "mlx5_0/1", From: "0x02", To: "0x01"}, + } + r := Analyze(nodes, changes, now) + if got := r.Physical[0].Changes; len(got) != 1 || got[0].To != "0x02" { + t.Errorf("changes in the last day = %+v", got) + } + problem(t, r, Warn, "the master subnet manager changed 20m0s ago") +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index 1279f302..d9e1c2d6 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -16,10 +16,12 @@ package fleetmetrics import ( "strconv" + "strings" "time" "github.com/prometheus/client_golang/prometheus" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" ) @@ -71,6 +73,10 @@ type Collector struct { ibActive *prometheus.Desc ibRate *prometheus.Desc ibCounter *prometheus.Desc + + fabricSMs *prometheus.Desc + fabricMaster *prometheus.Desc + fabricProblems *prometheus.Desc } // loadFresh is how old a node's load may be and still be reported. A node @@ -160,6 +166,15 @@ func New(src Source) *Collector { ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps", "A port's negotiated rate. A link that trained below its usual speed is a "+ "cable or transceiver to look at.", []string{"node", "hca", "port"}, nil), + fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers", + "Subnet managers a physical fabric lists, by state. No standby means the fabric "+ + "cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil), + fabricMaster: prometheus.NewDesc("llmgw_fabric_master_sm", + "1 for a physical fabric's master subnet manager, as most of its ports see it; a "+ + "change of sm_guid is a failover.", []string{"fabric", "physical", "sm_guid"}, nil), + fabricProblems: prometheus.NewDesc("llmgw_fabric_problems", + "Disagreements about the fabrics, by level: fail means multi-node work will not "+ + "place or will not run where it should.", []string{"level"}, nil), ibCounter: prometheus.NewDesc("llmgw_ib_port_counter_total", "A port's cumulative counters, by sysfs name: symbol_error, link_downed and the "+ "other errors say a link is going bad before a job fails on it; port_xmit_data "+ @@ -173,7 +188,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) { c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, c.nodeDiskFree, c.nodeDiskSize, c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory, - c.ibActive, c.ibRate, c.ibCounter, + c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems, c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, @@ -285,6 +300,30 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { } } + // The fabrics as the adapters find them: their subnet managers, and how + // many things disagree. + rep := fabrics.Analyze(c.src.Nodes(), nil, now) + for _, ph := range rep.Physical { + name := strings.Join(ph.Declared, ",") + if ph.SM != nil && ph.SM.GUID != "" { + ch <- prometheus.MustNewConstMetric(c.fabricMaster, prometheus.GaugeValue, 1, name, ph.Key, ph.SM.GUID) + } + byState := map[string]int{} + for _, sm := range ph.SMs { + byState[sm.State]++ + } + for st, n := range byState { + ch <- prometheus.MustNewConstMetric(c.fabricSMs, prometheus.GaugeValue, float64(n), name, ph.Key, st) + } + } + levels := map[fabrics.Level]int{fabrics.Fail: 0, fabrics.Warn: 0, fabrics.Info: 0} + for _, p := range rep.Problems { + levels[p.Level]++ + } + for l, n := range levels { + ch <- prometheus.MustNewConstMetric(c.fabricProblems, prometheus.GaugeValue, float64(n), string(l)) + } + // Replicas by model and state, so a model stuck starting is visible as a // shape on a graph rather than only in a log line nobody tails. counts := map[[2]string]int{} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 6a63bd2c..6c114bd2 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -142,6 +142,75 @@ type IBPort struct { SMError string // Network is a RoCE port's IP network. Network string + // SMs is every subnet manager on the port's fabric, master and standbys, + // on the one port per fabric that asked; SMsError says why it could not. + SMs []SubnetManager + SMsError string +} + +// SubnetManager is one subnet manager on a fabric. +type SubnetManager struct { + GUID string + LID int + Priority int + State string +} + +// CanonicalSMs maps every subnet manager GUID the cluster has heard of to +// one name for its fabric: the lowest GUID among the managers listed +// together. A fabric's master changes on a failover and its list does not, +// so ports that see the old master and ports that already see the new one +// stay one fabric through it. +func CanonicalSMs(nodes []Node) map[string]string { + parent := map[string]string{} + var find func(string) string + find = func(x string) string { + p, ok := parent[x] + if !ok || p == x { + parent[x] = x + return x + } + root := find(p) + parent[x] = root + return root + } + union := func(a, b string) { + ra, rb := find(a), find(b) + if ra == rb { + return + } + if ra < rb { + parent[rb] = ra + } else { + parent[ra] = rb + } + } + for _, n := range nodes { + for _, p := range n.IBPorts { + if p.SMGUID != "" { + find(p.SMGUID) + } + for _, sm := range p.SMs { + union(sm.GUID, p.SMGUID) + if p.SMGUID == "" { + find(sm.GUID) + } + } + } + } + out := make(map[string]string, len(parent)) + for g := range parent { + out[g] = find(g) + } + return out +} + +// SMChange is a port seeing a different master subnet manager than before. +type SMChange struct { + At time.Time `json:"at"` + Node string `json:"node"` + Port string `json:"port"` + From, To string } // Active reports whether the port can carry traffic. @@ -153,7 +222,11 @@ func (p IBPort) RoCE() bool { return strings.EqualFold(p.LinkLayer, "Ethernet") // FabricKey names the physical fabric the port is on: its master subnet // manager on InfiniBand, its IP network on RoCE. Empty when not known, and // for a port that is not up. -func (p IBPort) FabricKey() string { +func (p IBPort) FabricKey() string { return p.FabricKeyIn(nil) } + +// FabricKeyIn is FabricKey with the subnet manager named as canon has it, +// so a fabric keeps one key through a failover. +func (p IBPort) FabricKeyIn(canon map[string]string) string { if !p.Active() { return "" } @@ -161,6 +234,9 @@ func (p IBPort) FabricKey() string { case p.RoCE() && p.Network != "": return "roce:" + p.Network case !p.RoCE() && p.SMGUID != "": + if c, ok := canon[p.SMGUID]; ok { + return "sm:" + c + } return "sm:" + p.SMGUID } return "" @@ -169,13 +245,13 @@ func (p IBPort) FabricKey() string { // PhysicalFabric is the physical fabric this node's adapters for a declared // fabric are on. Known is false when none of them says; ok is false when // they disagree — two adapters declared as one fabric that are cabled to two. -func (n Node) PhysicalFabric(fabric string) (key string, known, ok bool) { +func (n Node) PhysicalFabric(fabric string, canon map[string]string) (key string, known, ok bool) { hcas := n.HCAsFor(fabric) for _, p := range n.IBPorts { if !slices.Contains(hcas, p.HCA) { continue } - k := p.FabricKey() + k := p.FabricKeyIn(canon) if k == "" { continue } @@ -279,6 +355,8 @@ type Store struct { nodes map[string]*Node // allocs maps node id to the shards placed there. allocs map[string][]Allocation + // smChanges are ports' changes of master subnet manager, newest last. + smChanges []SMChange } // NewStore returns an empty inventory. @@ -288,6 +366,25 @@ func NewStore() *Store { // Upsert records a node's registration, preserving operator-set state such as // a cordon: an agent reconnecting must not silently undo an operator's decision. +// RecordSMChange keeps a port's change of master subnet manager, the last +// few hundred of them: a failover is the fabric's own event, and nothing +// else would remember it once the ports agree again. +func (s *Store) RecordSMChange(c SMChange) { + s.mu.Lock() + defer s.mu.Unlock() + s.smChanges = append(s.smChanges, c) + if len(s.smChanges) > 500 { + s.smChanges = slices.Clone(s.smChanges[len(s.smChanges)-500:]) + } +} + +// SMChanges returns the recorded changes, oldest first. +func (s *Store) SMChanges() []SMChange { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.smChanges) +} + func (s *Store) Upsert(n Node) { s.mu.Lock() defer s.mu.Unlock() diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 1b686a3e..38fd0d2b 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -246,9 +246,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej }} } + canon := inventory.CanonicalSMs(s.inv.Nodes()) for _, fabric := range fabrics { all := s.eligibleNodes(p, fabric, avoid, &rejections) - for _, group := range physicalGroups(fabric, all, p, &rejections) { + for _, group := range physicalGroups(fabric, all, p, canon, &rejections) { pool := fabric + "/" + p.classLabel() if group.key != "" && group.split { pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel() @@ -278,10 +279,10 @@ type fabricGroup struct { // subnet manager each node's ports see (by IP network on RoCE), a replica // stays on one. Nodes that do not say — an agent without infiniband-diags — // are a group of their own rather than trusted to join one that does. -func physicalGroups(fabric string, cands []candidate, p Profile, rejections *[]Rejection) []fabricGroup { +func physicalGroups(fabric string, cands []candidate, p Profile, canon map[string]string, rejections *[]Rejection) []fabricGroup { by := map[string][]candidate{} for _, c := range cands { - key, _, ok := c.node.PhysicalFabric(fabric) + key, _, ok := c.node.PhysicalFabric(fabric, canon) if !ok { *rejections = append(*rejections, Rejection{ Profile: p.Name, Pool: fabric + "/" + p.classLabel(), Node: c.node.ID, diff --git a/internal/control/placement/placement_test.go b/internal/control/placement/placement_test.go index 9b91d1d0..8c47558a 100644 --- a/internal/control/placement/placement_test.go +++ b/internal/control/placement/placement_test.go @@ -958,3 +958,15 @@ func TestAdaptersOfOneDeclaredFabricOnTwoAreRefused(t *testing.T) { t.Errorf("expected a refusal naming the cabling:\n%s", res.Explain()) } } + +func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) { + // One port already sees the new master, the other the old; the managers + // listed are the same, so the two nodes are still one fabric. + a := onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01") + a.IBPorts[0].SMs = []inventory.SubnetManager{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}} + b := onSM(node("a2", "h100-80g", 8, "ib-a"), "0x02") + res := New(store(a, b)).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("a failover split the fabric:\n%s", res.Explain()) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 2f5a9082..512d61d9 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -62,7 +62,11 @@ type Facts struct { HCAs []discovery.HCA `json:"hcas"` // SMErrors say, per port, why the subnet manager could not be asked. SMErrors map[string]string `json:"sm_errors,omitempty"` - Modules map[string]bool `json:"modules"` + // SMLists are every subnet manager on each fabric, by master GUID; + // SMListErrors why a fabric's could not be listed. + SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"` + SMListErrors map[string]string `json:"sm_list_errors,omitempty"` + Modules map[string]bool `json:"modules"` // ConfigPath is the agent configuration read, empty when there was none. ConfigPath string `json:"config_path,omitempty"` @@ -136,6 +140,25 @@ func Collect(ctx context.Context, o Options) Facts { } return sm, err }) + f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{} + for _, h := range f.HCAs { + for _, p := range h.Ports { + if p.SM == nil { + continue + } + if _, done := f.SMLists[p.SM.GUID]; done { + continue + } + if _, done := f.SMListErrors[p.SM.GUID]; done { + continue + } + if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil { + f.SMListErrors[p.SM.GUID] = err.Error() + } else { + f.SMLists[p.SM.GUID] = recs + } + } + } for _, m := range watchedModules { _, err := os.Stat(filepath.Join("/sys/module", m)) f.Modules[m] = err == nil @@ -340,6 +363,26 @@ func Evaluate(f Facts) []Finding { } } } + for _, master := range sortedKeys(f.SMLists) { + recs := f.SMLists[master] + var parts []string + standby := 0 + for _, r := range recs { + parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority)) + if r.State == "standby" { + standby++ + } + } + if standby == 0 { + add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "), + "run a second subnet manager, on a switch or a host, at a lower priority") + } else { + add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "") + } + } + for _, master := range sortedKeys(f.SMListErrors) { + add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "") + } if len(f.HCAs) > 0 { if !f.Modules["ib_uverbs"] { add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs") @@ -478,3 +521,12 @@ func orUnknown(s string) string { } return s } + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index d84d7784..560f5c06 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -27,6 +27,8 @@ func healthy() Facts { PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1, SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}} f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs) + f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": { + {GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}} return f } diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index d9e5e57d..bec61ecc 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -4,8 +4,6 @@ import ( "context" "errors" "fmt" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" - "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "html/template" "net/http" "sort" @@ -13,6 +11,8 @@ import ( "strings" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck" diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index e2f0b372..5b8e344b 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -2,13 +2,13 @@ package web import ( "context" - "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "html/template" "log/slog" "net/http" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index db8666d9..26b8b96b 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -146,6 +146,18 @@ {{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}not reported{{end}} {{end}} {{end}} +{{with .Fabrics}}{{if .Physical}} +

Subnet managers

+ +{{range .Physical}} + + + + + + +{{end}}
FabricIs physicallyMasterStandbyNodesLast change of master
{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}{{$d}}{{end}}{{else}}undeclared{{end}}{{.Label}}{{if .RateGbps}} · {{.RateGbps}} Gb/s{{end}}{{if eq .Kind "roce"}}none: RoCE{{else}}{{with .SM}}{{.GUID}}{{if .LID}} LID {{.LID}}, priority {{.Priority}}{{end}}{{else}}—{{end}}{{end}}{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}none{{end}}{{else}}not listed{{end}}{{len .Members}}{{with .Changes}}{{with index . 0}}{{ago .At}}: {{.From}} → {{.To}}{{end}}{{if gt (len .) 1}} ({{len .}} in a day){{end}}{{else}}none in a day{{end}}
+{{end}}{{end}}

A fabric's name is what the nodes were told; what it is physically is the subnet manager its InfiniBand ports see, or the network its RoCE ports are on. A replica spanning nodes is only ever formed on one of those. Fully free counts whole diff --git a/internal/web/web_test.go b/internal/web/web_test.go index a2838a45..8b4fc75d 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -657,14 +657,18 @@ type fabricSource struct{ fakeSource } func (fabricSource) Fabrics() fabrics.Report { return fabrics.Report{ Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)", - Declared: []string{"ib-a"}, RateGbps: 400}}, + Declared: []string{"ib-a"}, RateGbps: 400, + SM: &fabrics.SM{GUID: "0x01", LID: 1, Priority: 15, State: "master"}, + SMs: []fabrics.SM{{GUID: "0x01", State: "master"}}, + Changes: []fabrics.Change{{At: time.Now().Add(-3 * time.Hour), From: "0x02", To: "0x01", Ports: 4}}}}, Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}}, } } func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) { _, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil) - for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `fail`} { + for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `fail`, + "

Subnet managers

", "LID 1, priority 15", `none`, "3 hours ago: 0x02 → 0x01"} { if !strings.Contains(body, want) { t.Errorf("missing %q", want) }