Watch each fabric's subnet managers, and keep a fabric one through a failover
A fabric's identity was its master subnet manager, which changes on a failover: for the minute its ports took to agree, one fabric read as two, and a name covering it as two separate fabrics. Each agent now also lists every subnet manager on a fabric (saquery SMIR, through one port per master, once a minute); managers listed together are one fabric, named by the lowest GUID among them, so the identity survives the master changing. The same list says what a fabric can survive. The cluster page's new subnet manager table, /api/v1/fabrics, doctor and metrics (llmgw_fabric_master_sm, llmgw_fabric_subnet_managers, llmgw_fabric_problems) show each fabric's master, priority and standbys, and say when a fabric has no standby, its ports disagree on the master, ports are up with no subnet manager (INIT), or the master changed in the last hour. Every change of master a port reports is kept, and a fabric's last day of them is listed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
77fe801892
commit
23648fa19a
20 files changed
+857
-136
No files matched your search
+204
-105
@@ -90,7 +90,7 @@ func (x ShardStatus_State) Number() protoreflect.EnumNumber {
|
||||
|
||||
// Deprecated: Use ShardStatus_State.Descriptor instead.
|
||||
func (ShardStatus_State) EnumDescriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16, 0}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17, 0}
|
||||
}
|
||||
|
||||
// AgentMessage is anything a node tells the control plane.
|
||||
@@ -1415,10 +1415,16 @@ type IBPort struct {
|
||||
SmError string `protobuf:"bytes,14,opt,name=sm_error,json=smError,proto3" json:"sm_error,omitempty"`
|
||||
// network is, on RoCE, the IP network of the RoCE v2 address: RoCE has no
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"`
|
||||
GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
Network string `protobuf:"bytes,15,opt,name=network,proto3" json:"network,omitempty"`
|
||||
GidPrefix string `protobuf:"bytes,16,opt,name=gid_prefix,json=gidPrefix,proto3" json:"gid_prefix,omitempty"`
|
||||
// subnet_managers is every subnet manager the fabric's subnet
|
||||
// administrator lists — master and standbys — asked through one port per
|
||||
// fabric, so it is on that port only. It is what keeps a fabric one fabric
|
||||
// through a failover: the master changes, the list does not.
|
||||
SubnetManagers []*SubnetManager `protobuf:"bytes,17,rep,name=subnet_managers,json=subnetManagers,proto3" json:"subnet_managers,omitempty"`
|
||||
SubnetManagersError string `protobuf:"bytes,18,opt,name=subnet_managers_error,json=subnetManagersError,proto3" json:"subnet_managers_error,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *IBPort) Reset() {
|
||||
@@ -1563,6 +1569,90 @@ func (x *IBPort) GetGidPrefix() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSubnetManagers() []*SubnetManager {
|
||||
if x != nil {
|
||||
return x.SubnetManagers
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *IBPort) GetSubnetManagersError() string {
|
||||
if x != nil {
|
||||
return x.SubnetManagersError
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SubnetManager is one subnet manager on a fabric.
|
||||
type SubnetManager struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Guid string `protobuf:"bytes,1,opt,name=guid,proto3" json:"guid,omitempty"`
|
||||
Lid int32 `protobuf:"varint,2,opt,name=lid,proto3" json:"lid,omitempty"`
|
||||
Priority int32 `protobuf:"varint,3,opt,name=priority,proto3" json:"priority,omitempty"`
|
||||
// state is master, standby, discovering or not active.
|
||||
State string `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *SubnetManager) Reset() {
|
||||
*x = SubnetManager{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *SubnetManager) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*SubnetManager) ProtoMessage() {}
|
||||
|
||||
func (x *SubnetManager) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use SubnetManager.ProtoReflect.Descriptor instead.
|
||||
func (*SubnetManager) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
|
||||
}
|
||||
|
||||
func (x *SubnetManager) GetGuid() string {
|
||||
if x != nil {
|
||||
return x.Guid
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *SubnetManager) GetLid() int32 {
|
||||
if x != nil {
|
||||
return x.Lid
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *SubnetManager) GetPriority() int32 {
|
||||
if x != nil {
|
||||
return x.Priority
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *SubnetManager) GetState() string {
|
||||
if x != nil {
|
||||
return x.State
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"`
|
||||
@@ -1618,7 +1708,7 @@ type Heartbeat struct {
|
||||
|
||||
func (x *Heartbeat) Reset() {
|
||||
*x = Heartbeat{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1630,7 +1720,7 @@ func (x *Heartbeat) String() string {
|
||||
func (*Heartbeat) ProtoMessage() {}
|
||||
|
||||
func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[12]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1643,7 +1733,7 @@ func (x *Heartbeat) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead.
|
||||
func (*Heartbeat) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{12}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
}
|
||||
|
||||
func (x *Heartbeat) GetNodeId() string {
|
||||
@@ -1770,7 +1860,7 @@ type Provision struct {
|
||||
|
||||
func (x *Provision) Reset() {
|
||||
*x = Provision{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1782,7 +1872,7 @@ func (x *Provision) String() string {
|
||||
func (*Provision) ProtoMessage() {}
|
||||
|
||||
func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[13]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1795,7 +1885,7 @@ func (x *Provision) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Provision.ProtoReflect.Descriptor instead.
|
||||
func (*Provision) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{13}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
}
|
||||
|
||||
func (x *Provision) GetEngines() []*EngineWant {
|
||||
@@ -1816,7 +1906,7 @@ type EngineWant struct {
|
||||
|
||||
func (x *EngineWant) Reset() {
|
||||
*x = EngineWant{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1828,7 +1918,7 @@ func (x *EngineWant) String() string {
|
||||
func (*EngineWant) ProtoMessage() {}
|
||||
|
||||
func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[14]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1841,7 +1931,7 @@ func (x *EngineWant) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineWant.ProtoReflect.Descriptor instead.
|
||||
func (*EngineWant) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{14}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
}
|
||||
|
||||
func (x *EngineWant) GetEngine() string {
|
||||
@@ -1874,7 +1964,7 @@ type ProvisionStatus struct {
|
||||
|
||||
func (x *ProvisionStatus) Reset() {
|
||||
*x = ProvisionStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1886,7 +1976,7 @@ func (x *ProvisionStatus) String() string {
|
||||
func (*ProvisionStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[15]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1899,7 +1989,7 @@ func (x *ProvisionStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ProvisionStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ProvisionStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{15}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
}
|
||||
|
||||
func (x *ProvisionStatus) GetEngine() string {
|
||||
@@ -1963,7 +2053,7 @@ type ShardStatus struct {
|
||||
|
||||
func (x *ShardStatus) Reset() {
|
||||
*x = ShardStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -1975,7 +2065,7 @@ func (x *ShardStatus) String() string {
|
||||
func (*ShardStatus) ProtoMessage() {}
|
||||
|
||||
func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[16]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -1988,7 +2078,7 @@ func (x *ShardStatus) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use ShardStatus.ProtoReflect.Descriptor instead.
|
||||
func (*ShardStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{16}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
}
|
||||
|
||||
func (x *ShardStatus) GetNodeId() string {
|
||||
@@ -2062,7 +2152,7 @@ type FetchProgress struct {
|
||||
|
||||
func (x *FetchProgress) Reset() {
|
||||
*x = FetchProgress{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2074,7 +2164,7 @@ func (x *FetchProgress) String() string {
|
||||
func (*FetchProgress) ProtoMessage() {}
|
||||
|
||||
func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[17]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2087,7 +2177,7 @@ func (x *FetchProgress) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use FetchProgress.ProtoReflect.Descriptor instead.
|
||||
func (*FetchProgress) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{17}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
}
|
||||
|
||||
func (x *FetchProgress) GetSource() string {
|
||||
@@ -2143,7 +2233,7 @@ type StartShard struct {
|
||||
|
||||
func (x *StartShard) Reset() {
|
||||
*x = StartShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2155,7 +2245,7 @@ func (x *StartShard) String() string {
|
||||
func (*StartShard) ProtoMessage() {}
|
||||
|
||||
func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[18]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2168,7 +2258,7 @@ func (x *StartShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StartShard.ProtoReflect.Descriptor instead.
|
||||
func (*StartShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{18}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
}
|
||||
|
||||
func (x *StartShard) GetReplicaId() string {
|
||||
@@ -2295,7 +2385,7 @@ type EngineProfile struct {
|
||||
|
||||
func (x *EngineProfile) Reset() {
|
||||
*x = EngineProfile{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2307,7 +2397,7 @@ func (x *EngineProfile) String() string {
|
||||
func (*EngineProfile) ProtoMessage() {}
|
||||
|
||||
func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[19]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2320,7 +2410,7 @@ func (x *EngineProfile) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use EngineProfile.ProtoReflect.Descriptor instead.
|
||||
func (*EngineProfile) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{19}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
}
|
||||
|
||||
func (x *EngineProfile) GetName() string {
|
||||
@@ -2416,7 +2506,7 @@ type RemoveWeights struct {
|
||||
|
||||
func (x *RemoveWeights) Reset() {
|
||||
*x = RemoveWeights{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2428,7 +2518,7 @@ func (x *RemoveWeights) String() string {
|
||||
func (*RemoveWeights) ProtoMessage() {}
|
||||
|
||||
func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[20]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2441,7 +2531,7 @@ func (x *RemoveWeights) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use RemoveWeights.ProtoReflect.Descriptor instead.
|
||||
func (*RemoveWeights) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{20}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
}
|
||||
|
||||
func (x *RemoveWeights) GetPaths() []string {
|
||||
@@ -2461,7 +2551,7 @@ type StopShard struct {
|
||||
|
||||
func (x *StopShard) Reset() {
|
||||
*x = StopShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2473,7 +2563,7 @@ func (x *StopShard) String() string {
|
||||
func (*StopShard) ProtoMessage() {}
|
||||
|
||||
func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[21]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2486,7 +2576,7 @@ func (x *StopShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use StopShard.ProtoReflect.Descriptor instead.
|
||||
func (*StopShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{21}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
}
|
||||
|
||||
func (x *StopShard) GetShardId() string {
|
||||
@@ -2506,7 +2596,7 @@ type DrainShard struct {
|
||||
|
||||
func (x *DrainShard) Reset() {
|
||||
*x = DrainShard{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2518,7 +2608,7 @@ func (x *DrainShard) String() string {
|
||||
func (*DrainShard) ProtoMessage() {}
|
||||
|
||||
func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[22]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2531,7 +2621,7 @@ func (x *DrainShard) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use DrainShard.ProtoReflect.Descriptor instead.
|
||||
func (*DrainShard) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{22}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
|
||||
}
|
||||
|
||||
func (x *DrainShard) GetShardId() string {
|
||||
@@ -2553,7 +2643,7 @@ type Ack struct {
|
||||
|
||||
func (x *Ack) Reset() {
|
||||
*x = Ack{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2565,7 +2655,7 @@ func (x *Ack) String() string {
|
||||
func (*Ack) ProtoMessage() {}
|
||||
|
||||
func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[23]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2578,7 +2668,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use Ack.ProtoReflect.Descriptor instead.
|
||||
func (*Ack) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{23}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
|
||||
}
|
||||
|
||||
func (x *Ack) GetMessageId() string {
|
||||
@@ -2622,7 +2712,7 @@ type LogsRequest struct {
|
||||
|
||||
func (x *LogsRequest) Reset() {
|
||||
*x = LogsRequest{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2634,7 +2724,7 @@ func (x *LogsRequest) String() string {
|
||||
func (*LogsRequest) ProtoMessage() {}
|
||||
|
||||
func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[24]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2647,7 +2737,7 @@ func (x *LogsRequest) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsRequest.ProtoReflect.Descriptor instead.
|
||||
func (*LogsRequest) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{24}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
|
||||
}
|
||||
|
||||
func (x *LogsRequest) GetRequestId() string {
|
||||
@@ -2687,7 +2777,7 @@ type LogsReply struct {
|
||||
|
||||
func (x *LogsReply) Reset() {
|
||||
*x = LogsReply{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
@@ -2699,7 +2789,7 @@ func (x *LogsReply) String() string {
|
||||
func (*LogsReply) ProtoMessage() {}
|
||||
|
||||
func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[25]
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[26]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
@@ -2712,7 +2802,7 @@ func (x *LogsReply) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use LogsReply.ProtoReflect.Descriptor instead.
|
||||
func (*LogsReply) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{25}
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{26}
|
||||
}
|
||||
|
||||
func (x *LogsReply) GetRequestId() string {
|
||||
@@ -2868,7 +2958,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\tShardLoad\x12\x19\n" +
|
||||
"\bshard_id\x18\x01 \x01(\tR\ashardId\x12\x1b\n" +
|
||||
"\tcpu_cores\x18\x02 \x01(\x01R\bcpuCores\x12(\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x92\x04\n" +
|
||||
"\x10memory_rss_bytes\x18\x03 \x01(\x03R\x0ememoryRssBytes\"\x88\x05\n" +
|
||||
"\x06IBPort\x12\x10\n" +
|
||||
"\x03hca\x18\x01 \x01(\tR\x03hca\x12\x12\n" +
|
||||
"\x04port\x18\x02 \x01(\x05R\x04port\x12\x14\n" +
|
||||
@@ -2890,10 +2980,17 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\bsm_error\x18\x0e \x01(\tR\asmError\x12\x18\n" +
|
||||
"\anetwork\x18\x0f \x01(\tR\anetwork\x12\x1d\n" +
|
||||
"\n" +
|
||||
"gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x1a;\n" +
|
||||
"gid_prefix\x18\x10 \x01(\tR\tgidPrefix\x12@\n" +
|
||||
"\x0fsubnet_managers\x18\x11 \x03(\v2\x17.llmgw.v1.SubnetManagerR\x0esubnetManagers\x122\n" +
|
||||
"\x15subnet_managers_error\x18\x12 \x01(\tR\x13subnetManagersError\x1a;\n" +
|
||||
"\rCountersEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"\xfc\x05\n" +
|
||||
"\x05value\x18\x02 \x01(\x04R\x05value:\x028\x01\"g\n" +
|
||||
"\rSubnetManager\x12\x12\n" +
|
||||
"\x04guid\x18\x01 \x01(\tR\x04guid\x12\x10\n" +
|
||||
"\x03lid\x18\x02 \x01(\x05R\x03lid\x12\x1a\n" +
|
||||
"\bpriority\x18\x03 \x01(\x05R\bpriority\x12\x14\n" +
|
||||
"\x05state\x18\x04 \x01(\tR\x05state\"\xfc\x05\n" +
|
||||
"\tHeartbeat\x12\x17\n" +
|
||||
"\anode_id\x18\x01 \x01(\tR\x06nodeId\x12+\n" +
|
||||
"\x12gpu_memory_used_mb\x18\x02 \x03(\x05R\x0fgpuMemoryUsedMb\x12!\n" +
|
||||
@@ -3033,7 +3130,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 32)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 33)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -3048,70 +3145,72 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*NodeLoad)(nil), // 10: llmgw.v1.NodeLoad
|
||||
(*ShardLoad)(nil), // 11: llmgw.v1.ShardLoad
|
||||
(*IBPort)(nil), // 12: llmgw.v1.IBPort
|
||||
(*Heartbeat)(nil), // 13: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 14: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 15: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 16: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 17: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 18: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 19: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 20: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 21: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 22: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 23: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 24: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 25: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 26: llmgw.v1.LogsReply
|
||||
nil, // 27: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 28: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 29: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 30: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 31: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 32: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*SubnetManager)(nil), // 13: llmgw.v1.SubnetManager
|
||||
(*Heartbeat)(nil), // 14: llmgw.v1.Heartbeat
|
||||
(*Provision)(nil), // 15: llmgw.v1.Provision
|
||||
(*EngineWant)(nil), // 16: llmgw.v1.EngineWant
|
||||
(*ProvisionStatus)(nil), // 17: llmgw.v1.ProvisionStatus
|
||||
(*ShardStatus)(nil), // 18: llmgw.v1.ShardStatus
|
||||
(*FetchProgress)(nil), // 19: llmgw.v1.FetchProgress
|
||||
(*StartShard)(nil), // 20: llmgw.v1.StartShard
|
||||
(*EngineProfile)(nil), // 21: llmgw.v1.EngineProfile
|
||||
(*RemoveWeights)(nil), // 22: llmgw.v1.RemoveWeights
|
||||
(*StopShard)(nil), // 23: llmgw.v1.StopShard
|
||||
(*DrainShard)(nil), // 24: llmgw.v1.DrainShard
|
||||
(*Ack)(nil), // 25: llmgw.v1.Ack
|
||||
(*LogsRequest)(nil), // 26: llmgw.v1.LogsRequest
|
||||
(*LogsReply)(nil), // 27: llmgw.v1.LogsReply
|
||||
nil, // 28: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 29: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 30: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 31: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 32: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 33: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
3, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
13, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
17, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
26, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
19, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
22, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
23, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
24, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
21, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
14, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
25, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
27, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
14, // 1: llmgw.v1.AgentMessage.heartbeat:type_name -> llmgw.v1.Heartbeat
|
||||
18, // 2: llmgw.v1.AgentMessage.shard:type_name -> llmgw.v1.ShardStatus
|
||||
27, // 3: llmgw.v1.AgentMessage.logs:type_name -> llmgw.v1.LogsReply
|
||||
20, // 4: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
23, // 5: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
24, // 6: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
25, // 7: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
22, // 8: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
15, // 9: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
26, // 10: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
28, // 11: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
7, // 12: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
8, // 13: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
28, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
29, // 14: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
6, // 15: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
9, // 16: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
5, // 17: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
4, // 18: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
0, // 19: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
29, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
30, // 20: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
11, // 21: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
30, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
5, // 23: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 24: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
31, // 25: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
16, // 26: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 27: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 28: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 29: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
15, // 30: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 31: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
18, // 32: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
20, // 33: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
32, // 34: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 35: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 36: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
36, // [36:37] is the sub-list for method output_type
|
||||
35, // [35:36] is the sub-list for method input_type
|
||||
35, // [35:35] is the sub-list for extension type_name
|
||||
35, // [35:35] is the sub-list for extension extendee
|
||||
0, // [0:35] is the sub-list for field type_name
|
||||
31, // 22: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
13, // 23: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
5, // 24: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 25: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
32, // 26: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
17, // 27: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
10, // 28: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
12, // 29: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
8, // 30: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
16, // 31: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
0, // 32: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
19, // 33: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
21, // 34: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
33, // 35: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
1, // 36: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
2, // 37: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
37, // [37:38] is the sub-list for method output_type
|
||||
36, // [36:37] is the sub-list for method input_type
|
||||
36, // [36:36] is the sub-list for extension type_name
|
||||
36, // [36:36] is the sub-list for extension extendee
|
||||
0, // [0:36] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -3140,7 +3239,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 32,
|
||||
NumMessages: 33,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -296,6 +296,21 @@ message IBPort {
|
||||
// subnet manager, and the network is what its fabric is.
|
||||
string network = 15;
|
||||
string gid_prefix = 16;
|
||||
// subnet_managers is every subnet manager the fabric's subnet
|
||||
// administrator lists — master and standbys — asked through one port per
|
||||
// fabric, so it is on that port only. It is what keeps a fabric one fabric
|
||||
// through a failover: the master changes, the list does not.
|
||||
repeated SubnetManager subnet_managers = 17;
|
||||
string subnet_managers_error = 18;
|
||||
}
|
||||
|
||||
// SubnetManager is one subnet manager on a fabric.
|
||||
message SubnetManager {
|
||||
string guid = 1;
|
||||
int32 lid = 2;
|
||||
int32 priority = 3;
|
||||
// state is master, standby, discovering or not active.
|
||||
string state = 4;
|
||||
}
|
||||
|
||||
message Heartbeat {
|
||||
|
||||
+41
-2
@@ -19,6 +19,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -364,6 +365,10 @@ type Agent struct {
|
||||
sm map[string]*discovery.SMInfo
|
||||
smErr map[string]string
|
||||
smAsked time.Time
|
||||
// sms is every subnet manager on a fabric, asked through one port per
|
||||
// master, keyed by that port; smsErr says why a port could not ask.
|
||||
sms map[string][]discovery.SMRecord
|
||||
smsErr map[string]string
|
||||
// load measures the machine between heartbeats.
|
||||
load *discovery.LoadSampler
|
||||
|
||||
@@ -663,13 +668,18 @@ func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
|
||||
RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
|
||||
NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
|
||||
}
|
||||
a.mu.Lock()
|
||||
if p.SM != nil {
|
||||
ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
|
||||
} else {
|
||||
a.mu.Lock()
|
||||
ip.SmError = a.smErr[smKey(h.Name, p.Num)]
|
||||
a.mu.Unlock()
|
||||
}
|
||||
for _, r := range a.sms[smKey(h.Name, p.Num)] {
|
||||
ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{
|
||||
Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State})
|
||||
}
|
||||
ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)]
|
||||
a.mu.Unlock()
|
||||
hb.IbPorts = append(hb.IbPorts, ip)
|
||||
}
|
||||
}
|
||||
@@ -1629,6 +1639,17 @@ func ResolveManagement(c Config) (addr, iface string) {
|
||||
|
||||
func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
|
||||
|
||||
// querySMs is discovery.QuerySMs, replaced in tests.
|
||||
var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
|
||||
return discovery.QuerySMs(context.Background(), hca, port)
|
||||
}
|
||||
|
||||
func splitSMKey(k string) (string, int) {
|
||||
i := strings.LastIndexByte(k, '/')
|
||||
port, _ := strconv.Atoi(k[i+1:])
|
||||
return k[:i], port
|
||||
}
|
||||
|
||||
// querySM is discovery.QuerySM, replaced in tests.
|
||||
var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
|
||||
return discovery.QuerySM(context.Background(), hca, port)
|
||||
@@ -1651,6 +1672,23 @@ func (a *Agent) attachSM(hcas []discovery.HCA) {
|
||||
sm[smKey(hca, port)] = got
|
||||
return got, nil
|
||||
})
|
||||
// Every subnet manager on each fabric, through one port per master:
|
||||
// the same list through every port of one fabric is one query too
|
||||
// many for each.
|
||||
sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{}
|
||||
asked := map[string]bool{}
|
||||
for k, v := range sm {
|
||||
if asked[v.GUID] {
|
||||
continue
|
||||
}
|
||||
asked[v.GUID] = true
|
||||
hca, port := splitSMKey(k)
|
||||
if recs, err := querySMs(hca, port); err != nil {
|
||||
smsErr[k] = err.Error()
|
||||
} else {
|
||||
sms[k] = recs
|
||||
}
|
||||
}
|
||||
a.mu.Lock()
|
||||
for k, v := range sm {
|
||||
if old := a.sm[k]; old != nil && old.GUID != v.GUID {
|
||||
@@ -1658,6 +1696,7 @@ func (a *Agent) attachSM(hcas []discovery.HCA) {
|
||||
}
|
||||
}
|
||||
a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
|
||||
a.sms, a.smsErr = sms, smsErr
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -828,3 +828,36 @@ func TestTheManagementInterfaceIsFoundNotAssumed(t *testing.T) {
|
||||
t.Errorf("interface = %q, want the loopback interface holding 127.0.0.1", cfg.MgmtInterface)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachFabricsSubnetManagersAreAskedOnce(t *testing.T) {
|
||||
// Two adapters on one fabric see one master. Its managers are listed
|
||||
// through one of them — the same list twice is a query too many — and
|
||||
// every port says which master it sees.
|
||||
oldSM, oldSMs := querySM, querySMs
|
||||
t.Cleanup(func() { querySM, querySMs = oldSM, oldSMs })
|
||||
querySM = func(string, int) (*discovery.SMInfo, error) {
|
||||
return &discovery.SMInfo{GUID: "0x01", LID: 1, State: "SMINFO_MASTER"}, nil
|
||||
}
|
||||
lists := 0
|
||||
querySMs = func(string, int) ([]discovery.SMRecord, error) {
|
||||
lists++
|
||||
return []discovery.SMRecord{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}}, nil
|
||||
}
|
||||
h := start(t, func(c *Config) {
|
||||
c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1")
|
||||
c.Fabrics = []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}}
|
||||
})
|
||||
hb := h.ag.heartbeat().GetHeartbeat()
|
||||
withList := 0
|
||||
for _, p := range hb.GetIbPorts() {
|
||||
if p.GetSmGuid() != "0x01" {
|
||||
t.Errorf("%s sees master %q", p.GetHca(), p.GetSmGuid())
|
||||
}
|
||||
if len(p.GetSubnetManagers()) > 0 {
|
||||
withList++
|
||||
}
|
||||
}
|
||||
if lists != 1 || withList != 1 {
|
||||
t.Errorf("listed %d times, carried on %d ports; want once, on one", lists, withList)
|
||||
}
|
||||
}
|
||||
@@ -323,3 +323,31 @@ func TestARoCEPortsNetworkIsItsFabric(t *testing.T) {
|
||||
t.Errorf("RoCE network %q key %q", roce.Network, roce.FabricKey())
|
||||
}
|
||||
}
|
||||
|
||||
// smirDump is `saquery SMIR` on a fabric with a master and a standby, in the
|
||||
// layout infiniband-diags prints.
|
||||
const smirDump = "SMInfoRecord dump:\n\t\tRID\n\t\tLID...................1\n\t\tSMInfo dump:\n" +
|
||||
"\t\tGUID..................0x248a07030030f0c0\n\t\tSM_Key................0x0000000000000000\n" +
|
||||
"\t\tActCount..............386773\n\t\tPriority..............15\n\t\tSMState...............3\n" +
|
||||
"SMInfoRecord dump:\n\t\tRID\n\t\tLID...................12\n\t\tSMInfo dump:\n" +
|
||||
"\t\tGUID..................0x248a07030030f1d4\n\t\tSM_Key................0x0000000000000000\n" +
|
||||
"\t\tActCount..............12\n\t\tPriority..............14\n\t\tSMState...............2\n"
|
||||
|
||||
func TestEverySubnetManagerOnAFabricIsListed(t *testing.T) {
|
||||
recs, err := ParseSMIR(smirDump)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(recs) != 2 {
|
||||
t.Fatalf("records = %+v", recs)
|
||||
}
|
||||
if recs[0].GUID != "0x248a07030030f0c0" || recs[0].State != "master" || recs[0].Priority != 15 || recs[0].LID != 1 {
|
||||
t.Errorf("master = %+v", recs[0])
|
||||
}
|
||||
if recs[1].State != "standby" || recs[1].LID != 12 {
|
||||
t.Errorf("standby = %+v", recs[1])
|
||||
}
|
||||
if _, err := ParseSMIR("ibwarn: can't open UMAD port"); err == nil {
|
||||
t.Error("an error should not parse")
|
||||
}
|
||||
}
|
||||
@@ -71,3 +71,76 @@ func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) err
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
// SMRecord is one subnet manager on a fabric, as the subnet administrator
|
||||
// lists them: master and standbys alike.
|
||||
type SMRecord struct {
|
||||
GUID string `json:"guid"`
|
||||
LID int `json:"lid"`
|
||||
Priority int `json:"priority"`
|
||||
// State is master, standby, discovering or not active.
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// smStates are SMInfo's state numbers.
|
||||
var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"}
|
||||
|
||||
// smirField is one line of saquery's record dump: "LID.........1".
|
||||
var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`)
|
||||
|
||||
// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet
|
||||
// administrator knows of, one "SMInfoRecord dump:" block each.
|
||||
func ParseSMIR(out string) ([]SMRecord, error) {
|
||||
var recs []SMRecord
|
||||
var cur *SMRecord
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
if strings.Contains(line, "SMInfoRecord dump") {
|
||||
recs = append(recs, SMRecord{})
|
||||
cur = &recs[len(recs)-1]
|
||||
continue
|
||||
}
|
||||
m := smirField.FindStringSubmatch(line)
|
||||
if m == nil || cur == nil {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(m[1]) {
|
||||
case "lid":
|
||||
cur.LID, _ = strconv.Atoi(m[2])
|
||||
case "guid":
|
||||
cur.GUID = normalizeGUID(m[2])
|
||||
case "priority":
|
||||
cur.Priority, _ = strconv.Atoi(m[2])
|
||||
case "smstate":
|
||||
cur.State = smStates[m[2]]
|
||||
if cur.State == "" {
|
||||
cur.State = m[2]
|
||||
}
|
||||
}
|
||||
}
|
||||
out2 := recs[:0]
|
||||
for _, r := range recs {
|
||||
if r.GUID != "" {
|
||||
out2 = append(out2, r)
|
||||
}
|
||||
}
|
||||
if len(out2) == 0 {
|
||||
return nil, fmt.Errorf("no subnet manager records in saquery output")
|
||||
}
|
||||
return out2, nil
|
||||
}
|
||||
|
||||
// QuerySMs asks the subnet administrator, through one port, for every subnet
|
||||
// manager on the fabric.
|
||||
func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
|
||||
bin, err := exec.LookPath("saquery")
|
||||
if err != nil {
|
||||
return nil, ErrNoDiags
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return ParseSMIR(string(out))
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
@@ -20,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -836,5 +836,6 @@ func (a *API) profileChanged(model, profile, was string) bool {
|
||||
// Fabrics is the physical fabrics the adapters are on, against the names
|
||||
// the nodes gave them.
|
||||
func (a *API) Fabrics() fabrics.Report {
|
||||
return fabrics.Analyze(a.ctl.Inventory().Nodes())
|
||||
inv := a.ctl.Inventory()
|
||||
return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now())
|
||||
}
|
||||
@@ -405,9 +405,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
n.Load = loadFrom(l, time.Now())
|
||||
}
|
||||
if ports := hb.GetIbPorts(); len(ports) > 0 {
|
||||
n.IBPorts = n.IBPorts[:0]
|
||||
before := map[string]string{}
|
||||
for _, p := range n.IBPorts {
|
||||
before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID
|
||||
}
|
||||
n.IBPorts = nil
|
||||
for _, p := range ports {
|
||||
// A different master than last time is a failover, or a port
|
||||
// moved: the fabric's own event, kept for the page to show.
|
||||
where := fmt.Sprintf("%s/%d", p.GetHca(), p.GetPort())
|
||||
if old := before[where]; old != "" && p.GetSmGuid() != "" && old != p.GetSmGuid() {
|
||||
s.log.Warn("a port sees a different master subnet manager", "node", nodeID,
|
||||
"port", where, "from", old, "to", p.GetSmGuid())
|
||||
s.inv.RecordSMChange(inventory.SMChange{At: time.Now(), Node: nodeID, Port: where,
|
||||
From: old, To: p.GetSmGuid()})
|
||||
}
|
||||
var sms []inventory.SubnetManager
|
||||
for _, r := range p.GetSubnetManagers() {
|
||||
sms = append(sms, inventory.SubnetManager{GUID: r.GetGuid(), LID: int(r.GetLid()),
|
||||
Priority: int(r.GetPriority()), State: r.GetState()})
|
||||
}
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{
|
||||
SMs: sms, SMsError: p.GetSubnetManagersError(),
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
|
||||
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
@@ -41,8 +42,17 @@ type Physical struct {
|
||||
Kind string `json:"kind"`
|
||||
// Label is the key as a person reads it.
|
||||
Label string `json:"label"`
|
||||
// SM is the master subnet manager, on InfiniBand.
|
||||
// SM is the master subnet manager, on InfiniBand: the one most of its
|
||||
// ports see.
|
||||
SM *SM `json:"sm,omitempty"`
|
||||
// SMs is every subnet manager the fabric lists, master first; empty when
|
||||
// no port could ask. Standbys counts those that could take over.
|
||||
SMs []SM `json:"sms,omitempty"`
|
||||
Standbys int `json:"standbys"`
|
||||
// Masters is how many ports see each master, when they do not all agree.
|
||||
Masters map[string]int `json:"masters,omitempty"`
|
||||
// Changes are the master changes seen in the last day, newest first.
|
||||
Changes []Change `json:"changes,omitempty"`
|
||||
// Declared are the names nodes gave this fabric; one, when all is well.
|
||||
Declared []string `json:"declared"`
|
||||
Members []Member `json:"members"`
|
||||
@@ -58,6 +68,14 @@ type SM struct {
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// Change is a fabric's master subnet manager changing, as its ports saw it.
|
||||
type Change struct {
|
||||
At time.Time `json:"at"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
Ports int `json:"ports"`
|
||||
}
|
||||
|
||||
// Member is one node's adapters on a physical fabric.
|
||||
type Member struct {
|
||||
Node string `json:"node"`
|
||||
@@ -76,9 +94,15 @@ type Report struct {
|
||||
Unverified map[string][]string `json:"unverified,omitempty"`
|
||||
}
|
||||
|
||||
// Analyze reads every node's ports.
|
||||
func Analyze(nodes []inventory.Node) Report {
|
||||
// Analyze reads every node's ports, and the changes of master subnet manager
|
||||
// the control plane has recorded.
|
||||
func Analyze(nodes []inventory.Node, changes []inventory.SMChange, now time.Time) Report {
|
||||
r := Report{Unverified: map[string][]string{}}
|
||||
canon := inventory.CanonicalSMs(nodes)
|
||||
masters := map[string]map[string]int{} // key -> master GUID -> ports
|
||||
smLists := map[string]map[string]inventory.SubnetManager{}
|
||||
listErr := map[string]string{}
|
||||
noSM := map[string][]string{} // declared -> node/port up with no SM
|
||||
byKey := map[string]*Physical{}
|
||||
declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes
|
||||
|
||||
@@ -94,8 +118,24 @@ func Analyze(nodes []inventory.Node) Report {
|
||||
members := map[string]*Member{} // key -> this node's membership
|
||||
|
||||
for _, p := range n.IBPorts {
|
||||
key := p.FabricKey()
|
||||
key := p.FabricKeyIn(canon)
|
||||
decl := declOf[p.HCA]
|
||||
if decl != "" && !p.RoCE() && strings.EqualFold(p.State, "INIT") && strings.EqualFold(p.PhysState, "LinkUp") {
|
||||
noSM[decl] = appendOnce(noSM[decl], fmt.Sprintf("%s %s/%d", n.ID, p.HCA, p.Port))
|
||||
}
|
||||
if key != "" && !p.RoCE() {
|
||||
if masters[key] == nil {
|
||||
masters[key] = map[string]int{}
|
||||
smLists[key] = map[string]inventory.SubnetManager{}
|
||||
}
|
||||
masters[key][p.SMGUID]++
|
||||
for _, sm := range p.SMs {
|
||||
smLists[key][sm.GUID] = sm
|
||||
}
|
||||
if p.SMsError != "" {
|
||||
listErr[key] = p.SMsError
|
||||
}
|
||||
}
|
||||
if key == "" {
|
||||
if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" {
|
||||
r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError)
|
||||
@@ -109,7 +149,7 @@ func Analyze(nodes []inventory.Node) Report {
|
||||
ph.Kind, ph.Label = "roce", "RoCE "+p.Network
|
||||
} else {
|
||||
ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState}
|
||||
ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID)
|
||||
ph.Label = "fabric of subnet manager " + strings.TrimPrefix(key, "sm:")
|
||||
}
|
||||
byKey[key] = ph
|
||||
}
|
||||
@@ -162,7 +202,10 @@ func Analyze(nodes []inventory.Node) Report {
|
||||
"%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name",
|
||||
decl, len(keys), strings.Join(parts, "; "))})
|
||||
}
|
||||
for _, ph := range byKey {
|
||||
for key, ph := range byKey {
|
||||
if ph.Kind == "infiniband" {
|
||||
analyzeSMs(&r, ph, masters[key], smLists[key], listErr[key], nodes, changes, now)
|
||||
}
|
||||
sort.Strings(ph.Declared)
|
||||
sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node })
|
||||
if len(ph.Declared) > 1 {
|
||||
@@ -195,6 +238,11 @@ func Analyze(nodes []inventory.Node) Report {
|
||||
len(labels), prefix)})
|
||||
}
|
||||
}
|
||||
for decl, ports := range noSM {
|
||||
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
|
||||
"%s: %d port(s) have a link but no subnet manager has configured them (INIT): %s",
|
||||
decl, len(ports), strings.Join(ports, ", "))})
|
||||
}
|
||||
for decl, why := range r.Unverified {
|
||||
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf(
|
||||
"%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))})
|
||||
@@ -209,6 +257,94 @@ func Analyze(nodes []inventory.Node) Report {
|
||||
return r
|
||||
}
|
||||
|
||||
// analyzeSMs fills in a fabric's subnet managers and what is wrong with them.
|
||||
func analyzeSMs(r *Report, ph *Physical, masters map[string]int, list map[string]inventory.SubnetManager,
|
||||
listErr string, nodes []inventory.Node, changes []inventory.SMChange, now time.Time,
|
||||
) {
|
||||
name := first(ph.Declared)
|
||||
if name == "~" {
|
||||
name = ph.Label
|
||||
}
|
||||
// The master is the one most ports see.
|
||||
best, most := "", -1
|
||||
for g, n := range masters {
|
||||
if n > most || (n == most && g < best) {
|
||||
best, most = g, n
|
||||
}
|
||||
}
|
||||
if best != "" && ph.SM != nil && ph.SM.GUID != best {
|
||||
ph.SM = &SM{GUID: best}
|
||||
}
|
||||
if len(masters) > 1 {
|
||||
ph.Masters = masters
|
||||
var parts []string
|
||||
for _, g := range sortedKeys(masters) {
|
||||
parts = append(parts, fmt.Sprintf("%s by %d", g, masters[g]))
|
||||
}
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
|
||||
"%s: its ports do not agree on the master subnet manager (%s) — a failover in progress, or a fabric split in two",
|
||||
name, strings.Join(parts, ", "))})
|
||||
}
|
||||
|
||||
for _, sm := range list {
|
||||
v := SM{GUID: sm.GUID, LID: sm.LID, Priority: sm.Priority, State: sm.State}
|
||||
ph.SMs = append(ph.SMs, v)
|
||||
if sm.State == "standby" {
|
||||
ph.Standbys++
|
||||
}
|
||||
if ph.SM != nil && sm.GUID == ph.SM.GUID {
|
||||
ph.SM.LID, ph.SM.Priority, ph.SM.State = sm.LID, sm.Priority, sm.State
|
||||
}
|
||||
}
|
||||
sort.Slice(ph.SMs, func(i, j int) bool {
|
||||
if (ph.SMs[i].State == "master") != (ph.SMs[j].State == "master") {
|
||||
return ph.SMs[i].State == "master"
|
||||
}
|
||||
return ph.SMs[i].Priority > ph.SMs[j].Priority
|
||||
})
|
||||
switch {
|
||||
case len(list) == 0 && listErr != "":
|
||||
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: name, Message: fmt.Sprintf(
|
||||
"%s: its standby subnet managers cannot be listed: %s", name, listErr)})
|
||||
case len(list) > 0 && ph.Standbys == 0:
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
|
||||
"%s has one subnet manager and no standby: if it stops, the fabric keeps its routes but cannot recover from any change — a cable, a switch, a node rebooting",
|
||||
name)})
|
||||
}
|
||||
|
||||
// Its recent failovers: the recorded changes of ports on this fabric,
|
||||
// one per minute of changes rather than one per port.
|
||||
onFabric := map[string]bool{}
|
||||
for _, m := range ph.Members {
|
||||
for _, h := range m.HCAs {
|
||||
onFabric[m.Node+" "+h] = true
|
||||
}
|
||||
}
|
||||
day := now.Add(-24 * time.Hour)
|
||||
for _, c := range changes {
|
||||
hca := c.Port
|
||||
if i := strings.LastIndexByte(hca, '/'); i >= 0 {
|
||||
hca = hca[:i]
|
||||
}
|
||||
if c.At.Before(day) || !onFabric[c.Node+" "+hca] {
|
||||
continue
|
||||
}
|
||||
if n := len(ph.Changes); n > 0 && ph.Changes[n-1].To == c.To && c.At.Sub(ph.Changes[n-1].At) < 2*time.Minute {
|
||||
ph.Changes[n-1].Ports++
|
||||
continue
|
||||
}
|
||||
ph.Changes = append(ph.Changes, Change{At: c.At, From: c.From, To: c.To, Ports: 1})
|
||||
}
|
||||
slices.Reverse(ph.Changes)
|
||||
if len(ph.Changes) > 0 && now.Sub(ph.Changes[0].At) < time.Hour {
|
||||
c := ph.Changes[0]
|
||||
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
|
||||
"%s: the master subnet manager changed %s ago, from %s to %s",
|
||||
name, now.Sub(c.At).Round(time.Minute), c.From, c.To)})
|
||||
}
|
||||
_ = nodes
|
||||
}
|
||||
|
||||
func rank(l Level) int {
|
||||
switch l {
|
||||
case Fail:
|
||||
|
||||
@@ -3,6 +3,7 @@ package fabrics
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
@@ -30,7 +31,7 @@ func problem(t *testing.T, r Report, l Level, fragment string) {
|
||||
}
|
||||
|
||||
func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
r := analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"),
|
||||
ibNode("b1", "ib-b", "0x02", "mlx5_0"),
|
||||
})
|
||||
@@ -50,14 +51,14 @@ func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
r := analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"),
|
||||
})
|
||||
problem(t, r, Fail, "ib-a is 2 separate fabrics")
|
||||
}
|
||||
|
||||
func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
|
||||
r := Analyze([]inventory.Node{
|
||||
r := analyze([]inventory.Node{
|
||||
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"),
|
||||
})
|
||||
problem(t, r, Warn, "ib-a and ib-b are one fabric")
|
||||
@@ -66,26 +67,84 @@ func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
|
||||
func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) {
|
||||
n := ibNode("a1", "ib-a", "0x01", "mlx5_0")
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"})
|
||||
problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09")
|
||||
problem(t, analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on fabric of subnet manager 0x09")
|
||||
}
|
||||
|
||||
func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) {
|
||||
n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1")
|
||||
n.IBPorts[1].SMGUID = "0x02"
|
||||
problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
|
||||
problem(t, analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
|
||||
}
|
||||
|
||||
func TestANodeThatCannotSayIsReported(t *testing.T) {
|
||||
n := ibNode("x", "ib-a", "", "mlx5_0")
|
||||
n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags"
|
||||
problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags")
|
||||
problem(t, analyze([]inventory.Node{n}), Info, "install infiniband-diags")
|
||||
}
|
||||
|
||||
func TestRoCEIsItsNetwork(t *testing.T) {
|
||||
n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}},
|
||||
IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}}
|
||||
r := Analyze([]inventory.Node{n})
|
||||
r := analyze([]inventory.Node{n})
|
||||
if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" {
|
||||
t.Errorf("physical = %+v", r.Physical)
|
||||
}
|
||||
}
|
||||
|
||||
func analyze(nodes []inventory.Node) Report { return Analyze(nodes, nil, time.Now()) }
|
||||
|
||||
func withSMs(n inventory.Node, sms ...inventory.SubnetManager) inventory.Node {
|
||||
n.IBPorts[0].SMs = sms
|
||||
return n
|
||||
}
|
||||
|
||||
func TestAFailoverKeepsAFabricOneFabric(t *testing.T) {
|
||||
// Mid-failover some ports see the old master, some the new. Keyed by the
|
||||
// master alone, one fabric read as two for a minute — and a declared name
|
||||
// covering it as "two separate fabrics". The list of managers is the same
|
||||
// before and after, which is what keeps it one.
|
||||
sms := []inventory.SubnetManager{{GUID: "0x01", State: "master", Priority: 15}, {GUID: "0x02", State: "standby", Priority: 14}}
|
||||
r := analyze([]inventory.Node{
|
||||
withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), sms...),
|
||||
ibNode("a2", "ib-a", "0x02", "mlx5_0"),
|
||||
})
|
||||
if len(r.Physical) != 1 {
|
||||
t.Fatalf("one fabric read as %d: %+v", len(r.Physical), r.Physical)
|
||||
}
|
||||
for _, p := range r.Problems {
|
||||
if p.Level == Fail {
|
||||
t.Errorf("a failover was reported as a failure: %s", p.Message)
|
||||
}
|
||||
}
|
||||
problem(t, r, Warn, "do not agree on the master subnet manager")
|
||||
if r.Physical[0].Standbys != 1 || len(r.Physical[0].SMs) != 2 || r.Physical[0].SMs[0].State != "master" {
|
||||
t.Errorf("managers = %+v", r.Physical[0].SMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFabricWithNoStandbyIsSaid(t *testing.T) {
|
||||
r := analyze([]inventory.Node{withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"),
|
||||
inventory.SubnetManager{GUID: "0x01", State: "master"})})
|
||||
problem(t, r, Warn, "one subnet manager and no standby")
|
||||
}
|
||||
|
||||
func TestPortsUpWithoutASubnetManagerAreAFailure(t *testing.T) {
|
||||
n := ibNode("a1", "ib-a", "", "mlx5_0")
|
||||
n.IBPorts[0].State, n.IBPorts[0].PhysState = "INIT", "LinkUp"
|
||||
problem(t, analyze([]inventory.Node{n}), Fail, "no subnet manager has configured them")
|
||||
}
|
||||
|
||||
func TestARecentFailoverIsShown(t *testing.T) {
|
||||
now := time.Now()
|
||||
nodes := []inventory.Node{withSMs(ibNode("a1", "ib-a", "0x02", "mlx5_0"),
|
||||
inventory.SubnetManager{GUID: "0x01", State: "standby"}, inventory.SubnetManager{GUID: "0x02", State: "master"})}
|
||||
changes := []inventory.SMChange{
|
||||
{At: now.Add(-20 * time.Minute), Node: "a1", Port: "mlx5_0/1", From: "0x01", To: "0x02"},
|
||||
{At: now.Add(-48 * time.Hour), Node: "a1", Port: "mlx5_0/1", From: "0x02", To: "0x01"},
|
||||
}
|
||||
r := Analyze(nodes, changes, now)
|
||||
if got := r.Physical[0].Changes; len(got) != 1 || got[0].To != "0x02" {
|
||||
t.Errorf("changes in the last day = %+v", got)
|
||||
}
|
||||
problem(t, r, Warn, "the master subnet manager changed 20m0s ago")
|
||||
}
|
||||
@@ -16,10 +16,12 @@ package fleetmetrics
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
@@ -71,6 +73,10 @@ type Collector struct {
|
||||
ibActive *prometheus.Desc
|
||||
ibRate *prometheus.Desc
|
||||
ibCounter *prometheus.Desc
|
||||
|
||||
fabricSMs *prometheus.Desc
|
||||
fabricMaster *prometheus.Desc
|
||||
fabricProblems *prometheus.Desc
|
||||
}
|
||||
|
||||
// loadFresh is how old a node's load may be and still be reported. A node
|
||||
@@ -160,6 +166,15 @@ func New(src Source) *Collector {
|
||||
ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps",
|
||||
"A port's negotiated rate. A link that trained below its usual speed is a "+
|
||||
"cable or transceiver to look at.", []string{"node", "hca", "port"}, nil),
|
||||
fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers",
|
||||
"Subnet managers a physical fabric lists, by state. No standby means the fabric "+
|
||||
"cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil),
|
||||
fabricMaster: prometheus.NewDesc("llmgw_fabric_master_sm",
|
||||
"1 for a physical fabric's master subnet manager, as most of its ports see it; a "+
|
||||
"change of sm_guid is a failover.", []string{"fabric", "physical", "sm_guid"}, nil),
|
||||
fabricProblems: prometheus.NewDesc("llmgw_fabric_problems",
|
||||
"Disagreements about the fabrics, by level: fail means multi-node work will not "+
|
||||
"place or will not run where it should.", []string{"level"}, nil),
|
||||
ibCounter: prometheus.NewDesc("llmgw_ib_port_counter_total",
|
||||
"A port's cumulative counters, by sysfs name: symbol_error, link_downed and the "+
|
||||
"other errors say a link is going bad before a job fails on it; port_xmit_data "+
|
||||
@@ -173,7 +188,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
||||
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
|
||||
c.nodeDiskFree, c.nodeDiskSize,
|
||||
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
|
||||
c.ibActive, c.ibRate, c.ibCounter,
|
||||
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
|
||||
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
|
||||
c.gpuNVLink, c.gpuECC,
|
||||
c.replicaState, c.replicaReady, c.replicaIdle,
|
||||
@@ -285,6 +300,30 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
|
||||
// The fabrics as the adapters find them: their subnet managers, and how
|
||||
// many things disagree.
|
||||
rep := fabrics.Analyze(c.src.Nodes(), nil, now)
|
||||
for _, ph := range rep.Physical {
|
||||
name := strings.Join(ph.Declared, ",")
|
||||
if ph.SM != nil && ph.SM.GUID != "" {
|
||||
ch <- prometheus.MustNewConstMetric(c.fabricMaster, prometheus.GaugeValue, 1, name, ph.Key, ph.SM.GUID)
|
||||
}
|
||||
byState := map[string]int{}
|
||||
for _, sm := range ph.SMs {
|
||||
byState[sm.State]++
|
||||
}
|
||||
for st, n := range byState {
|
||||
ch <- prometheus.MustNewConstMetric(c.fabricSMs, prometheus.GaugeValue, float64(n), name, ph.Key, st)
|
||||
}
|
||||
}
|
||||
levels := map[fabrics.Level]int{fabrics.Fail: 0, fabrics.Warn: 0, fabrics.Info: 0}
|
||||
for _, p := range rep.Problems {
|
||||
levels[p.Level]++
|
||||
}
|
||||
for l, n := range levels {
|
||||
ch <- prometheus.MustNewConstMetric(c.fabricProblems, prometheus.GaugeValue, float64(n), string(l))
|
||||
}
|
||||
|
||||
// Replicas by model and state, so a model stuck starting is visible as a
|
||||
// shape on a graph rather than only in a log line nobody tails.
|
||||
counts := map[[2]string]int{}
|
||||
|
||||
@@ -142,6 +142,75 @@ type IBPort struct {
|
||||
SMError string
|
||||
// Network is a RoCE port's IP network.
|
||||
Network string
|
||||
// SMs is every subnet manager on the port's fabric, master and standbys,
|
||||
// on the one port per fabric that asked; SMsError says why it could not.
|
||||
SMs []SubnetManager
|
||||
SMsError string
|
||||
}
|
||||
|
||||
// SubnetManager is one subnet manager on a fabric.
|
||||
type SubnetManager struct {
|
||||
GUID string
|
||||
LID int
|
||||
Priority int
|
||||
State string
|
||||
}
|
||||
|
||||
// CanonicalSMs maps every subnet manager GUID the cluster has heard of to
|
||||
// one name for its fabric: the lowest GUID among the managers listed
|
||||
// together. A fabric's master changes on a failover and its list does not,
|
||||
// so ports that see the old master and ports that already see the new one
|
||||
// stay one fabric through it.
|
||||
func CanonicalSMs(nodes []Node) map[string]string {
|
||||
parent := map[string]string{}
|
||||
var find func(string) string
|
||||
find = func(x string) string {
|
||||
p, ok := parent[x]
|
||||
if !ok || p == x {
|
||||
parent[x] = x
|
||||
return x
|
||||
}
|
||||
root := find(p)
|
||||
parent[x] = root
|
||||
return root
|
||||
}
|
||||
union := func(a, b string) {
|
||||
ra, rb := find(a), find(b)
|
||||
if ra == rb {
|
||||
return
|
||||
}
|
||||
if ra < rb {
|
||||
parent[rb] = ra
|
||||
} else {
|
||||
parent[ra] = rb
|
||||
}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.SMGUID != "" {
|
||||
find(p.SMGUID)
|
||||
}
|
||||
for _, sm := range p.SMs {
|
||||
union(sm.GUID, p.SMGUID)
|
||||
if p.SMGUID == "" {
|
||||
find(sm.GUID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out := make(map[string]string, len(parent))
|
||||
for g := range parent {
|
||||
out[g] = find(g)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// SMChange is a port seeing a different master subnet manager than before.
|
||||
type SMChange struct {
|
||||
At time.Time `json:"at"`
|
||||
Node string `json:"node"`
|
||||
Port string `json:"port"`
|
||||
From, To string
|
||||
}
|
||||
|
||||
// Active reports whether the port can carry traffic.
|
||||
@@ -153,7 +222,11 @@ func (p IBPort) RoCE() bool { return strings.EqualFold(p.LinkLayer, "Ethernet")
|
||||
// FabricKey names the physical fabric the port is on: its master subnet
|
||||
// manager on InfiniBand, its IP network on RoCE. Empty when not known, and
|
||||
// for a port that is not up.
|
||||
func (p IBPort) FabricKey() string {
|
||||
func (p IBPort) FabricKey() string { return p.FabricKeyIn(nil) }
|
||||
|
||||
// FabricKeyIn is FabricKey with the subnet manager named as canon has it,
|
||||
// so a fabric keeps one key through a failover.
|
||||
func (p IBPort) FabricKeyIn(canon map[string]string) string {
|
||||
if !p.Active() {
|
||||
return ""
|
||||
}
|
||||
@@ -161,6 +234,9 @@ func (p IBPort) FabricKey() string {
|
||||
case p.RoCE() && p.Network != "":
|
||||
return "roce:" + p.Network
|
||||
case !p.RoCE() && p.SMGUID != "":
|
||||
if c, ok := canon[p.SMGUID]; ok {
|
||||
return "sm:" + c
|
||||
}
|
||||
return "sm:" + p.SMGUID
|
||||
}
|
||||
return ""
|
||||
@@ -169,13 +245,13 @@ func (p IBPort) FabricKey() string {
|
||||
// PhysicalFabric is the physical fabric this node's adapters for a declared
|
||||
// fabric are on. Known is false when none of them says; ok is false when
|
||||
// they disagree — two adapters declared as one fabric that are cabled to two.
|
||||
func (n Node) PhysicalFabric(fabric string) (key string, known, ok bool) {
|
||||
func (n Node) PhysicalFabric(fabric string, canon map[string]string) (key string, known, ok bool) {
|
||||
hcas := n.HCAsFor(fabric)
|
||||
for _, p := range n.IBPorts {
|
||||
if !slices.Contains(hcas, p.HCA) {
|
||||
continue
|
||||
}
|
||||
k := p.FabricKey()
|
||||
k := p.FabricKeyIn(canon)
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
@@ -279,6 +355,8 @@ type Store struct {
|
||||
nodes map[string]*Node
|
||||
// allocs maps node id to the shards placed there.
|
||||
allocs map[string][]Allocation
|
||||
// smChanges are ports' changes of master subnet manager, newest last.
|
||||
smChanges []SMChange
|
||||
}
|
||||
|
||||
// NewStore returns an empty inventory.
|
||||
@@ -288,6 +366,25 @@ func NewStore() *Store {
|
||||
|
||||
// Upsert records a node's registration, preserving operator-set state such as
|
||||
// a cordon: an agent reconnecting must not silently undo an operator's decision.
|
||||
// RecordSMChange keeps a port's change of master subnet manager, the last
|
||||
// few hundred of them: a failover is the fabric's own event, and nothing
|
||||
// else would remember it once the ports agree again.
|
||||
func (s *Store) RecordSMChange(c SMChange) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.smChanges = append(s.smChanges, c)
|
||||
if len(s.smChanges) > 500 {
|
||||
s.smChanges = slices.Clone(s.smChanges[len(s.smChanges)-500:])
|
||||
}
|
||||
}
|
||||
|
||||
// SMChanges returns the recorded changes, oldest first.
|
||||
func (s *Store) SMChanges() []SMChange {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return slices.Clone(s.smChanges)
|
||||
}
|
||||
|
||||
func (s *Store) Upsert(n Node) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
@@ -246,9 +246,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
|
||||
}}
|
||||
}
|
||||
|
||||
canon := inventory.CanonicalSMs(s.inv.Nodes())
|
||||
for _, fabric := range fabrics {
|
||||
all := s.eligibleNodes(p, fabric, avoid, &rejections)
|
||||
for _, group := range physicalGroups(fabric, all, p, &rejections) {
|
||||
for _, group := range physicalGroups(fabric, all, p, canon, &rejections) {
|
||||
pool := fabric + "/" + p.classLabel()
|
||||
if group.key != "" && group.split {
|
||||
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
|
||||
@@ -278,10 +279,10 @@ type fabricGroup struct {
|
||||
// subnet manager each node's ports see (by IP network on RoCE), a replica
|
||||
// stays on one. Nodes that do not say — an agent without infiniband-diags —
|
||||
// are a group of their own rather than trusted to join one that does.
|
||||
func physicalGroups(fabric string, cands []candidate, p Profile, rejections *[]Rejection) []fabricGroup {
|
||||
func physicalGroups(fabric string, cands []candidate, p Profile, canon map[string]string, rejections *[]Rejection) []fabricGroup {
|
||||
by := map[string][]candidate{}
|
||||
for _, c := range cands {
|
||||
key, _, ok := c.node.PhysicalFabric(fabric)
|
||||
key, _, ok := c.node.PhysicalFabric(fabric, canon)
|
||||
if !ok {
|
||||
*rejections = append(*rejections, Rejection{
|
||||
Profile: p.Name, Pool: fabric + "/" + p.classLabel(), Node: c.node.ID,
|
||||
|
||||
@@ -958,3 +958,15 @@ func TestAdaptersOfOneDeclaredFabricOnTwoAreRefused(t *testing.T) {
|
||||
t.Errorf("expected a refusal naming the cabling:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) {
|
||||
// One port already sees the new master, the other the old; the managers
|
||||
// listed are the same, so the two nodes are still one fabric.
|
||||
a := onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01")
|
||||
a.IBPorts[0].SMs = []inventory.SubnetManager{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}}
|
||||
b := onSM(node("a2", "h100-80g", 8, "ib-a"), "0x02")
|
||||
res := New(store(a, b)).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("a failover split the fabric:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
@@ -62,7 +62,11 @@ type Facts struct {
|
||||
HCAs []discovery.HCA `json:"hcas"`
|
||||
// SMErrors say, per port, why the subnet manager could not be asked.
|
||||
SMErrors map[string]string `json:"sm_errors,omitempty"`
|
||||
Modules map[string]bool `json:"modules"`
|
||||
// SMLists are every subnet manager on each fabric, by master GUID;
|
||||
// SMListErrors why a fabric's could not be listed.
|
||||
SMLists map[string][]discovery.SMRecord `json:"sm_lists,omitempty"`
|
||||
SMListErrors map[string]string `json:"sm_list_errors,omitempty"`
|
||||
Modules map[string]bool `json:"modules"`
|
||||
|
||||
// ConfigPath is the agent configuration read, empty when there was none.
|
||||
ConfigPath string `json:"config_path,omitempty"`
|
||||
@@ -136,6 +140,25 @@ func Collect(ctx context.Context, o Options) Facts {
|
||||
}
|
||||
return sm, err
|
||||
})
|
||||
f.SMLists, f.SMListErrors = map[string][]discovery.SMRecord{}, map[string]string{}
|
||||
for _, h := range f.HCAs {
|
||||
for _, p := range h.Ports {
|
||||
if p.SM == nil {
|
||||
continue
|
||||
}
|
||||
if _, done := f.SMLists[p.SM.GUID]; done {
|
||||
continue
|
||||
}
|
||||
if _, done := f.SMListErrors[p.SM.GUID]; done {
|
||||
continue
|
||||
}
|
||||
if recs, err := discovery.QuerySMs(ctx, h.Name, p.Num); err != nil {
|
||||
f.SMListErrors[p.SM.GUID] = err.Error()
|
||||
} else {
|
||||
f.SMLists[p.SM.GUID] = recs
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range watchedModules {
|
||||
_, err := os.Stat(filepath.Join("/sys/module", m))
|
||||
f.Modules[m] = err == nil
|
||||
@@ -340,6 +363,26 @@ func Evaluate(f Facts) []Finding {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, master := range sortedKeys(f.SMLists) {
|
||||
recs := f.SMLists[master]
|
||||
var parts []string
|
||||
standby := 0
|
||||
for _, r := range recs {
|
||||
parts = append(parts, fmt.Sprintf("%s (%s, priority %d)", r.GUID, r.State, r.Priority))
|
||||
if r.State == "standby" {
|
||||
standby++
|
||||
}
|
||||
}
|
||||
if standby == 0 {
|
||||
add(Warn, "fabric", "the fabric of subnet manager "+master+" has no standby: "+strings.Join(parts, ", "),
|
||||
"run a second subnet manager, on a switch or a host, at a lower priority")
|
||||
} else {
|
||||
add(OK, "fabric", "subnet managers of "+master+"'s fabric: "+strings.Join(parts, ", "), "")
|
||||
}
|
||||
}
|
||||
for _, master := range sortedKeys(f.SMListErrors) {
|
||||
add(Info, "fabric", "the subnet managers of "+master+"'s fabric could not be listed: "+f.SMListErrors[master], "")
|
||||
}
|
||||
if len(f.HCAs) > 0 {
|
||||
if !f.Modules["ib_uverbs"] {
|
||||
add(Fail, "fabric", "ib_uverbs is not loaded: nothing in user space can use RDMA", "modprobe ib_uverbs")
|
||||
@@ -478,3 +521,12 @@ func orUnknown(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -27,6 +27,8 @@ func healthy() Facts {
|
||||
PhysState: "LinkUp", RateGbps: 400, LinkLayer: discovery.LinkInfiniBand, SMLID: 1, RoCEv2GID: -1,
|
||||
SM: &discovery.SMInfo{GUID: "0x98039b03009fcfd6", LID: 1, State: "SMINFO_MASTER"}}}}}
|
||||
f.Fabrics = discovery.CheckFabrics(map[string][]string{"ib-a": {"mlx5_0"}}, f.HCAs)
|
||||
f.SMLists = map[string][]discovery.SMRecord{"0x98039b03009fcfd6": {
|
||||
{GUID: "0x98039b03009fcfd6", State: "master", Priority: 15}, {GUID: "0x98039b03009fd000", State: "standby", Priority: 14}}}
|
||||
return f
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"sort"
|
||||
@@ -13,6 +11,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
|
||||
@@ -2,13 +2,13 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
|
||||
@@ -146,6 +146,18 @@
|
||||
<td>{{if .NodesWithWeights}}{{.Models}} model(s) on {{.NodesWithWeights}}/{{.Nodes}} nodes{{else}}<span class="note">not reported</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}
|
||||
{{with .Fabrics}}{{if .Physical}}
|
||||
<h3>Subnet managers</h3>
|
||||
<table><tr><th>Fabric</th><th>Is physically</th><th>Master</th><th>Standby</th><th class="num">Nodes</th><th>Last change of master</th></tr>
|
||||
{{range .Physical}}<tr>
|
||||
<td>{{if .Declared}}{{range $i, $d := .Declared}}{{if $i}}, {{end}}<code>{{$d}}</code>{{end}}{{else}}<span class="note">undeclared</span>{{end}}</td>
|
||||
<td>{{.Label}}{{if .RateGbps}} <span class="note">· {{.RateGbps}} Gb/s</span>{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}<span class="note">none: RoCE</span>{{else}}{{with .SM}}<code>{{.GUID}}</code>{{if .LID}} <span class="note">LID {{.LID}}, priority {{.Priority}}</span>{{end}}{{else}}—{{end}}{{end}}</td>
|
||||
<td>{{if eq .Kind "roce"}}—{{else if .SMs}}{{if .Standbys}}{{.Standbys}}{{else}}<span class="bad">none</span>{{end}}{{else}}<span class="note">not listed</span>{{end}}</td>
|
||||
<td class="num">{{len .Members}}</td>
|
||||
<td>{{with .Changes}}{{with index . 0}}{{ago .At}}: <code>{{.From}}</code> → <code>{{.To}}</code>{{end}}{{if gt (len .) 1}} <span class="note">({{len .}} in a day)</span>{{end}}{{else}}<span class="note">none in a day</span>{{end}}</td>
|
||||
</tr>{{end}}</table>
|
||||
{{end}}{{end}}
|
||||
<p class="note">A fabric's name is what the nodes were told; what it is physically is
|
||||
the subnet manager its InfiniBand ports see, or the network its RoCE ports are on.
|
||||
A replica spanning nodes is only ever formed on one of those. Fully free counts whole
|
||||
|
||||
@@ -657,14 +657,18 @@ type fabricSource struct{ fakeSource }
|
||||
func (fabricSource) Fabrics() fabrics.Report {
|
||||
return fabrics.Report{
|
||||
Physical: []fabrics.Physical{{Key: "sm:0x01", Kind: "infiniband", Label: "subnet manager 0x01 (LID 1)",
|
||||
Declared: []string{"ib-a"}, RateGbps: 400}},
|
||||
Declared: []string{"ib-a"}, RateGbps: 400,
|
||||
SM: &fabrics.SM{GUID: "0x01", LID: 1, Priority: 15, State: "master"},
|
||||
SMs: []fabrics.SM{{GUID: "0x01", State: "master"}},
|
||||
Changes: []fabrics.Change{{At: time.Now().Add(-3 * time.Hour), From: "0x02", To: "0x01", Ports: 4}}}},
|
||||
Problems: []fabrics.Problem{{Level: fabrics.Fail, Fabric: "ib-a", Message: "ib-a is 2 separate fabrics"}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheClusterPageSaysWhatEachFabricPhysicallyIs(t *testing.T) {
|
||||
_, body := get(t, Dashboard(fabricSource{}, fakeUsage{}, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`} {
|
||||
for _, want := range []string{"subnet manager 0x01 (LID 1)", "400 Gb/s", "ib-a is 2 separate fabrics", `<span class="tag bad">fail</span>`,
|
||||
"<h3>Subnet managers</h3>", "LID 1, priority 15", `<span class="bad">none</span>`, "3 hours ago: <code>0x02</code> → <code>0x01</code>"} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("missing %q", want)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user