A fabric's identity was its master subnet manager, which changes on a failover: for the minute its ports took to agree, one fabric read as two, and a name covering it as two separate fabrics. Each agent now also lists every subnet manager on a fabric (saquery SMIR, through one port per master, once a minute); managers listed together are one fabric, named by the lowest GUID among them, so the identity survives the master changing. The same list says what a fabric can survive. The cluster page's new subnet manager table, /api/v1/fabrics, doctor and metrics (llmgw_fabric_master_sm, llmgw_fabric_subnet_managers, llmgw_fabric_problems) show each fabric's master, priority and standbys, and say when a fabric has no standby, its ports disagree on the master, ports are up with no subnet manager (INIT), or the master changed in the last hour. Every change of master a port reports is kept, and a fabric's last day of them is listed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
147 lines
4.3 KiB
Go
147 lines
4.3 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os/exec"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
|
|
// cannot be asked who it is.
|
|
var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")
|
|
|
|
// smLine is sminfo's one line of answer:
|
|
//
|
|
// sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
|
|
var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)
|
|
|
|
// ParseSMInfo reads sminfo's output.
|
|
func ParseSMInfo(out string) (*SMInfo, error) {
|
|
m := smLine.FindStringSubmatch(out)
|
|
if m == nil {
|
|
return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
|
|
}
|
|
lid, _ := strconv.Atoi(m[1])
|
|
prio, _ := strconv.Atoi(m[3])
|
|
return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
|
|
}
|
|
|
|
// QuerySM asks, through one port, which subnet manager is master. It is a
|
|
// management datagram to the SM, not a scan: cheap, but not free, which is
|
|
// why the agent asks once a minute rather than every heartbeat.
|
|
func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
|
|
bin, err := exec.LookPath("sminfo")
|
|
if err != nil {
|
|
return nil, ErrNoDiags
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
|
|
}
|
|
return ParseSMInfo(string(out))
|
|
}
|
|
|
|
// AttachSM fills in the subnet manager each active InfiniBand port sees.
|
|
// RoCE has none. The first error is returned for reporting; the ports it
|
|
// could answer for keep their answer.
|
|
func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
|
|
var first error
|
|
for i := range hcas {
|
|
for j := range hcas[i].Ports {
|
|
p := &hcas[i].Ports[j]
|
|
if !p.Active() || p.LinkLayer == LinkEthernet {
|
|
continue
|
|
}
|
|
sm, err := query(hcas[i].Name, p.Num)
|
|
if err != nil {
|
|
if first == nil {
|
|
first = err
|
|
}
|
|
continue
|
|
}
|
|
p.SM = sm
|
|
}
|
|
}
|
|
return first
|
|
}
|
|
|
|
// SMRecord is one subnet manager on a fabric, as the subnet administrator
|
|
// lists them: master and standbys alike.
|
|
type SMRecord struct {
|
|
GUID string `json:"guid"`
|
|
LID int `json:"lid"`
|
|
Priority int `json:"priority"`
|
|
// State is master, standby, discovering or not active.
|
|
State string `json:"state"`
|
|
}
|
|
|
|
// smStates are SMInfo's state numbers.
|
|
var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"}
|
|
|
|
// smirField is one line of saquery's record dump: "LID.........1".
|
|
var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`)
|
|
|
|
// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet
|
|
// administrator knows of, one "SMInfoRecord dump:" block each.
|
|
func ParseSMIR(out string) ([]SMRecord, error) {
|
|
var recs []SMRecord
|
|
var cur *SMRecord
|
|
for _, line := range strings.Split(out, "\n") {
|
|
if strings.Contains(line, "SMInfoRecord dump") {
|
|
recs = append(recs, SMRecord{})
|
|
cur = &recs[len(recs)-1]
|
|
continue
|
|
}
|
|
m := smirField.FindStringSubmatch(line)
|
|
if m == nil || cur == nil {
|
|
continue
|
|
}
|
|
switch strings.ToLower(m[1]) {
|
|
case "lid":
|
|
cur.LID, _ = strconv.Atoi(m[2])
|
|
case "guid":
|
|
cur.GUID = normalizeGUID(m[2])
|
|
case "priority":
|
|
cur.Priority, _ = strconv.Atoi(m[2])
|
|
case "smstate":
|
|
cur.State = smStates[m[2]]
|
|
if cur.State == "" {
|
|
cur.State = m[2]
|
|
}
|
|
}
|
|
}
|
|
out2 := recs[:0]
|
|
for _, r := range recs {
|
|
if r.GUID != "" {
|
|
out2 = append(out2, r)
|
|
}
|
|
}
|
|
if len(out2) == 0 {
|
|
return nil, fmt.Errorf("no subnet manager records in saquery output")
|
|
}
|
|
return out2, nil
|
|
}
|
|
|
|
// QuerySMs asks the subnet administrator, through one port, for every subnet
|
|
// manager on the fabric.
|
|
func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
|
|
bin, err := exec.LookPath("saquery")
|
|
if err != nil {
|
|
return nil, ErrNoDiags
|
|
}
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
|
|
}
|
|
return ParseSMIR(string(out))
|
|
}
|