Files
llmgw/internal/agent/discovery/sminfo.go
T
Ilker ManapandClaude Opus 5.5 23648fa19a Watch each fabric's subnet managers, and keep a fabric one through a failover
A fabric's identity was its master subnet manager, which changes on a
failover: for the minute its ports took to agree, one fabric read as
two, and a name covering it as two separate fabrics. Each agent now
also lists every subnet manager on a fabric (saquery SMIR, through one
port per master, once a minute); managers listed together are one
fabric, named by the lowest GUID among them, so the identity survives
the master changing.

The same list says what a fabric can survive. The cluster page's new
subnet manager table, /api/v1/fabrics, doctor and metrics
(llmgw_fabric_master_sm, llmgw_fabric_subnet_managers,
llmgw_fabric_problems) show each fabric's master, priority and
standbys, and say when a fabric has no standby, its ports disagree on
the master, ports are up with no subnet manager (INIT), or the master
changed in the last hour. Every change of master a port reports is
kept, and a fabric's last day of them is listed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 07:47:03 +02:00

147 lines
4.3 KiB
Go

package discovery
import (
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
// cannot be asked who it is.
var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")
// smLine is sminfo's one line of answer:
//
// sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)
// ParseSMInfo reads sminfo's output.
func ParseSMInfo(out string) (*SMInfo, error) {
m := smLine.FindStringSubmatch(out)
if m == nil {
return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
}
lid, _ := strconv.Atoi(m[1])
prio, _ := strconv.Atoi(m[3])
return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
}
// QuerySM asks, through one port, which subnet manager is master. It is a
// management datagram to the SM, not a scan: cheap, but not free, which is
// why the agent asks once a minute rather than every heartbeat.
func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
bin, err := exec.LookPath("sminfo")
if err != nil {
return nil, ErrNoDiags
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
}
return ParseSMInfo(string(out))
}
// AttachSM fills in the subnet manager each active InfiniBand port sees.
// RoCE has none. The first error is returned for reporting; the ports it
// could answer for keep their answer.
func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
var first error
for i := range hcas {
for j := range hcas[i].Ports {
p := &hcas[i].Ports[j]
if !p.Active() || p.LinkLayer == LinkEthernet {
continue
}
sm, err := query(hcas[i].Name, p.Num)
if err != nil {
if first == nil {
first = err
}
continue
}
p.SM = sm
}
}
return first
}
// SMRecord is one subnet manager on a fabric, as the subnet administrator
// lists them: master and standbys alike.
type SMRecord struct {
GUID string `json:"guid"`
LID int `json:"lid"`
Priority int `json:"priority"`
// State is master, standby, discovering or not active.
State string `json:"state"`
}
// smStates are SMInfo's state numbers.
var smStates = map[string]string{"0": "not active", "1": "discovering", "2": "standby", "3": "master"}
// smirField is one line of saquery's record dump: "LID.........1".
var smirField = regexp.MustCompile(`^\s*([A-Za-z_]+)\.{2,}\s*(\S+)`)
// ParseSMIR reads `saquery SMIR`: every subnet manager the fabric's subnet
// administrator knows of, one "SMInfoRecord dump:" block each.
func ParseSMIR(out string) ([]SMRecord, error) {
var recs []SMRecord
var cur *SMRecord
for _, line := range strings.Split(out, "\n") {
if strings.Contains(line, "SMInfoRecord dump") {
recs = append(recs, SMRecord{})
cur = &recs[len(recs)-1]
continue
}
m := smirField.FindStringSubmatch(line)
if m == nil || cur == nil {
continue
}
switch strings.ToLower(m[1]) {
case "lid":
cur.LID, _ = strconv.Atoi(m[2])
case "guid":
cur.GUID = normalizeGUID(m[2])
case "priority":
cur.Priority, _ = strconv.Atoi(m[2])
case "smstate":
cur.State = smStates[m[2]]
if cur.State == "" {
cur.State = m[2]
}
}
}
out2 := recs[:0]
for _, r := range recs {
if r.GUID != "" {
out2 = append(out2, r)
}
}
if len(out2) == 0 {
return nil, fmt.Errorf("no subnet manager records in saquery output")
}
return out2, nil
}
// QuerySMs asks the subnet administrator, through one port, for every subnet
// manager on the fabric.
func QuerySMs(ctx context.Context, hca string, port int) ([]SMRecord, error) {
bin, err := exec.LookPath("saquery")
if err != nil {
return nil, ErrNoDiags
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port), "SMIR").CombinedOutput()
if err != nil {
return nil, fmt.Errorf("saquery on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
}
return ParseSMIR(string(out))
}