Files
llmgw/internal/agent/discovery/sminfo.go
T
Ilker ManapandClaude Opus 5.5 77fe801892 Tell independent fabrics apart by their subnet manager, not their name
A fabric was what agent.yaml called it. With several independent
InfiniBand fabrics at a site, two given one name would have had a
replica spread across both, starting and then hanging on its first
collective. Each active InfiniBand port now reports the master subnet
manager it sees (sminfo, asked once a minute), which every port on one
fabric shares and ports on separate fabrics cannot; a RoCE port reports
the IP network of its RoCE v2 address. Ports also carry their node and
port GUIDs.

Placement divides a declared fabric by that physical identity and forms
a replica on one part only; a node whose adapters for one name are on
two fabrics is refused, and a node that cannot say is kept apart from
those that can. /api/v1/fabrics and the cluster page list each physical
fabric with its subnet manager, and say when one name covers two
fabrics, two names one fabric, an adapter is up but undeclared, or a
node cannot be checked. doctor reports each port's subnet manager, and
an agent on a node with adapters installs infiniband-diags when sminfo
is missing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 07:29:16 +02:00

74 lines
2.2 KiB
Go

package discovery
import (
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// ErrNoDiags means infiniband-diags is not installed, so the subnet manager
// cannot be asked who it is.
var ErrNoDiags = errors.New("sminfo not found: install infiniband-diags")
// smLine is sminfo's one line of answer:
//
// sminfo: sm lid 1 sm guid 0x98039b03009fcfd6, activity count 5452 priority 15 state 3 SMINFO_MASTER
var smLine = regexp.MustCompile(`sm lid (\d+) sm guid (0x[0-9a-fA-F]+), activity count \d+ priority (\d+) state \d+ (\S+)`)
// ParseSMInfo reads sminfo's output.
func ParseSMInfo(out string) (*SMInfo, error) {
m := smLine.FindStringSubmatch(out)
if m == nil {
return nil, fmt.Errorf("unrecognised sminfo output: %q", strings.TrimSpace(out))
}
lid, _ := strconv.Atoi(m[1])
prio, _ := strconv.Atoi(m[3])
return &SMInfo{GUID: normalizeGUID(m[2]), LID: lid, Priority: prio, State: m[4]}, nil
}
// QuerySM asks, through one port, which subnet manager is master. It is a
// management datagram to the SM, not a scan: cheap, but not free, which is
// why the agent asks once a minute rather than every heartbeat.
func QuerySM(ctx context.Context, hca string, port int) (*SMInfo, error) {
bin, err := exec.LookPath("sminfo")
if err != nil {
return nil, ErrNoDiags
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
out, err := exec.CommandContext(ctx, bin, "-C", hca, "-P", strconv.Itoa(port)).CombinedOutput()
if err != nil {
return nil, fmt.Errorf("sminfo on %s/%d: %w: %s", hca, port, err, strings.TrimSpace(string(out)))
}
return ParseSMInfo(string(out))
}
// AttachSM fills in the subnet manager each active InfiniBand port sees.
// RoCE has none. The first error is returned for reporting; the ports it
// could answer for keep their answer.
func AttachSM(hcas []HCA, query func(hca string, port int) (*SMInfo, error)) error {
var first error
for i := range hcas {
for j := range hcas[i].Ports {
p := &hcas[i].Ports[j]
if !p.Active() || p.LinkLayer == LinkEthernet {
continue
}
sm, err := query(hcas[i].Name, p.Num)
if err != nil {
if first == nil {
first = err
}
continue
}
p.SM = sm
}
}
return first
}