Files
llmgw/internal/control/fabrics/fabrics.go
T
Ilker ManapandClaude Opus 5.5 77fe801892 Tell independent fabrics apart by their subnet manager, not their name
A fabric was what agent.yaml called it. With several independent
InfiniBand fabrics at a site, two given one name would have had a
replica spread across both, starting and then hanging on its first
collective. Each active InfiniBand port now reports the master subnet
manager it sees (sminfo, asked once a minute), which every port on one
fabric shares and ports on separate fabrics cannot; a RoCE port reports
the IP network of its RoCE v2 address. Ports also carry their node and
port GUIDs.

Placement divides a declared fabric by that physical identity and forms
a replica on one part only; a node whose adapters for one name are on
two fabrics is refused, and a node that cannot say is kept apart from
those that can. /api/v1/fabrics and the cluster page list each physical
fabric with its subnet manager, and say when one name covers two
fabrics, two names one fabric, an adapter is up but undeclared, or a
node cannot be checked. doctor reports each port's subnet manager, and
an agent on a node with adapters installs infiniband-diags when sminfo
is missing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 07:29:16 +02:00

244 lines
7.4 KiB
Go

// Package fabrics works out which physical fabrics the cluster's adapters
// are on, and where that disagrees with what the nodes were told.
//
// A fabric's name is configuration. What it is physically is what the
// adapters report: on InfiniBand the master subnet manager every port on one
// fabric sees, on RoCE the IP network. Kept apart so a site with several
// independent fabrics can see each, and see at once when two were given one
// name, one was given two, or an adapter is cabled somewhere nobody declared.
package fabrics
import (
"fmt"
"slices"
"sort"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Level is how much a problem matters.
type Level string
// The levels.
const (
Fail Level = "fail"
Warn Level = "warn"
Info Level = "info"
)
// Problem is one disagreement, in words.
type Problem struct {
Level Level `json:"level"`
Fabric string `json:"fabric,omitempty"`
Message string `json:"message"`
}
// Physical is one fabric as the adapters find it.
type Physical struct {
Key string `json:"key"`
// Kind is infiniband or roce.
Kind string `json:"kind"`
// Label is the key as a person reads it.
Label string `json:"label"`
// SM is the master subnet manager, on InfiniBand.
SM *SM `json:"sm,omitempty"`
// Declared are the names nodes gave this fabric; one, when all is well.
Declared []string `json:"declared"`
Members []Member `json:"members"`
RateGbps int `json:"rate_gbps"`
GIDPrefix string `json:"gid_prefix,omitempty"`
}
// SM is a subnet manager as the ports see it.
type SM struct {
GUID string `json:"guid"`
LID int `json:"lid"`
Priority int `json:"priority"`
State string `json:"state"`
}
// Member is one node's adapters on a physical fabric.
type Member struct {
Node string `json:"node"`
HCAs []string `json:"hcas"`
// Declared is the name this node gives these adapters, empty when it
// declares them in no fabric.
Declared string `json:"declared,omitempty"`
}
// Report is the whole picture.
type Report struct {
Physical []Physical `json:"physical"`
Problems []Problem `json:"problems"`
// Unverified are declared fabrics some of whose nodes cannot say which
// physical fabric they are on, with why.
Unverified map[string][]string `json:"unverified,omitempty"`
}
// Analyze reads every node's ports.
func Analyze(nodes []inventory.Node) Report {
r := Report{Unverified: map[string][]string{}}
byKey := map[string]*Physical{}
declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes
for _, n := range nodes {
declOf := map[string]string{} // hca -> declared fabric
for _, f := range n.Fabrics {
for _, h := range f.HCAs {
declOf[h] = f.Fabric
}
}
// Which physical fabric each declared fabric's adapters are on here.
seenDecl := map[string]map[string]bool{}
members := map[string]*Member{} // key -> this node's membership
for _, p := range n.IBPorts {
key := p.FabricKey()
decl := declOf[p.HCA]
if key == "" {
if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" {
r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError)
}
continue
}
ph := byKey[key]
if ph == nil {
ph = &Physical{Key: key, Kind: "infiniband", GIDPrefix: p.GIDPrefix}
if p.RoCE() {
ph.Kind, ph.Label = "roce", "RoCE "+p.Network
} else {
ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState}
ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID)
}
byKey[key] = ph
}
if p.RateGbps > 0 && (ph.RateGbps == 0 || p.RateGbps < ph.RateGbps) {
ph.RateGbps = p.RateGbps
}
m := members[key]
if m == nil {
m = &Member{Node: n.ID, Declared: decl}
members[key] = m
}
m.HCAs = appendOnce(m.HCAs, p.HCA)
if decl != "" {
if seenDecl[decl] == nil {
seenDecl[decl] = map[string]bool{}
}
seenDecl[decl][key] = true
ph.Declared = appendOnce(ph.Declared, decl)
if declaredKeys[decl] == nil {
declaredKeys[decl] = map[string][]string{}
}
declaredKeys[decl][key] = appendOnce(declaredKeys[decl][key], n.ID)
} else {
r.Problems = append(r.Problems, Problem{Level: Warn, Message: fmt.Sprintf(
"%s: %s is up on %s but declared in no fabric, so llmgw sends nothing over it",
n.ID, p.HCA, ph.Label)})
}
}
for decl, keys := range seenDecl {
if len(keys) > 1 {
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
"%s: the adapters it declares as %s are cabled to %d different fabrics; it is left out of multi-node work on %s",
n.ID, decl, len(keys), decl)})
}
}
for key, m := range members {
byKey[key].Members = append(byKey[key].Members, *m)
}
}
for decl, keys := range declaredKeys {
if len(keys) < 2 {
continue
}
var parts []string
for _, k := range sortedKeys(keys) {
parts = append(parts, fmt.Sprintf("%s: %s", byKey[k].Label, strings.Join(keys[k], ", ")))
}
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
"%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name",
decl, len(keys), strings.Join(parts, "; "))})
}
for _, ph := range byKey {
sort.Strings(ph.Declared)
sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node })
if len(ph.Declared) > 1 {
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: ph.Declared[0], Message: fmt.Sprintf(
"%s are one fabric (%s): a replica could use all of them, but each name is placed on alone",
strings.Join(ph.Declared, " and "), ph.Label)})
}
r.Physical = append(r.Physical, *ph)
}
sort.Slice(r.Physical, func(i, j int) bool {
a, b := r.Physical[i], r.Physical[j]
if na, nb := first(a.Declared), first(b.Declared); na != nb {
return na < nb
}
return a.Key < b.Key
})
// Separate InfiniBand fabrics on one subnet prefix: harmless while they
// stay apart, and the reason their GIDs cannot tell them apart.
byPrefix := map[string][]string{}
for _, ph := range r.Physical {
if ph.Kind == "infiniband" && ph.GIDPrefix != "" {
byPrefix[ph.GIDPrefix] = append(byPrefix[ph.GIDPrefix], ph.Label)
}
}
for prefix, labels := range byPrefix {
if len(labels) > 1 {
r.Problems = append(r.Problems, Problem{Level: Info, Message: fmt.Sprintf(
"%d separate fabrics share the subnet prefix %s; that is the default, and fine while they stay apart — a distinct subnet_prefix per subnet manager makes their GIDs unique",
len(labels), prefix)})
}
}
for decl, why := range r.Unverified {
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf(
"%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))})
}
sort.SliceStable(r.Problems, func(i, j int) bool {
if li, lj := rank(r.Problems[i].Level), rank(r.Problems[j].Level); li != lj {
return li < lj
}
return r.Problems[i].Message < r.Problems[j].Message
})
return r
}
func rank(l Level) int {
switch l {
case Fail:
return 0
case Warn:
return 1
}
return 2
}
func first(s []string) string {
if len(s) == 0 {
return "~" // undeclared last
}
return s[0]
}
func appendOnce(s []string, v string) []string {
if slices.Contains(s, v) {
return s
}
return append(s, v)
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
sort.Strings(out)
return out
}