// Package fabrics works out which physical fabrics the cluster's adapters // are on, and where that disagrees with what the nodes were told. // // A fabric's name is configuration. What it is physically is what the // adapters report: on InfiniBand the master subnet manager every port on one // fabric sees, on RoCE the IP network. Kept apart so a site with several // independent fabrics can see each, and see at once when two were given one // name, one was given two, or an adapter is cabled somewhere nobody declared. package fabrics import ( "fmt" "slices" "sort" "strings" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) // Level is how much a problem matters. type Level string // The levels. const ( Fail Level = "fail" Warn Level = "warn" Info Level = "info" ) // Problem is one disagreement, in words. type Problem struct { Level Level `json:"level"` Fabric string `json:"fabric,omitempty"` Message string `json:"message"` } // Physical is one fabric as the adapters find it. type Physical struct { Key string `json:"key"` // Kind is infiniband or roce. Kind string `json:"kind"` // Label is the key as a person reads it. Label string `json:"label"` // SM is the master subnet manager, on InfiniBand: the one most of its // ports see. SM *SM `json:"sm,omitempty"` // SMs is every subnet manager the fabric lists, master first; empty when // no port could ask. Standbys counts those that could take over. SMs []SM `json:"sms,omitempty"` Standbys int `json:"standbys"` // Masters is how many ports see each master, when they do not all agree. Masters map[string]int `json:"masters,omitempty"` // Changes are the master changes seen in the last day, newest first. Changes []Change `json:"changes,omitempty"` // Declared are the names nodes gave this fabric; one, when all is well. Declared []string `json:"declared"` Members []Member `json:"members"` RateGbps int `json:"rate_gbps"` GIDPrefix string `json:"gid_prefix,omitempty"` } // SM is a subnet manager as the ports see it. type SM struct { GUID string `json:"guid"` LID int `json:"lid"` Priority int `json:"priority"` State string `json:"state"` } // Change is a fabric's master subnet manager changing, as its ports saw it. type Change struct { At time.Time `json:"at"` From string `json:"from"` To string `json:"to"` Ports int `json:"ports"` } // Member is one node's adapters on a physical fabric. type Member struct { Node string `json:"node"` HCAs []string `json:"hcas"` // Declared is the name this node gives these adapters, empty when it // declares them in no fabric. Declared string `json:"declared,omitempty"` } // Report is the whole picture. type Report struct { Physical []Physical `json:"physical"` Problems []Problem `json:"problems"` // Unverified are declared fabrics some of whose nodes cannot say which // physical fabric they are on, with why. Unverified map[string][]string `json:"unverified,omitempty"` } // Analyze reads every node's ports, and the changes of master subnet manager // the control plane has recorded. func Analyze(nodes []inventory.Node, changes []inventory.SMChange, now time.Time) Report { r := Report{Unverified: map[string][]string{}} canon := inventory.CanonicalSMs(nodes) masters := map[string]map[string]int{} // key -> master GUID -> ports smLists := map[string]map[string]inventory.SubnetManager{} listErr := map[string]string{} noSM := map[string][]string{} // declared -> node/port up with no SM byKey := map[string]*Physical{} declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes for _, n := range nodes { declOf := map[string]string{} // hca -> declared fabric for _, f := range n.Fabrics { for _, h := range f.HCAs { declOf[h] = f.Fabric } } // Which physical fabric each declared fabric's adapters are on here. seenDecl := map[string]map[string]bool{} members := map[string]*Member{} // key -> this node's membership for _, p := range n.IBPorts { key := p.FabricKeyIn(canon) decl := declOf[p.HCA] if decl != "" && !p.RoCE() && strings.EqualFold(p.State, "INIT") && strings.EqualFold(p.PhysState, "LinkUp") { noSM[decl] = appendOnce(noSM[decl], fmt.Sprintf("%s %s/%d", n.ID, p.HCA, p.Port)) } if key != "" && !p.RoCE() { if masters[key] == nil { masters[key] = map[string]int{} smLists[key] = map[string]inventory.SubnetManager{} } masters[key][p.SMGUID]++ for _, sm := range p.SMs { smLists[key][sm.GUID] = sm } if p.SMsError != "" { listErr[key] = p.SMsError } } if key == "" { if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" { r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError) } continue } ph := byKey[key] if ph == nil { ph = &Physical{Key: key, Kind: "infiniband", GIDPrefix: p.GIDPrefix} if p.RoCE() { ph.Kind, ph.Label = "roce", "RoCE "+p.Network } else { ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState} ph.Label = "fabric of subnet manager " + strings.TrimPrefix(key, "sm:") } byKey[key] = ph } if p.RateGbps > 0 && (ph.RateGbps == 0 || p.RateGbps < ph.RateGbps) { ph.RateGbps = p.RateGbps } m := members[key] if m == nil { m = &Member{Node: n.ID, Declared: decl} members[key] = m } m.HCAs = appendOnce(m.HCAs, p.HCA) if decl != "" { if seenDecl[decl] == nil { seenDecl[decl] = map[string]bool{} } seenDecl[decl][key] = true ph.Declared = appendOnce(ph.Declared, decl) if declaredKeys[decl] == nil { declaredKeys[decl] = map[string][]string{} } declaredKeys[decl][key] = appendOnce(declaredKeys[decl][key], n.ID) } else { r.Problems = append(r.Problems, Problem{Level: Warn, Message: fmt.Sprintf( "%s: %s is up on %s but declared in no fabric, so llmgw sends nothing over it", n.ID, p.HCA, ph.Label)}) } } for decl, keys := range seenDecl { if len(keys) > 1 { r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( "%s: the adapters it declares as %s are cabled to %d different fabrics; it is left out of multi-node work on %s", n.ID, decl, len(keys), decl)}) } } for key, m := range members { byKey[key].Members = append(byKey[key].Members, *m) } } for decl, keys := range declaredKeys { if len(keys) < 2 { continue } var parts []string for _, k := range sortedKeys(keys) { parts = append(parts, fmt.Sprintf("%s: %s", byKey[k].Label, strings.Join(keys[k], ", "))) } r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( "%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name", decl, len(keys), strings.Join(parts, "; "))}) } for key, ph := range byKey { if ph.Kind == "infiniband" { analyzeSMs(&r, ph, masters[key], smLists[key], listErr[key], nodes, changes, now) } sort.Strings(ph.Declared) sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node }) if len(ph.Declared) > 1 { r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: ph.Declared[0], Message: fmt.Sprintf( "%s are one fabric (%s): a replica could use all of them, but each name is placed on alone", strings.Join(ph.Declared, " and "), ph.Label)}) } r.Physical = append(r.Physical, *ph) } sort.Slice(r.Physical, func(i, j int) bool { a, b := r.Physical[i], r.Physical[j] if na, nb := first(a.Declared), first(b.Declared); na != nb { return na < nb } return a.Key < b.Key }) // Separate InfiniBand fabrics on one subnet prefix: harmless while they // stay apart, and the reason their GIDs cannot tell them apart. byPrefix := map[string][]string{} for _, ph := range r.Physical { if ph.Kind == "infiniband" && ph.GIDPrefix != "" { byPrefix[ph.GIDPrefix] = append(byPrefix[ph.GIDPrefix], ph.Label) } } for prefix, labels := range byPrefix { if len(labels) > 1 { r.Problems = append(r.Problems, Problem{Level: Info, Message: fmt.Sprintf( "%d separate fabrics share the subnet prefix %s; that is the default, and fine while they stay apart — a distinct subnet_prefix per subnet manager makes their GIDs unique", len(labels), prefix)}) } } for decl, ports := range noSM { r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf( "%s: %d port(s) have a link but no subnet manager has configured them (INIT): %s", decl, len(ports), strings.Join(ports, ", "))}) } for decl, why := range r.Unverified { r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf( "%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))}) } sort.SliceStable(r.Problems, func(i, j int) bool { if li, lj := rank(r.Problems[i].Level), rank(r.Problems[j].Level); li != lj { return li < lj } return r.Problems[i].Message < r.Problems[j].Message }) return r } // analyzeSMs fills in a fabric's subnet managers and what is wrong with them. func analyzeSMs(r *Report, ph *Physical, masters map[string]int, list map[string]inventory.SubnetManager, listErr string, nodes []inventory.Node, changes []inventory.SMChange, now time.Time, ) { name := first(ph.Declared) if name == "~" { name = ph.Label } // The master is the one most ports see. best, most := "", -1 for g, n := range masters { if n > most || (n == most && g < best) { best, most = g, n } } if best != "" && ph.SM != nil && ph.SM.GUID != best { ph.SM = &SM{GUID: best} } if len(masters) > 1 { ph.Masters = masters var parts []string for _, g := range sortedKeys(masters) { parts = append(parts, fmt.Sprintf("%s by %d", g, masters[g])) } r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( "%s: its ports do not agree on the master subnet manager (%s) — a failover in progress, or a fabric split in two", name, strings.Join(parts, ", "))}) } for _, sm := range list { v := SM{GUID: sm.GUID, LID: sm.LID, Priority: sm.Priority, State: sm.State} ph.SMs = append(ph.SMs, v) if sm.State == "standby" { ph.Standbys++ } if ph.SM != nil && sm.GUID == ph.SM.GUID { ph.SM.LID, ph.SM.Priority, ph.SM.State = sm.LID, sm.Priority, sm.State } } sort.Slice(ph.SMs, func(i, j int) bool { if (ph.SMs[i].State == "master") != (ph.SMs[j].State == "master") { return ph.SMs[i].State == "master" } return ph.SMs[i].Priority > ph.SMs[j].Priority }) switch { case len(list) == 0 && listErr != "": r.Problems = append(r.Problems, Problem{Level: Info, Fabric: name, Message: fmt.Sprintf( "%s: its standby subnet managers cannot be listed: %s", name, listErr)}) case len(list) > 0 && ph.Standbys == 0: r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( "%s has one subnet manager and no standby: if it stops, the fabric keeps its routes but cannot recover from any change — a cable, a switch, a node rebooting", name)}) } // Its recent failovers: the recorded changes of ports on this fabric, // one per minute of changes rather than one per port. onFabric := map[string]bool{} for _, m := range ph.Members { for _, h := range m.HCAs { onFabric[m.Node+" "+h] = true } } day := now.Add(-24 * time.Hour) for _, c := range changes { hca := c.Port if i := strings.LastIndexByte(hca, '/'); i >= 0 { hca = hca[:i] } if c.At.Before(day) || !onFabric[c.Node+" "+hca] { continue } if n := len(ph.Changes); n > 0 && ph.Changes[n-1].To == c.To && c.At.Sub(ph.Changes[n-1].At) < 2*time.Minute { ph.Changes[n-1].Ports++ continue } ph.Changes = append(ph.Changes, Change{At: c.At, From: c.From, To: c.To, Ports: 1}) } slices.Reverse(ph.Changes) if len(ph.Changes) > 0 && now.Sub(ph.Changes[0].At) < time.Hour { c := ph.Changes[0] r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf( "%s: the master subnet manager changed %s ago, from %s to %s", name, now.Sub(c.At).Round(time.Minute), c.From, c.To)}) } _ = nodes } func rank(l Level) int { switch l { case Fail: return 0 case Warn: return 1 } return 2 } func first(s []string) string { if len(s) == 0 { return "~" // undeclared last } return s[0] } func appendOnce(s []string, v string) []string { if slices.Contains(s, v) { return s } return append(s, v) } func sortedKeys[V any](m map[string]V) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out }