Watch each fabric's subnet managers, and keep a fabric one through a failover

A fabric's identity was its master subnet manager, which changes on a
failover: for the minute its ports took to agree, one fabric read as
two, and a name covering it as two separate fabrics. Each agent now
also lists every subnet manager on a fabric (saquery SMIR, through one
port per master, once a minute); managers listed together are one
fabric, named by the lowest GUID among them, so the identity survives
the master changing.

The same list says what a fabric can survive. The cluster page's new
subnet manager table, /api/v1/fabrics, doctor and metrics
(llmgw_fabric_master_sm, llmgw_fabric_subnet_managers,
llmgw_fabric_problems) show each fabric's master, priority and
standbys, and say when a fabric has no standby, its ports disagree on
the master, ports are up with no subnet manager (INIT), or the master
changed in the last hour. Every change of master a port reports is
kept, and a fabric's last day of them is listed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 07:47:03 +02:00
1 parent 77fe801892
commit 23648fa19a
20 files changed
+857 -136

No files matched your search

+3 -2
View File
@@ -10,7 +10,6 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"io"
"net/http"
"slices"
@@ -20,6 +19,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/bench"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -836,5 +836,6 @@ func (a *API) profileChanged(model, profile, was string) bool {
// Fabrics is the physical fabrics the adapters are on, against the names
// the nodes gave them.
func (a *API) Fabrics() fabrics.Report {
return fabrics.Analyze(a.ctl.Inventory().Nodes())
inv := a.ctl.Inventory()
return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now())
}
+20 -1
View File
@@ -405,9 +405,28 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
n.Load = loadFrom(l, time.Now())
}
if ports := hb.GetIbPorts(); len(ports) > 0 {
n.IBPorts = n.IBPorts[:0]
before := map[string]string{}
for _, p := range n.IBPorts {
before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID
}
n.IBPorts = nil
for _, p := range ports {
// A different master than last time is a failover, or a port
// moved: the fabric's own event, kept for the page to show.
where := fmt.Sprintf("%s/%d", p.GetHca(), p.GetPort())
if old := before[where]; old != "" && p.GetSmGuid() != "" && old != p.GetSmGuid() {
s.log.Warn("a port sees a different master subnet manager", "node", nodeID,
"port", where, "from", old, "to", p.GetSmGuid())
s.inv.RecordSMChange(inventory.SMChange{At: time.Now(), Node: nodeID, Port: where,
From: old, To: p.GetSmGuid()})
}
var sms []inventory.SubnetManager
for _, r := range p.GetSubnetManagers() {
sms = append(sms, inventory.SubnetManager{GUID: r.GetGuid(), LID: int(r.GetLid()),
Priority: int(r.GetPriority()), State: r.GetState()})
}
n.IBPorts = append(n.IBPorts, inventory.IBPort{
SMs: sms, SMsError: p.GetSubnetManagersError(),
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
+142 -6
View File
@@ -13,6 +13,7 @@ import (
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -41,8 +42,17 @@ type Physical struct {
Kind string `json:"kind"`
// Label is the key as a person reads it.
Label string `json:"label"`
// SM is the master subnet manager, on InfiniBand.
// SM is the master subnet manager, on InfiniBand: the one most of its
// ports see.
SM *SM `json:"sm,omitempty"`
// SMs is every subnet manager the fabric lists, master first; empty when
// no port could ask. Standbys counts those that could take over.
SMs []SM `json:"sms,omitempty"`
Standbys int `json:"standbys"`
// Masters is how many ports see each master, when they do not all agree.
Masters map[string]int `json:"masters,omitempty"`
// Changes are the master changes seen in the last day, newest first.
Changes []Change `json:"changes,omitempty"`
// Declared are the names nodes gave this fabric; one, when all is well.
Declared []string `json:"declared"`
Members []Member `json:"members"`
@@ -58,6 +68,14 @@ type SM struct {
State string `json:"state"`
}
// Change is a fabric's master subnet manager changing, as its ports saw it.
type Change struct {
At time.Time `json:"at"`
From string `json:"from"`
To string `json:"to"`
Ports int `json:"ports"`
}
// Member is one node's adapters on a physical fabric.
type Member struct {
Node string `json:"node"`
@@ -76,9 +94,15 @@ type Report struct {
Unverified map[string][]string `json:"unverified,omitempty"`
}
// Analyze reads every node's ports.
func Analyze(nodes []inventory.Node) Report {
// Analyze reads every node's ports, and the changes of master subnet manager
// the control plane has recorded.
func Analyze(nodes []inventory.Node, changes []inventory.SMChange, now time.Time) Report {
r := Report{Unverified: map[string][]string{}}
canon := inventory.CanonicalSMs(nodes)
masters := map[string]map[string]int{} // key -> master GUID -> ports
smLists := map[string]map[string]inventory.SubnetManager{}
listErr := map[string]string{}
noSM := map[string][]string{} // declared -> node/port up with no SM
byKey := map[string]*Physical{}
declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes
@@ -94,8 +118,24 @@ func Analyze(nodes []inventory.Node) Report {
members := map[string]*Member{} // key -> this node's membership
for _, p := range n.IBPorts {
key := p.FabricKey()
key := p.FabricKeyIn(canon)
decl := declOf[p.HCA]
if decl != "" && !p.RoCE() && strings.EqualFold(p.State, "INIT") && strings.EqualFold(p.PhysState, "LinkUp") {
noSM[decl] = appendOnce(noSM[decl], fmt.Sprintf("%s %s/%d", n.ID, p.HCA, p.Port))
}
if key != "" && !p.RoCE() {
if masters[key] == nil {
masters[key] = map[string]int{}
smLists[key] = map[string]inventory.SubnetManager{}
}
masters[key][p.SMGUID]++
for _, sm := range p.SMs {
smLists[key][sm.GUID] = sm
}
if p.SMsError != "" {
listErr[key] = p.SMsError
}
}
if key == "" {
if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" {
r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError)
@@ -109,7 +149,7 @@ func Analyze(nodes []inventory.Node) Report {
ph.Kind, ph.Label = "roce", "RoCE "+p.Network
} else {
ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState}
ph.Label = fmt.Sprintf("subnet manager %s (LID %d)", p.SMGUID, p.SMLID)
ph.Label = "fabric of subnet manager " + strings.TrimPrefix(key, "sm:")
}
byKey[key] = ph
}
@@ -162,7 +202,10 @@ func Analyze(nodes []inventory.Node) Report {
"%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name",
decl, len(keys), strings.Join(parts, "; "))})
}
for _, ph := range byKey {
for key, ph := range byKey {
if ph.Kind == "infiniband" {
analyzeSMs(&r, ph, masters[key], smLists[key], listErr[key], nodes, changes, now)
}
sort.Strings(ph.Declared)
sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node })
if len(ph.Declared) > 1 {
@@ -195,6 +238,11 @@ func Analyze(nodes []inventory.Node) Report {
len(labels), prefix)})
}
}
for decl, ports := range noSM {
r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
"%s: %d port(s) have a link but no subnet manager has configured them (INIT): %s",
decl, len(ports), strings.Join(ports, ", "))})
}
for decl, why := range r.Unverified {
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf(
"%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))})
@@ -209,6 +257,94 @@ func Analyze(nodes []inventory.Node) Report {
return r
}
// analyzeSMs fills in a fabric's subnet managers and what is wrong with them.
func analyzeSMs(r *Report, ph *Physical, masters map[string]int, list map[string]inventory.SubnetManager,
listErr string, nodes []inventory.Node, changes []inventory.SMChange, now time.Time,
) {
name := first(ph.Declared)
if name == "~" {
name = ph.Label
}
// The master is the one most ports see.
best, most := "", -1
for g, n := range masters {
if n > most || (n == most && g < best) {
best, most = g, n
}
}
if best != "" && ph.SM != nil && ph.SM.GUID != best {
ph.SM = &SM{GUID: best}
}
if len(masters) > 1 {
ph.Masters = masters
var parts []string
for _, g := range sortedKeys(masters) {
parts = append(parts, fmt.Sprintf("%s by %d", g, masters[g]))
}
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
"%s: its ports do not agree on the master subnet manager (%s) — a failover in progress, or a fabric split in two",
name, strings.Join(parts, ", "))})
}
for _, sm := range list {
v := SM{GUID: sm.GUID, LID: sm.LID, Priority: sm.Priority, State: sm.State}
ph.SMs = append(ph.SMs, v)
if sm.State == "standby" {
ph.Standbys++
}
if ph.SM != nil && sm.GUID == ph.SM.GUID {
ph.SM.LID, ph.SM.Priority, ph.SM.State = sm.LID, sm.Priority, sm.State
}
}
sort.Slice(ph.SMs, func(i, j int) bool {
if (ph.SMs[i].State == "master") != (ph.SMs[j].State == "master") {
return ph.SMs[i].State == "master"
}
return ph.SMs[i].Priority > ph.SMs[j].Priority
})
switch {
case len(list) == 0 && listErr != "":
r.Problems = append(r.Problems, Problem{Level: Info, Fabric: name, Message: fmt.Sprintf(
"%s: its standby subnet managers cannot be listed: %s", name, listErr)})
case len(list) > 0 && ph.Standbys == 0:
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
"%s has one subnet manager and no standby: if it stops, the fabric keeps its routes but cannot recover from any change — a cable, a switch, a node rebooting",
name)})
}
// Its recent failovers: the recorded changes of ports on this fabric,
// one per minute of changes rather than one per port.
onFabric := map[string]bool{}
for _, m := range ph.Members {
for _, h := range m.HCAs {
onFabric[m.Node+" "+h] = true
}
}
day := now.Add(-24 * time.Hour)
for _, c := range changes {
hca := c.Port
if i := strings.LastIndexByte(hca, '/'); i >= 0 {
hca = hca[:i]
}
if c.At.Before(day) || !onFabric[c.Node+" "+hca] {
continue
}
if n := len(ph.Changes); n > 0 && ph.Changes[n-1].To == c.To && c.At.Sub(ph.Changes[n-1].At) < 2*time.Minute {
ph.Changes[n-1].Ports++
continue
}
ph.Changes = append(ph.Changes, Change{At: c.At, From: c.From, To: c.To, Ports: 1})
}
slices.Reverse(ph.Changes)
if len(ph.Changes) > 0 && now.Sub(ph.Changes[0].At) < time.Hour {
c := ph.Changes[0]
r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
"%s: the master subnet manager changed %s ago, from %s to %s",
name, now.Sub(c.At).Round(time.Minute), c.From, c.To)})
}
_ = nodes
}
func rank(l Level) int {
switch l {
case Fail:
+66 -7
View File
@@ -3,6 +3,7 @@ package fabrics
import (
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -30,7 +31,7 @@ func problem(t *testing.T, r Report, l Level, fragment string) {
}
func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
r := Analyze([]inventory.Node{
r := analyze([]inventory.Node{
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("a2", "ib-a", "0x01", "mlx5_0"),
ibNode("b1", "ib-b", "0x02", "mlx5_0"),
})
@@ -50,14 +51,14 @@ func TestIndependentFabricsWithTheirOwnNamesAreFine(t *testing.T) {
}
func TestOneNameOnTwoFabricsIsAFailure(t *testing.T) {
r := Analyze([]inventory.Node{
r := analyze([]inventory.Node{
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-a", "0x02", "mlx5_0"),
})
problem(t, r, Fail, "ib-a is 2 separate fabrics")
}
func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
r := Analyze([]inventory.Node{
r := analyze([]inventory.Node{
ibNode("a1", "ib-a", "0x01", "mlx5_0"), ibNode("b1", "ib-b", "0x01", "mlx5_0"),
})
problem(t, r, Warn, "ib-a and ib-b are one fabric")
@@ -66,26 +67,84 @@ func TestTwoNamesOnOneFabricAreSaid(t *testing.T) {
func TestAnAdapterUpButUndeclaredIsSaid(t *testing.T) {
n := ibNode("a1", "ib-a", "0x01", "mlx5_0")
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: "mlx5_3", Port: 1, State: "ACTIVE", LinkLayer: "InfiniBand", SMGUID: "0x09"})
problem(t, Analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on subnet manager 0x09")
problem(t, analyze([]inventory.Node{n}), Warn, "mlx5_3 is up on fabric of subnet manager 0x09")
}
func TestOneNodesAdaptersOnTwoFabricsAreAFailure(t *testing.T) {
n := ibNode("x", "ib-a", "0x01", "mlx5_0", "mlx5_1")
n.IBPorts[1].SMGUID = "0x02"
problem(t, Analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
problem(t, analyze([]inventory.Node{n}), Fail, "cabled to 2 different fabrics")
}
func TestANodeThatCannotSayIsReported(t *testing.T) {
n := ibNode("x", "ib-a", "", "mlx5_0")
n.IBPorts[0].SMError = "sminfo not found: install infiniband-diags"
problem(t, Analyze([]inventory.Node{n}), Info, "install infiniband-diags")
problem(t, analyze([]inventory.Node{n}), Info, "install infiniband-diags")
}
func TestRoCEIsItsNetwork(t *testing.T) {
n := inventory.Node{ID: "r1", Fabrics: []inventory.FabricAttachment{{Fabric: "roce-a", HCAs: []string{"mlx5_0"}}},
IBPorts: []inventory.IBPort{{HCA: "mlx5_0", Port: 1, State: "ACTIVE", LinkLayer: "Ethernet", Network: "10.8.0.0/24"}}}
r := Analyze([]inventory.Node{n})
r := analyze([]inventory.Node{n})
if len(r.Physical) != 1 || r.Physical[0].Kind != "roce" || r.Physical[0].Label != "RoCE 10.8.0.0/24" {
t.Errorf("physical = %+v", r.Physical)
}
}
func analyze(nodes []inventory.Node) Report { return Analyze(nodes, nil, time.Now()) }
func withSMs(n inventory.Node, sms ...inventory.SubnetManager) inventory.Node {
n.IBPorts[0].SMs = sms
return n
}
func TestAFailoverKeepsAFabricOneFabric(t *testing.T) {
// Mid-failover some ports see the old master, some the new. Keyed by the
// master alone, one fabric read as two for a minute — and a declared name
// covering it as "two separate fabrics". The list of managers is the same
// before and after, which is what keeps it one.
sms := []inventory.SubnetManager{{GUID: "0x01", State: "master", Priority: 15}, {GUID: "0x02", State: "standby", Priority: 14}}
r := analyze([]inventory.Node{
withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"), sms...),
ibNode("a2", "ib-a", "0x02", "mlx5_0"),
})
if len(r.Physical) != 1 {
t.Fatalf("one fabric read as %d: %+v", len(r.Physical), r.Physical)
}
for _, p := range r.Problems {
if p.Level == Fail {
t.Errorf("a failover was reported as a failure: %s", p.Message)
}
}
problem(t, r, Warn, "do not agree on the master subnet manager")
if r.Physical[0].Standbys != 1 || len(r.Physical[0].SMs) != 2 || r.Physical[0].SMs[0].State != "master" {
t.Errorf("managers = %+v", r.Physical[0].SMs)
}
}
func TestAFabricWithNoStandbyIsSaid(t *testing.T) {
r := analyze([]inventory.Node{withSMs(ibNode("a1", "ib-a", "0x01", "mlx5_0"),
inventory.SubnetManager{GUID: "0x01", State: "master"})})
problem(t, r, Warn, "one subnet manager and no standby")
}
func TestPortsUpWithoutASubnetManagerAreAFailure(t *testing.T) {
n := ibNode("a1", "ib-a", "", "mlx5_0")
n.IBPorts[0].State, n.IBPorts[0].PhysState = "INIT", "LinkUp"
problem(t, analyze([]inventory.Node{n}), Fail, "no subnet manager has configured them")
}
func TestARecentFailoverIsShown(t *testing.T) {
now := time.Now()
nodes := []inventory.Node{withSMs(ibNode("a1", "ib-a", "0x02", "mlx5_0"),
inventory.SubnetManager{GUID: "0x01", State: "standby"}, inventory.SubnetManager{GUID: "0x02", State: "master"})}
changes := []inventory.SMChange{
{At: now.Add(-20 * time.Minute), Node: "a1", Port: "mlx5_0/1", From: "0x01", To: "0x02"},
{At: now.Add(-48 * time.Hour), Node: "a1", Port: "mlx5_0/1", From: "0x02", To: "0x01"},
}
r := Analyze(nodes, changes, now)
if got := r.Physical[0].Changes; len(got) != 1 || got[0].To != "0x02" {
t.Errorf("changes in the last day = %+v", got)
}
problem(t, r, Warn, "the master subnet manager changed 20m0s ago")
}
+40 -1
View File
@@ -16,10 +16,12 @@ package fleetmetrics
import (
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
@@ -71,6 +73,10 @@ type Collector struct {
ibActive *prometheus.Desc
ibRate *prometheus.Desc
ibCounter *prometheus.Desc
fabricSMs *prometheus.Desc
fabricMaster *prometheus.Desc
fabricProblems *prometheus.Desc
}
// loadFresh is how old a node's load may be and still be reported. A node
@@ -160,6 +166,15 @@ func New(src Source) *Collector {
ibRate: prometheus.NewDesc("llmgw_ib_port_rate_gbps",
"A port's negotiated rate. A link that trained below its usual speed is a "+
"cable or transceiver to look at.", []string{"node", "hca", "port"}, nil),
fabricSMs: prometheus.NewDesc("llmgw_fabric_subnet_managers",
"Subnet managers a physical fabric lists, by state. No standby means the fabric "+
"cannot recover from a change if its master stops.", []string{"fabric", "physical", "state"}, nil),
fabricMaster: prometheus.NewDesc("llmgw_fabric_master_sm",
"1 for a physical fabric's master subnet manager, as most of its ports see it; a "+
"change of sm_guid is a failover.", []string{"fabric", "physical", "sm_guid"}, nil),
fabricProblems: prometheus.NewDesc("llmgw_fabric_problems",
"Disagreements about the fabrics, by level: fail means multi-node work will not "+
"place or will not run where it should.", []string{"level"}, nil),
ibCounter: prometheus.NewDesc("llmgw_ib_port_counter_total",
"A port's cumulative counters, by sysfs name: symbol_error, link_downed and the "+
"other errors say a link is going bad before a job fails on it; port_xmit_data "+
@@ -173,7 +188,7 @@ func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen,
c.nodeDiskFree, c.nodeDiskSize,
c.nodeCPU, c.nodeCPUs, c.nodeLoad, c.nodeMemUsed, c.nodeMemAvail, c.shardCPU, c.shardMemory,
c.ibActive, c.ibRate, c.ibCounter,
c.ibActive, c.ibRate, c.ibCounter, c.fabricSMs, c.fabricMaster, c.fabricProblems,
c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy,
c.gpuNVLink, c.gpuECC,
c.replicaState, c.replicaReady, c.replicaIdle,
@@ -285,6 +300,30 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) {
}
}
// The fabrics as the adapters find them: their subnet managers, and how
// many things disagree.
rep := fabrics.Analyze(c.src.Nodes(), nil, now)
for _, ph := range rep.Physical {
name := strings.Join(ph.Declared, ",")
if ph.SM != nil && ph.SM.GUID != "" {
ch <- prometheus.MustNewConstMetric(c.fabricMaster, prometheus.GaugeValue, 1, name, ph.Key, ph.SM.GUID)
}
byState := map[string]int{}
for _, sm := range ph.SMs {
byState[sm.State]++
}
for st, n := range byState {
ch <- prometheus.MustNewConstMetric(c.fabricSMs, prometheus.GaugeValue, float64(n), name, ph.Key, st)
}
}
levels := map[fabrics.Level]int{fabrics.Fail: 0, fabrics.Warn: 0, fabrics.Info: 0}
for _, p := range rep.Problems {
levels[p.Level]++
}
for l, n := range levels {
ch <- prometheus.MustNewConstMetric(c.fabricProblems, prometheus.GaugeValue, float64(n), string(l))
}
// Replicas by model and state, so a model stuck starting is visible as a
// shape on a graph rather than only in a log line nobody tails.
counts := map[[2]string]int{}
+100 -3
View File
@@ -142,6 +142,75 @@ type IBPort struct {
SMError string
// Network is a RoCE port's IP network.
Network string
// SMs is every subnet manager on the port's fabric, master and standbys,
// on the one port per fabric that asked; SMsError says why it could not.
SMs []SubnetManager
SMsError string
}
// SubnetManager is one subnet manager on a fabric.
type SubnetManager struct {
GUID string
LID int
Priority int
State string
}
// CanonicalSMs maps every subnet manager GUID the cluster has heard of to
// one name for its fabric: the lowest GUID among the managers listed
// together. A fabric's master changes on a failover and its list does not,
// so ports that see the old master and ports that already see the new one
// stay one fabric through it.
func CanonicalSMs(nodes []Node) map[string]string {
parent := map[string]string{}
var find func(string) string
find = func(x string) string {
p, ok := parent[x]
if !ok || p == x {
parent[x] = x
return x
}
root := find(p)
parent[x] = root
return root
}
union := func(a, b string) {
ra, rb := find(a), find(b)
if ra == rb {
return
}
if ra < rb {
parent[rb] = ra
} else {
parent[ra] = rb
}
}
for _, n := range nodes {
for _, p := range n.IBPorts {
if p.SMGUID != "" {
find(p.SMGUID)
}
for _, sm := range p.SMs {
union(sm.GUID, p.SMGUID)
if p.SMGUID == "" {
find(sm.GUID)
}
}
}
}
out := make(map[string]string, len(parent))
for g := range parent {
out[g] = find(g)
}
return out
}
// SMChange is a port seeing a different master subnet manager than before.
type SMChange struct {
At time.Time `json:"at"`
Node string `json:"node"`
Port string `json:"port"`
From, To string
}
// Active reports whether the port can carry traffic.
@@ -153,7 +222,11 @@ func (p IBPort) RoCE() bool { return strings.EqualFold(p.LinkLayer, "Ethernet")
// FabricKey names the physical fabric the port is on: its master subnet
// manager on InfiniBand, its IP network on RoCE. Empty when not known, and
// for a port that is not up.
func (p IBPort) FabricKey() string {
func (p IBPort) FabricKey() string { return p.FabricKeyIn(nil) }
// FabricKeyIn is FabricKey with the subnet manager named as canon has it,
// so a fabric keeps one key through a failover.
func (p IBPort) FabricKeyIn(canon map[string]string) string {
if !p.Active() {
return ""
}
@@ -161,6 +234,9 @@ func (p IBPort) FabricKey() string {
case p.RoCE() && p.Network != "":
return "roce:" + p.Network
case !p.RoCE() && p.SMGUID != "":
if c, ok := canon[p.SMGUID]; ok {
return "sm:" + c
}
return "sm:" + p.SMGUID
}
return ""
@@ -169,13 +245,13 @@ func (p IBPort) FabricKey() string {
// PhysicalFabric is the physical fabric this node's adapters for a declared
// fabric are on. Known is false when none of them says; ok is false when
// they disagree — two adapters declared as one fabric that are cabled to two.
func (n Node) PhysicalFabric(fabric string) (key string, known, ok bool) {
func (n Node) PhysicalFabric(fabric string, canon map[string]string) (key string, known, ok bool) {
hcas := n.HCAsFor(fabric)
for _, p := range n.IBPorts {
if !slices.Contains(hcas, p.HCA) {
continue
}
k := p.FabricKey()
k := p.FabricKeyIn(canon)
if k == "" {
continue
}
@@ -279,6 +355,8 @@ type Store struct {
nodes map[string]*Node
// allocs maps node id to the shards placed there.
allocs map[string][]Allocation
// smChanges are ports' changes of master subnet manager, newest last.
smChanges []SMChange
}
// NewStore returns an empty inventory.
@@ -288,6 +366,25 @@ func NewStore() *Store {
// Upsert records a node's registration, preserving operator-set state such as
// a cordon: an agent reconnecting must not silently undo an operator's decision.
// RecordSMChange keeps a port's change of master subnet manager, the last
// few hundred of them: a failover is the fabric's own event, and nothing
// else would remember it once the ports agree again.
func (s *Store) RecordSMChange(c SMChange) {
s.mu.Lock()
defer s.mu.Unlock()
s.smChanges = append(s.smChanges, c)
if len(s.smChanges) > 500 {
s.smChanges = slices.Clone(s.smChanges[len(s.smChanges)-500:])
}
}
// SMChanges returns the recorded changes, oldest first.
func (s *Store) SMChanges() []SMChange {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.smChanges)
}
func (s *Store) Upsert(n Node) {
s.mu.Lock()
defer s.mu.Unlock()
+4 -3
View File
@@ -246,9 +246,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
}}
}
canon := inventory.CanonicalSMs(s.inv.Nodes())
for _, fabric := range fabrics {
all := s.eligibleNodes(p, fabric, avoid, &rejections)
for _, group := range physicalGroups(fabric, all, p, &rejections) {
for _, group := range physicalGroups(fabric, all, p, canon, &rejections) {
pool := fabric + "/" + p.classLabel()
if group.key != "" && group.split {
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
@@ -278,10 +279,10 @@ type fabricGroup struct {
// subnet manager each node's ports see (by IP network on RoCE), a replica
// stays on one. Nodes that do not say — an agent without infiniband-diags —
// are a group of their own rather than trusted to join one that does.
func physicalGroups(fabric string, cands []candidate, p Profile, rejections *[]Rejection) []fabricGroup {
func physicalGroups(fabric string, cands []candidate, p Profile, canon map[string]string, rejections *[]Rejection) []fabricGroup {
by := map[string][]candidate{}
for _, c := range cands {
key, _, ok := c.node.PhysicalFabric(fabric)
key, _, ok := c.node.PhysicalFabric(fabric, canon)
if !ok {
*rejections = append(*rejections, Rejection{
Profile: p.Name, Pool: fabric + "/" + p.classLabel(), Node: c.node.ID,
@@ -958,3 +958,15 @@ func TestAdaptersOfOneDeclaredFabricOnTwoAreRefused(t *testing.T) {
t.Errorf("expected a refusal naming the cabling:\n%s", res.Explain())
}
}
func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) {
// One port already sees the new master, the other the old; the managers
// listed are the same, so the two nodes are still one fabric.
a := onSM(node("a1", "h100-80g", 8, "ib-a"), "0x01")
a.IBPorts[0].SMs = []inventory.SubnetManager{{GUID: "0x01", State: "master"}, {GUID: "0x02", State: "standby"}}
b := onSM(node("a2", "h100-80g", 8, "ib-a"), "0x02")
res := New(store(a, b)).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("a failover split the fabric:\n%s", res.Explain())
}
}