llmgw/internal/control/fabrics/fabrics.go

// Package fabrics works out which physical fabrics the cluster's adapters
// are on, and where that disagrees with what the nodes were told.
//
// A fabric's name is configuration. What it is physically is what the
// adapters report: on InfiniBand the master subnet manager every port on one
// fabric sees, on RoCE the IP network. Kept apart so a site with several
// independent fabrics can see each, and see at once when two were given one
// name, one was given two, or an adapter is cabled somewhere nobody declared.
package fabrics

import (
	"fmt"
	"slices"
	"sort"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)

// Level is how much a problem matters.
type Level string

// The levels.
const (
	Fail Level = "fail"
	Warn Level = "warn"
	Info Level = "info"
)

// Problem is one disagreement, in words.
type Problem struct {
	Level   Level  `json:"level"`
	Fabric  string `json:"fabric,omitempty"`
	Message string `json:"message"`
}

// Physical is one fabric as the adapters find it.
type Physical struct {
	Key string `json:"key"`
	// Kind is infiniband or roce.
	Kind string `json:"kind"`
	// Label is the key as a person reads it.
	Label string `json:"label"`
	// SM is the master subnet manager, on InfiniBand: the one most of its
	// ports see.
	SM *SM `json:"sm,omitempty"`
	// SMs is every subnet manager the fabric lists, master first; empty when
	// no port could ask. Standbys counts those that could take over.
	SMs      []SM `json:"sms,omitempty"`
	Standbys int  `json:"standbys"`
	// Masters is how many ports see each master, when they do not all agree.
	Masters map[string]int `json:"masters,omitempty"`
	// Changes are the master changes seen in the last day, newest first.
	Changes []Change `json:"changes,omitempty"`
	// Declared are the names nodes gave this fabric; one, when all is well.
	Declared  []string `json:"declared"`
	Members   []Member `json:"members"`
	RateGbps  int      `json:"rate_gbps"`
	GIDPrefix string   `json:"gid_prefix,omitempty"`
}

// SM is a subnet manager as the ports see it.
type SM struct {
	GUID     string `json:"guid"`
	LID      int    `json:"lid"`
	Priority int    `json:"priority"`
	State    string `json:"state"`
}

// Change is a fabric's master subnet manager changing, as its ports saw it.
type Change struct {
	At    time.Time `json:"at"`
	From  string    `json:"from"`
	To    string    `json:"to"`
	Ports int       `json:"ports"`
}

// Member is one node's adapters on a physical fabric.
type Member struct {
	Node string   `json:"node"`
	HCAs []string `json:"hcas"`
	// Declared is the name this node gives these adapters, empty when it
	// declares them in no fabric.
	Declared string `json:"declared,omitempty"`
}

// Report is the whole picture.
type Report struct {
	Physical []Physical `json:"physical"`
	Problems []Problem  `json:"problems"`
	// Unverified are declared fabrics some of whose nodes cannot say which
	// physical fabric they are on, with why.
	Unverified map[string][]string `json:"unverified,omitempty"`
}

// Analyze reads every node's ports, and the changes of master subnet manager
// the control plane has recorded.
func Analyze(nodes []inventory.Node, changes []inventory.SMChange, now time.Time) Report {
	r := Report{Unverified: map[string][]string{}}
	canon := inventory.CanonicalSMs(nodes)
	masters := map[string]map[string]int{} // key -> master GUID -> ports
	smLists := map[string]map[string]inventory.SubnetManager{}
	listErr := map[string]string{}
	noSM := map[string][]string{} // declared -> node/port up with no SM
	byKey := map[string]*Physical{}
	declaredKeys := map[string]map[string][]string{} // declared -> key -> nodes

	for _, n := range nodes {
		declOf := map[string]string{} // hca -> declared fabric
		for _, f := range n.Fabrics {
			for _, h := range f.HCAs {
				declOf[h] = f.Fabric
			}
		}
		// Which physical fabric each declared fabric's adapters are on here.
		seenDecl := map[string]map[string]bool{}
		members := map[string]*Member{} // key -> this node's membership

		for _, p := range n.IBPorts {
			key := p.FabricKeyIn(canon)
			decl := declOf[p.HCA]
			if decl != "" && !p.RoCE() && strings.EqualFold(p.State, "INIT") && strings.EqualFold(p.PhysState, "LinkUp") {
				noSM[decl] = appendOnce(noSM[decl], fmt.Sprintf("%s %s/%d", n.ID, p.HCA, p.Port))
			}
			if key != "" && !p.RoCE() {
				if masters[key] == nil {
					masters[key] = map[string]int{}
					smLists[key] = map[string]inventory.SubnetManager{}
				}
				masters[key][p.SMGUID]++
				for _, sm := range p.SMs {
					smLists[key][sm.GUID] = sm
				}
				if p.SMsError != "" {
					listErr[key] = p.SMsError
				}
			}
			if key == "" {
				if p.Active() && decl != "" && !p.RoCE() && p.SMError != "" {
					r.Unverified[decl] = appendOnce(r.Unverified[decl], n.ID+": "+p.SMError)
				}
				continue
			}
			ph := byKey[key]
			if ph == nil {
				ph = &Physical{Key: key, Kind: "infiniband", GIDPrefix: p.GIDPrefix}
				if p.RoCE() {
					ph.Kind, ph.Label = "roce", "RoCE "+p.Network
				} else {
					ph.SM = &SM{GUID: p.SMGUID, LID: p.SMLID, Priority: p.SMPriority, State: p.SMState}
					ph.Label = "fabric of subnet manager " + strings.TrimPrefix(key, "sm:")
				}
				byKey[key] = ph
			}
			if p.RateGbps > 0 && (ph.RateGbps == 0 || p.RateGbps < ph.RateGbps) {
				ph.RateGbps = p.RateGbps
			}
			m := members[key]
			if m == nil {
				m = &Member{Node: n.ID, Declared: decl}
				members[key] = m
			}
			m.HCAs = appendOnce(m.HCAs, p.HCA)
			if decl != "" {
				if seenDecl[decl] == nil {
					seenDecl[decl] = map[string]bool{}
				}
				seenDecl[decl][key] = true
				ph.Declared = appendOnce(ph.Declared, decl)
				if declaredKeys[decl] == nil {
					declaredKeys[decl] = map[string][]string{}
				}
				declaredKeys[decl][key] = appendOnce(declaredKeys[decl][key], n.ID)
			} else {
				r.Problems = append(r.Problems, Problem{Level: Warn, Message: fmt.Sprintf(
					"%s: %s is up on %s but declared in no fabric, so llmgw sends nothing over it",
					n.ID, p.HCA, ph.Label)})
			}
		}
		for decl, keys := range seenDecl {
			if len(keys) > 1 {
				r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
					"%s: the adapters it declares as %s are cabled to %d different fabrics; it is left out of multi-node work on %s",
					n.ID, decl, len(keys), decl)})
			}
		}
		for key, m := range members {
			byKey[key].Members = append(byKey[key].Members, *m)
		}
	}

	for decl, keys := range declaredKeys {
		if len(keys) < 2 {
			continue
		}
		var parts []string
		for _, k := range sortedKeys(keys) {
			parts = append(parts, fmt.Sprintf("%s: %s", byKey[k].Label, strings.Join(keys[k], ", ")))
		}
		r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
			"%s is %d separate fabrics — %s. A replica is kept to one of them; give each its own name",
			decl, len(keys), strings.Join(parts, "; "))})
	}
	for key, ph := range byKey {
		if ph.Kind == "infiniband" {
			analyzeSMs(&r, ph, masters[key], smLists[key], listErr[key], nodes, changes, now)
		}
		sort.Strings(ph.Declared)
		sort.Slice(ph.Members, func(i, j int) bool { return ph.Members[i].Node < ph.Members[j].Node })
		if len(ph.Declared) > 1 {
			r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: ph.Declared[0], Message: fmt.Sprintf(
				"%s are one fabric (%s): a replica could use all of them, but each name is placed on alone",
				strings.Join(ph.Declared, " and "), ph.Label)})
		}
		r.Physical = append(r.Physical, *ph)
	}
	sort.Slice(r.Physical, func(i, j int) bool {
		a, b := r.Physical[i], r.Physical[j]
		if na, nb := first(a.Declared), first(b.Declared); na != nb {
			return na < nb
		}
		return a.Key < b.Key
	})

	// Separate InfiniBand fabrics on one subnet prefix: harmless while they
	// stay apart, and the reason their GIDs cannot tell them apart.
	byPrefix := map[string][]string{}
	for _, ph := range r.Physical {
		if ph.Kind == "infiniband" && ph.GIDPrefix != "" {
			byPrefix[ph.GIDPrefix] = append(byPrefix[ph.GIDPrefix], ph.Label)
		}
	}
	for prefix, labels := range byPrefix {
		if len(labels) > 1 {
			r.Problems = append(r.Problems, Problem{Level: Info, Message: fmt.Sprintf(
				"%d separate fabrics share the subnet prefix %s; that is the default, and fine while they stay apart — a distinct subnet_prefix per subnet manager makes their GIDs unique",
				len(labels), prefix)})
		}
	}
	for decl, ports := range noSM {
		r.Problems = append(r.Problems, Problem{Level: Fail, Fabric: decl, Message: fmt.Sprintf(
			"%s: %d port(s) have a link but no subnet manager has configured them (INIT): %s",
			decl, len(ports), strings.Join(ports, ", "))})
	}
	for decl, why := range r.Unverified {
		r.Problems = append(r.Problems, Problem{Level: Info, Fabric: decl, Message: fmt.Sprintf(
			"%s cannot be checked on every node: %s", decl, strings.Join(why, "; "))})
	}

	sort.SliceStable(r.Problems, func(i, j int) bool {
		if li, lj := rank(r.Problems[i].Level), rank(r.Problems[j].Level); li != lj {
			return li < lj
		}
		return r.Problems[i].Message < r.Problems[j].Message
	})
	return r
}

// analyzeSMs fills in a fabric's subnet managers and what is wrong with them.
func analyzeSMs(r *Report, ph *Physical, masters map[string]int, list map[string]inventory.SubnetManager,
	listErr string, nodes []inventory.Node, changes []inventory.SMChange, now time.Time,
) {
	name := first(ph.Declared)
	if name == "~" {
		name = ph.Label
	}
	// The master is the one most ports see.
	best, most := "", -1
	for g, n := range masters {
		if n > most || (n == most && g < best) {
			best, most = g, n
		}
	}
	if best != "" && ph.SM != nil && ph.SM.GUID != best {
		ph.SM = &SM{GUID: best}
	}
	if len(masters) > 1 {
		ph.Masters = masters
		var parts []string
		for _, g := range sortedKeys(masters) {
			parts = append(parts, fmt.Sprintf("%s by %d", g, masters[g]))
		}
		r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
			"%s: its ports do not agree on the master subnet manager (%s) — a failover in progress, or a fabric split in two",
			name, strings.Join(parts, ", "))})
	}

	for _, sm := range list {
		v := SM{GUID: sm.GUID, LID: sm.LID, Priority: sm.Priority, State: sm.State}
		ph.SMs = append(ph.SMs, v)
		if sm.State == "standby" {
			ph.Standbys++
		}
		if ph.SM != nil && sm.GUID == ph.SM.GUID {
			ph.SM.LID, ph.SM.Priority, ph.SM.State = sm.LID, sm.Priority, sm.State
		}
	}
	sort.Slice(ph.SMs, func(i, j int) bool {
		if (ph.SMs[i].State == "master") != (ph.SMs[j].State == "master") {
			return ph.SMs[i].State == "master"
		}
		return ph.SMs[i].Priority > ph.SMs[j].Priority
	})
	switch {
	case len(list) == 0 && listErr != "":
		r.Problems = append(r.Problems, Problem{Level: Info, Fabric: name, Message: fmt.Sprintf(
			"%s: its standby subnet managers cannot be listed: %s", name, listErr)})
	case len(list) > 0 && ph.Standbys == 0:
		r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
			"%s has one subnet manager and no standby: if it stops, the fabric keeps its routes but cannot recover from any change — a cable, a switch, a node rebooting",
			name)})
	}

	// Its recent failovers: the recorded changes of ports on this fabric,
	// one per minute of changes rather than one per port.
	onFabric := map[string]bool{}
	for _, m := range ph.Members {
		for _, h := range m.HCAs {
			onFabric[m.Node+" "+h] = true
		}
	}
	day := now.Add(-24 * time.Hour)
	for _, c := range changes {
		hca := c.Port
		if i := strings.LastIndexByte(hca, '/'); i >= 0 {
			hca = hca[:i]
		}
		if c.At.Before(day) || !onFabric[c.Node+" "+hca] {
			continue
		}
		if n := len(ph.Changes); n > 0 && ph.Changes[n-1].To == c.To && c.At.Sub(ph.Changes[n-1].At) < 2*time.Minute {
			ph.Changes[n-1].Ports++
			continue
		}
		ph.Changes = append(ph.Changes, Change{At: c.At, From: c.From, To: c.To, Ports: 1})
	}
	slices.Reverse(ph.Changes)
	if len(ph.Changes) > 0 && now.Sub(ph.Changes[0].At) < time.Hour {
		c := ph.Changes[0]
		r.Problems = append(r.Problems, Problem{Level: Warn, Fabric: name, Message: fmt.Sprintf(
			"%s: the master subnet manager changed %s ago, from %s to %s",
			name, now.Sub(c.At).Round(time.Minute), c.From, c.To)})
	}
	_ = nodes
}

func rank(l Level) int {
	switch l {
	case Fail:
		return 0
	case Warn:
		return 1
	}
	return 2
}

func first(s []string) string {
	if len(s) == 0 {
		return "~" // undeclared last
	}
	return s[0]
}

func appendOnce(s []string, v string) []string {
	if slices.Contains(s, v) {
		return s
	}
	return append(s, v)
}

func sortedKeys[V any](m map[string]V) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}