The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.
A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.
Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.
Static assets are named by their content, as the stylesheet already
was.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
391 lines
10 KiB
Go
391 lines
10 KiB
Go
// Package topology reads ibnetdiscover's map of a fabric and works out its
|
|
// shape: which switches are leaves and which spines, which adapter belongs to
|
|
// which node, which links trained below the rest, and whether the rails line
|
|
// up.
|
|
package topology
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Device is a switch, an adapter or a router on the fabric.
|
|
type Device struct {
|
|
GUID string `json:"guid"`
|
|
Kind string `json:"kind"` // switch, ca, router
|
|
Desc string `json:"desc"`
|
|
Ports int `json:"ports"`
|
|
LID int `json:"lid,omitempty"`
|
|
}
|
|
|
|
// End is one side of a link.
|
|
type End struct {
|
|
GUID string `json:"guid"`
|
|
Port int `json:"port"`
|
|
}
|
|
|
|
// Link is one cable, once.
|
|
type Link struct {
|
|
A End `json:"a"`
|
|
B End `json:"b"`
|
|
Width string `json:"width"` // 4x
|
|
Speed string `json:"speed"` // HDR
|
|
Gbps int `json:"gbps"`
|
|
}
|
|
|
|
// Fabric is a parsed ibnetdiscover.
|
|
type Fabric struct {
|
|
Devices map[string]*Device `json:"devices"`
|
|
Links []Link `json:"links"`
|
|
}
|
|
|
|
var (
|
|
header = regexp.MustCompile(`^(Switch|Ca|Rt)\s+(\d+)\s+"([SHR])-([0-9a-fA-F]+)"\s*(?:#\s*"([^"]*)"(.*))?$`)
|
|
portLn = regexp.MustCompile(`^\[(\d+)\](?:\([0-9a-fA-F]+\))?\s+"([SHR])-([0-9a-fA-F]+)"\[(\d+)\](?:\([0-9a-fA-F]+\))?\s*(?:#\s*(.*))?$`)
|
|
lidRe = regexp.MustCompile(`\blid (\d+)`)
|
|
rateRe = regexp.MustCompile(`\b(\d+)x([A-Z]+[0-9]*)\s*$`)
|
|
)
|
|
|
|
// laneGbps is the signalling rate of one lane at each speed, as the link
|
|
// reports it.
|
|
var laneGbps = map[string]float64{
|
|
"SDR": 2.5, "DDR": 5, "QDR": 10, "FDR10": 10, "FDR": 14, "EDR": 25, "HDR": 50, "NDR": 100, "XDR": 200,
|
|
}
|
|
|
|
// Parse reads ibnetdiscover's default output.
|
|
func Parse(out string) (*Fabric, error) {
|
|
f := &Fabric{Devices: map[string]*Device{}}
|
|
seen := map[string]bool{}
|
|
var cur *Device
|
|
for _, raw := range strings.Split(out, "\n") {
|
|
line := strings.TrimSpace(raw)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
if m := header.FindStringSubmatch(line); m != nil {
|
|
kind := map[string]string{"Switch": "switch", "Ca": "ca", "Rt": "router"}[m[1]]
|
|
n, _ := strconv.Atoi(m[2])
|
|
d := &Device{GUID: guid(m[4]), Kind: kind, Ports: n, Desc: m[5]}
|
|
if l := lidRe.FindStringSubmatch(m[6]); l != nil {
|
|
d.LID, _ = strconv.Atoi(l[1])
|
|
}
|
|
if old, ok := f.Devices[d.GUID]; ok && old.Desc != "" && d.Desc == "" {
|
|
d.Desc = old.Desc
|
|
}
|
|
f.Devices[d.GUID] = d
|
|
cur = d
|
|
continue
|
|
}
|
|
m := portLn.FindStringSubmatch(line)
|
|
if m == nil || cur == nil {
|
|
continue
|
|
}
|
|
port, _ := strconv.Atoi(m[1])
|
|
peer := guid(m[3])
|
|
peerPort, _ := strconv.Atoi(m[4])
|
|
comment := m[5]
|
|
// The peer's own block may come later; its description is in the
|
|
// comment, quoted, which is enough to name it until then.
|
|
if _, ok := f.Devices[peer]; !ok {
|
|
kind := map[string]string{"S": "switch", "H": "ca", "R": "router"}[m[2]]
|
|
d := &Device{GUID: peer, Kind: kind}
|
|
if q := strings.Index(comment, `"`); q >= 0 {
|
|
if e := strings.Index(comment[q+1:], `"`); e >= 0 {
|
|
d.Desc = comment[q+1 : q+1+e]
|
|
}
|
|
}
|
|
f.Devices[peer] = d
|
|
}
|
|
a, b := End{cur.GUID, port}, End{peer, peerPort}
|
|
key := linkKey(a, b)
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
l := Link{A: a, B: b}
|
|
if r := rateRe.FindStringSubmatch(comment); r != nil {
|
|
l.Width, l.Speed = r[1]+"x", r[2]
|
|
lanes, _ := strconv.Atoi(r[1])
|
|
l.Gbps = int(float64(lanes) * laneGbps[r[2]])
|
|
}
|
|
f.Links = append(f.Links, l)
|
|
}
|
|
if len(f.Devices) == 0 {
|
|
return nil, fmt.Errorf("no devices in ibnetdiscover output")
|
|
}
|
|
return f, nil
|
|
}
|
|
|
|
func guid(hex string) string {
|
|
hex = strings.ToLower(hex)
|
|
for len(hex) < 16 {
|
|
hex = "0" + hex
|
|
}
|
|
return "0x" + hex
|
|
}
|
|
|
|
func linkKey(a, b End) string {
|
|
x, y := fmt.Sprintf("%s/%d", a.GUID, a.Port), fmt.Sprintf("%s/%d", b.GUID, b.Port)
|
|
if x > y {
|
|
x, y = y, x
|
|
}
|
|
return x + "-" + y
|
|
}
|
|
|
|
// Host is an adapter the cluster knows: which node, which card.
|
|
type Host struct {
|
|
Node string
|
|
HCA string
|
|
}
|
|
|
|
// View is a fabric's shape, ready to draw.
|
|
type View struct {
|
|
Switches []SwitchView `json:"switches"`
|
|
Hosts []HostView `json:"hosts"`
|
|
Links []LinkView `json:"links"`
|
|
// Tiers is how many switch levels: 1 for a single switch, 2 for
|
|
// leaf and spine, 3 with a core.
|
|
Tiers int `json:"tiers"`
|
|
// Expected is the rate most links of each kind trained at; a link below
|
|
// it is degraded.
|
|
ExpectedHost int `json:"expected_host_gbps"`
|
|
ExpectedSwitch int `json:"expected_switch_gbps"`
|
|
Degraded int `json:"degraded"`
|
|
// Rails is, per rail — an adapter's position on its node — the leaves it
|
|
// reaches, and Aligned whether each rail has its own.
|
|
Rails []Rail `json:"rails"`
|
|
Aligned bool `json:"rail_aligned"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// SwitchView is a switch and its place in the tiers.
|
|
type SwitchView struct {
|
|
GUID string `json:"guid"`
|
|
Name string `json:"name"`
|
|
Tier int `json:"tier"` // 1 leaf, 2 spine, 3 core
|
|
Ports int `json:"ports"`
|
|
Used int `json:"used"`
|
|
LID int `json:"lid,omitempty"`
|
|
// Hosts counts adapters hanging off it.
|
|
Hosts int `json:"hosts"`
|
|
}
|
|
|
|
// HostView is an adapter, and the node it is in when the cluster knows it.
|
|
type HostView struct {
|
|
GUID string `json:"guid"`
|
|
Desc string `json:"desc"`
|
|
Node string `json:"node,omitempty"`
|
|
HCA string `json:"hca,omitempty"`
|
|
Rail int `json:"rail"`
|
|
Leaf string `json:"leaf,omitempty"`
|
|
}
|
|
|
|
// LinkView is a cable, with what it trained at and whether that is short of
|
|
// its kind.
|
|
type LinkView struct {
|
|
From string `json:"from"`
|
|
FromPort int `json:"from_port"`
|
|
To string `json:"to"`
|
|
ToPort int `json:"to_port"`
|
|
Rate string `json:"rate"`
|
|
Gbps int `json:"gbps"`
|
|
Host bool `json:"host"`
|
|
Degraded bool `json:"degraded"`
|
|
}
|
|
|
|
// Rail is the leaves one rail's adapters reach.
|
|
type Rail struct {
|
|
Index int `json:"index"`
|
|
Leaves []string `json:"leaves"`
|
|
}
|
|
|
|
// Build works out the shape. hosts maps adapter GUIDs to the cluster's nodes.
|
|
func Build(f *Fabric, hosts map[string]Host) View {
|
|
var v View
|
|
adj := map[string][]string{}
|
|
for _, l := range f.Links {
|
|
adj[l.A.GUID] = append(adj[l.A.GUID], l.B.GUID)
|
|
adj[l.B.GUID] = append(adj[l.B.GUID], l.A.GUID)
|
|
}
|
|
// A switch's tier is its distance from the nearest adapter: leaves hold
|
|
// adapters, spines hold leaves, cores hold spines.
|
|
tier := map[string]int{}
|
|
var frontier []string
|
|
for g, d := range f.Devices {
|
|
if d.Kind == "ca" {
|
|
tier[g] = 0
|
|
frontier = append(frontier, g)
|
|
}
|
|
}
|
|
for len(frontier) > 0 {
|
|
var next []string
|
|
for _, g := range frontier {
|
|
for _, p := range adj[g] {
|
|
if _, ok := tier[p]; ok || f.Devices[p] == nil || f.Devices[p].Kind == "ca" {
|
|
continue
|
|
}
|
|
tier[p] = tier[g] + 1
|
|
next = append(next, p)
|
|
}
|
|
}
|
|
frontier = next
|
|
}
|
|
|
|
name := func(g string) string {
|
|
d := f.Devices[g]
|
|
if d == nil || d.Desc == "" {
|
|
return g
|
|
}
|
|
desc := d.Desc
|
|
// Mellanox switches say "MF0;leaf-01:MSB7700/U1": the name is between.
|
|
if i := strings.Index(desc, ";"); i >= 0 {
|
|
desc = desc[i+1:]
|
|
if j := strings.Index(desc, ":"); j >= 0 {
|
|
desc = desc[:j]
|
|
}
|
|
}
|
|
return desc
|
|
}
|
|
|
|
// Expected rate per kind of link: the most common.
|
|
hostRates, swRates := map[int]int{}, map[int]int{}
|
|
isHost := func(l Link) bool {
|
|
return f.Devices[l.A.GUID].Kind == "ca" || f.Devices[l.B.GUID].Kind == "ca"
|
|
}
|
|
for _, l := range f.Links {
|
|
if isHost(l) {
|
|
hostRates[l.Gbps]++
|
|
} else {
|
|
swRates[l.Gbps]++
|
|
}
|
|
}
|
|
v.ExpectedHost, v.ExpectedSwitch = mode(hostRates), mode(swRates)
|
|
|
|
used := map[string]int{}
|
|
hostsOn := map[string]int{}
|
|
leafOf := map[string]string{}
|
|
for _, l := range f.Links {
|
|
used[l.A.GUID]++
|
|
used[l.B.GUID]++
|
|
lv := LinkView{From: l.A.GUID, FromPort: l.A.Port, To: l.B.GUID, ToPort: l.B.Port,
|
|
Rate: l.Width + l.Speed, Gbps: l.Gbps, Host: isHost(l)}
|
|
// Keep adapters on the To side, so a drawing can go down the tiers.
|
|
if f.Devices[lv.From].Kind == "ca" {
|
|
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
|
|
}
|
|
if lv.Host {
|
|
hostsOn[lv.From]++
|
|
leafOf[lv.To] = lv.From
|
|
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedHost
|
|
} else {
|
|
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedSwitch
|
|
if tier[lv.From] < tier[lv.To] {
|
|
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
|
|
}
|
|
}
|
|
if lv.Degraded {
|
|
v.Degraded++
|
|
}
|
|
v.Links = append(v.Links, lv)
|
|
}
|
|
|
|
for g, d := range f.Devices {
|
|
switch d.Kind {
|
|
case "switch":
|
|
t := tier[g]
|
|
if t == 0 {
|
|
t = 1 // a switch with nothing below it is drawn with the leaves
|
|
}
|
|
v.Switches = append(v.Switches, SwitchView{GUID: g, Name: name(g), Tier: t, Ports: d.Ports,
|
|
Used: used[g], LID: d.LID, Hosts: hostsOn[g]})
|
|
v.Tiers = max(v.Tiers, t)
|
|
case "ca":
|
|
h := HostView{GUID: g, Desc: d.Desc, Leaf: leafOf[g]}
|
|
if known, ok := hosts[g]; ok {
|
|
h.Node, h.HCA = known.Node, known.HCA
|
|
}
|
|
v.Hosts = append(v.Hosts, h)
|
|
}
|
|
}
|
|
sort.Slice(v.Switches, func(i, j int) bool {
|
|
a, b := v.Switches[i], v.Switches[j]
|
|
if a.Tier != b.Tier {
|
|
return a.Tier > b.Tier
|
|
}
|
|
return a.Name < b.Name
|
|
})
|
|
|
|
// Rails: an adapter's position among its node's adapters on this fabric.
|
|
byNode := map[string][]int{}
|
|
for i, h := range v.Hosts {
|
|
key := h.Node
|
|
if key == "" {
|
|
key = strings.Fields(h.Desc + " ?")[0] // "node01 mlx5_0" names its host
|
|
}
|
|
byNode[key] = append(byNode[key], i)
|
|
}
|
|
railLeaves := map[int]map[string]bool{}
|
|
for _, idx := range byNode {
|
|
sort.Slice(idx, func(a, b int) bool {
|
|
return hcaName(v.Hosts[idx[a]]) < hcaName(v.Hosts[idx[b]])
|
|
})
|
|
for r, i := range idx {
|
|
v.Hosts[i].Rail = r
|
|
if railLeaves[r] == nil {
|
|
railLeaves[r] = map[string]bool{}
|
|
}
|
|
if v.Hosts[i].Leaf != "" {
|
|
railLeaves[r][v.Hosts[i].Leaf] = true
|
|
}
|
|
}
|
|
}
|
|
sort.Slice(v.Hosts, func(i, j int) bool {
|
|
a, b := v.Hosts[i], v.Hosts[j]
|
|
if a.Leaf != b.Leaf {
|
|
return a.Leaf < b.Leaf
|
|
}
|
|
return a.Desc < b.Desc
|
|
})
|
|
owner := map[string]int{}
|
|
v.Aligned = len(railLeaves) > 1
|
|
for r := 0; r < len(railLeaves); r++ {
|
|
var leaves []string
|
|
for l := range railLeaves[r] {
|
|
leaves = append(leaves, name(l))
|
|
if prev, ok := owner[l]; ok && prev != r {
|
|
v.Aligned = false
|
|
}
|
|
owner[l] = r
|
|
}
|
|
sort.Strings(leaves)
|
|
v.Rails = append(v.Rails, Rail{Index: r, Leaves: leaves})
|
|
}
|
|
if v.Degraded > 0 {
|
|
v.Notes = append(v.Notes, fmt.Sprintf("%d link(s) trained below the rest of their kind", v.Degraded))
|
|
}
|
|
return v
|
|
}
|
|
|
|
func hcaName(h HostView) string {
|
|
if h.HCA != "" {
|
|
return h.HCA
|
|
}
|
|
return h.Desc
|
|
}
|
|
|
|
func mode(counts map[int]int) int {
|
|
best, n := 0, -1
|
|
for v, c := range counts {
|
|
if v == 0 {
|
|
continue
|
|
}
|
|
if c > n || (c == n && v > best) {
|
|
best, n = v, c
|
|
}
|
|
}
|
|
return best
|
|
}
|