// Package topology reads ibnetdiscover's map of a fabric and works out its // shape: which switches are leaves and which spines, which adapter belongs to // which node, which links trained below the rest, and whether the rails line // up. package topology import ( "fmt" "regexp" "sort" "strconv" "strings" ) // Device is a switch, an adapter or a router on the fabric. type Device struct { GUID string `json:"guid"` Kind string `json:"kind"` // switch, ca, router Desc string `json:"desc"` Ports int `json:"ports"` LID int `json:"lid,omitempty"` } // End is one side of a link. type End struct { GUID string `json:"guid"` Port int `json:"port"` } // Link is one cable, once. type Link struct { A End `json:"a"` B End `json:"b"` Width string `json:"width"` // 4x Speed string `json:"speed"` // HDR Gbps int `json:"gbps"` } // Fabric is a parsed ibnetdiscover. type Fabric struct { Devices map[string]*Device `json:"devices"` Links []Link `json:"links"` } var ( header = regexp.MustCompile(`^(Switch|Ca|Rt)\s+(\d+)\s+"([SHR])-([0-9a-fA-F]+)"\s*(?:#\s*"([^"]*)"(.*))?$`) portLn = regexp.MustCompile(`^\[(\d+)\](?:\([0-9a-fA-F]+\))?\s+"([SHR])-([0-9a-fA-F]+)"\[(\d+)\](?:\([0-9a-fA-F]+\))?\s*(?:#\s*(.*))?$`) lidRe = regexp.MustCompile(`\blid (\d+)`) rateRe = regexp.MustCompile(`\b(\d+)x([A-Z]+[0-9]*)\s*$`) ) // laneGbps is the signalling rate of one lane at each speed, as the link // reports it. var laneGbps = map[string]float64{ "SDR": 2.5, "DDR": 5, "QDR": 10, "FDR10": 10, "FDR": 14, "EDR": 25, "HDR": 50, "NDR": 100, "XDR": 200, } // Parse reads ibnetdiscover's default output. func Parse(out string) (*Fabric, error) { f := &Fabric{Devices: map[string]*Device{}} seen := map[string]bool{} var cur *Device for _, raw := range strings.Split(out, "\n") { line := strings.TrimSpace(raw) if line == "" || strings.HasPrefix(line, "#") { continue } if m := header.FindStringSubmatch(line); m != nil { kind := map[string]string{"Switch": "switch", "Ca": "ca", "Rt": "router"}[m[1]] n, _ := strconv.Atoi(m[2]) d := &Device{GUID: guid(m[4]), Kind: kind, Ports: n, Desc: m[5]} if l := lidRe.FindStringSubmatch(m[6]); l != nil { d.LID, _ = strconv.Atoi(l[1]) } if old, ok := f.Devices[d.GUID]; ok && old.Desc != "" && d.Desc == "" { d.Desc = old.Desc } f.Devices[d.GUID] = d cur = d continue } m := portLn.FindStringSubmatch(line) if m == nil || cur == nil { continue } port, _ := strconv.Atoi(m[1]) peer := guid(m[3]) peerPort, _ := strconv.Atoi(m[4]) comment := m[5] // The peer's own block may come later; its description is in the // comment, quoted, which is enough to name it until then. if _, ok := f.Devices[peer]; !ok { kind := map[string]string{"S": "switch", "H": "ca", "R": "router"}[m[2]] d := &Device{GUID: peer, Kind: kind} if q := strings.Index(comment, `"`); q >= 0 { if e := strings.Index(comment[q+1:], `"`); e >= 0 { d.Desc = comment[q+1 : q+1+e] } } f.Devices[peer] = d } a, b := End{cur.GUID, port}, End{peer, peerPort} key := linkKey(a, b) if seen[key] { continue } seen[key] = true l := Link{A: a, B: b} if r := rateRe.FindStringSubmatch(comment); r != nil { l.Width, l.Speed = r[1]+"x", r[2] lanes, _ := strconv.Atoi(r[1]) l.Gbps = int(float64(lanes) * laneGbps[r[2]]) } f.Links = append(f.Links, l) } if len(f.Devices) == 0 { return nil, fmt.Errorf("no devices in ibnetdiscover output") } return f, nil } func guid(hex string) string { hex = strings.ToLower(hex) for len(hex) < 16 { hex = "0" + hex } return "0x" + hex } func linkKey(a, b End) string { x, y := fmt.Sprintf("%s/%d", a.GUID, a.Port), fmt.Sprintf("%s/%d", b.GUID, b.Port) if x > y { x, y = y, x } return x + "-" + y } // Host is an adapter the cluster knows: which node, which card. type Host struct { Node string HCA string } // View is a fabric's shape, ready to draw. type View struct { Switches []SwitchView `json:"switches"` Hosts []HostView `json:"hosts"` Links []LinkView `json:"links"` // Tiers is how many switch levels: 1 for a single switch, 2 for // leaf and spine, 3 with a core. Tiers int `json:"tiers"` // Expected is the rate most links of each kind trained at; a link below // it is degraded. ExpectedHost int `json:"expected_host_gbps"` ExpectedSwitch int `json:"expected_switch_gbps"` Degraded int `json:"degraded"` // Rails is, per rail — an adapter's position on its node — the leaves it // reaches, and Aligned whether each rail has its own. Rails []Rail `json:"rails"` Aligned bool `json:"rail_aligned"` Notes []string `json:"notes,omitempty"` } // SwitchView is a switch and its place in the tiers. type SwitchView struct { GUID string `json:"guid"` Name string `json:"name"` Tier int `json:"tier"` // 1 leaf, 2 spine, 3 core Ports int `json:"ports"` Used int `json:"used"` LID int `json:"lid,omitempty"` // Hosts counts adapters hanging off it. Hosts int `json:"hosts"` } // HostView is an adapter, and the node it is in when the cluster knows it. type HostView struct { GUID string `json:"guid"` Desc string `json:"desc"` Node string `json:"node,omitempty"` HCA string `json:"hca,omitempty"` Rail int `json:"rail"` Leaf string `json:"leaf,omitempty"` } // LinkView is a cable, with what it trained at and whether that is short of // its kind. type LinkView struct { From string `json:"from"` FromPort int `json:"from_port"` To string `json:"to"` ToPort int `json:"to_port"` Rate string `json:"rate"` Gbps int `json:"gbps"` Host bool `json:"host"` Degraded bool `json:"degraded"` } // Rail is the leaves one rail's adapters reach. type Rail struct { Index int `json:"index"` Leaves []string `json:"leaves"` } // Build works out the shape. hosts maps adapter GUIDs to the cluster's nodes. func Build(f *Fabric, hosts map[string]Host) View { var v View adj := map[string][]string{} for _, l := range f.Links { adj[l.A.GUID] = append(adj[l.A.GUID], l.B.GUID) adj[l.B.GUID] = append(adj[l.B.GUID], l.A.GUID) } // A switch's tier is its distance from the nearest adapter: leaves hold // adapters, spines hold leaves, cores hold spines. tier := map[string]int{} var frontier []string for g, d := range f.Devices { if d.Kind == "ca" { tier[g] = 0 frontier = append(frontier, g) } } for len(frontier) > 0 { var next []string for _, g := range frontier { for _, p := range adj[g] { if _, ok := tier[p]; ok || f.Devices[p] == nil || f.Devices[p].Kind == "ca" { continue } tier[p] = tier[g] + 1 next = append(next, p) } } frontier = next } name := func(g string) string { d := f.Devices[g] if d == nil || d.Desc == "" { return g } desc := d.Desc // Mellanox switches say "MF0;leaf-01:MSB7700/U1": the name is between. if i := strings.Index(desc, ";"); i >= 0 { desc = desc[i+1:] if j := strings.Index(desc, ":"); j >= 0 { desc = desc[:j] } } return desc } // Expected rate per kind of link: the most common. hostRates, swRates := map[int]int{}, map[int]int{} isHost := func(l Link) bool { return f.Devices[l.A.GUID].Kind == "ca" || f.Devices[l.B.GUID].Kind == "ca" } for _, l := range f.Links { if isHost(l) { hostRates[l.Gbps]++ } else { swRates[l.Gbps]++ } } v.ExpectedHost, v.ExpectedSwitch = mode(hostRates), mode(swRates) used := map[string]int{} hostsOn := map[string]int{} leafOf := map[string]string{} for _, l := range f.Links { used[l.A.GUID]++ used[l.B.GUID]++ lv := LinkView{From: l.A.GUID, FromPort: l.A.Port, To: l.B.GUID, ToPort: l.B.Port, Rate: l.Width + l.Speed, Gbps: l.Gbps, Host: isHost(l)} // Keep adapters on the To side, so a drawing can go down the tiers. if f.Devices[lv.From].Kind == "ca" { lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort } if lv.Host { hostsOn[lv.From]++ leafOf[lv.To] = lv.From lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedHost } else { lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedSwitch if tier[lv.From] < tier[lv.To] { lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort } } if lv.Degraded { v.Degraded++ } v.Links = append(v.Links, lv) } for g, d := range f.Devices { switch d.Kind { case "switch": t := tier[g] if t == 0 { t = 1 // a switch with nothing below it is drawn with the leaves } v.Switches = append(v.Switches, SwitchView{GUID: g, Name: name(g), Tier: t, Ports: d.Ports, Used: used[g], LID: d.LID, Hosts: hostsOn[g]}) v.Tiers = max(v.Tiers, t) case "ca": h := HostView{GUID: g, Desc: d.Desc, Leaf: leafOf[g]} if known, ok := hosts[g]; ok { h.Node, h.HCA = known.Node, known.HCA } v.Hosts = append(v.Hosts, h) } } sort.Slice(v.Switches, func(i, j int) bool { a, b := v.Switches[i], v.Switches[j] if a.Tier != b.Tier { return a.Tier > b.Tier } return a.Name < b.Name }) // Rails: an adapter's position among its node's adapters on this fabric. byNode := map[string][]int{} for i, h := range v.Hosts { key := h.Node if key == "" { key = strings.Fields(h.Desc + " ?")[0] // "node01 mlx5_0" names its host } byNode[key] = append(byNode[key], i) } railLeaves := map[int]map[string]bool{} for _, idx := range byNode { sort.Slice(idx, func(a, b int) bool { return hcaName(v.Hosts[idx[a]]) < hcaName(v.Hosts[idx[b]]) }) for r, i := range idx { v.Hosts[i].Rail = r if railLeaves[r] == nil { railLeaves[r] = map[string]bool{} } if v.Hosts[i].Leaf != "" { railLeaves[r][v.Hosts[i].Leaf] = true } } } sort.Slice(v.Hosts, func(i, j int) bool { a, b := v.Hosts[i], v.Hosts[j] if a.Leaf != b.Leaf { return a.Leaf < b.Leaf } return a.Desc < b.Desc }) owner := map[string]int{} v.Aligned = len(railLeaves) > 1 for r := 0; r < len(railLeaves); r++ { var leaves []string for l := range railLeaves[r] { leaves = append(leaves, name(l)) if prev, ok := owner[l]; ok && prev != r { v.Aligned = false } owner[l] = r } sort.Strings(leaves) v.Rails = append(v.Rails, Rail{Index: r, Leaves: leaves}) } if v.Degraded > 0 { v.Notes = append(v.Notes, fmt.Sprintf("%d link(s) trained below the rest of their kind", v.Degraded)) } return v } func hcaName(h HostView) string { if h.HCA != "" { return h.HCA } return h.Desc } func mode(counts map[int]int) int { best, n := 0, -1 for v, c := range counts { if v == 0 { continue } if c > n || (c == n && v > best) { best, n = v, c } } return best }