Files
llmgw/internal/control/topology/topology_test.go
T
Ilker ManapandClaude Opus 5.5 5bbc619a31 Map every InfiniBand fabric, draw it, and place replicas by it
The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.

A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.

Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.

Static assets are named by their content, as the stylesheet already
was.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 08:46:23 +02:00

150 lines
4.7 KiB
Go

package topology
import (
"fmt"
"strings"
"testing"
)
// fatTree writes ibnetdiscover's default output for two spines, two leaves
// and four nodes of two adapters each, rail-aligned: every node's mlx5_0 on
// leaf-01 and mlx5_1 on leaf-02. node03's mlx5_1 trained at 2x instead of 4x.
func fatTree() string {
var b strings.Builder
w := func(format string, a ...any) { fmt.Fprintf(&b, format+"\n", a...) }
w("#\n# Topology file: generated on Mon Sep 29 10:00:00 2026\n#\n# Initiated from node 0000000000000a01 port 0000000000000a01\n")
spine := []string{"00000000000000c1", "00000000000000c2"}
leaf := []string{"00000000000000b1", "00000000000000b2"}
hca := func(n, r int) string { return fmt.Sprintf("000000000000%d%02d", r+10, n) }
for si, s := range spine {
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", s, s, s)
w("Switch\t64 \"S-%s\"\t\t# \"MF0;spine-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", s, si+1, 10+si)
for li, l := range leaf {
for k := 0; k < 2; k++ {
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;leaf-0%d:MQM9700/U1\" lid %d 4xNDR", li*2+k+1, l, 33+si*2+k, li+1, 20+li)
}
}
w("")
}
for li, l := range leaf {
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", l, l, l)
w("Switch\t64 \"S-%s\"\t\t# \"MF0;leaf-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", l, li+1, 20+li)
for n := 1; n <= 4; n++ {
rate := "4xNDR"
if n == 3 && li == 1 {
rate = "2xNDR"
}
h := hca(n, li)
w("[%d]\t\"H-%s\"[1](%s) \t\t# \"node0%d mlx5_%d\" lid %d %s", n, h, h, n, li, 100+n*2+li, rate)
}
for si, s := range spine {
for k := 0; k < 2; k++ {
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;spine-0%d:MQM9700/U1\" lid %d 4xNDR", 33+si*2+k, s, li*2+k+1, si+1, 10+si)
}
}
w("")
}
for n := 1; n <= 4; n++ {
for li, l := range leaf {
h := hca(n, li)
rate := "4xNDR"
if n == 3 && li == 1 {
rate = "2xNDR"
}
w("vendid=0x2c9\ndevid=0x1021\nsysimgguid=0x%s\ncaguid=0x%s", h, h)
w("Ca\t1 \"H-%s\"\t\t# \"node0%d mlx5_%d\"", h, n, li)
w("[1](%s) \t\"S-%s\"[%d]\t\t# lid %d lmc 0 \"MF0;leaf-0%d:MQM9700/U1\" lid %d %s", h, l, n, 100+n*2+li, li+1, 20+li, rate)
w("")
}
}
return b.String()
}
func TestAFatTreeIsReadIntoTiers(t *testing.T) {
f, err := Parse(fatTree())
if err != nil {
t.Fatal(err)
}
// 2 spines + 2 leaves + 8 adapters; 8 spine-leaf cables + 8 host cables,
// each once though both ends list it.
if len(f.Devices) != 12 || len(f.Links) != 16 {
t.Fatalf("%d devices, %d links", len(f.Devices), len(f.Links))
}
hosts := map[string]Host{}
for n := 1; n <= 4; n++ {
for r := 0; r < 2; r++ {
hosts[fmt.Sprintf("0x000000000000%d%02d", r+10, n)] = Host{Node: fmt.Sprintf("n%d", n), HCA: fmt.Sprintf("mlx5_%d", r)}
}
}
v := Build(f, hosts)
if v.Tiers != 2 {
t.Errorf("tiers = %d", v.Tiers)
}
tiers := map[string]int{}
for _, s := range v.Switches {
tiers[s.Name] = s.Tier
}
if tiers["spine-01"] != 2 || tiers["leaf-02"] != 1 {
t.Errorf("tiers = %v", tiers)
}
if v.ExpectedHost != 400 || v.ExpectedSwitch != 400 {
t.Errorf("expected rates %d / %d", v.ExpectedHost, v.ExpectedSwitch)
}
// node03's second adapter trained at half width.
if v.Degraded != 1 {
t.Errorf("degraded = %d", v.Degraded)
}
for _, l := range v.Links {
if l.Degraded && (l.Rate != "2xNDR" || !l.Host) {
t.Errorf("wrong link degraded: %+v", l)
}
}
// Every node's first adapter on leaf-01, its second on leaf-02: aligned.
if !v.Aligned || len(v.Rails) != 2 || v.Rails[0].Leaves[0] != "leaf-01" || v.Rails[1].Leaves[0] != "leaf-02" {
t.Errorf("rails = %+v aligned=%v", v.Rails, v.Aligned)
}
mapped := 0
for _, h := range v.Hosts {
if h.Node != "" {
mapped++
}
}
if mapped != 8 {
t.Errorf("%d adapters mapped to nodes", mapped)
}
}
func TestCrossedRailsAreNotAligned(t *testing.T) {
// node02's two cables swapped at the leaves: its mlx5_0 on leaf-02 and
// its mlx5_1 on leaf-01, so each leaf now carries both rails.
f, err := Parse(fatTree())
if err != nil {
t.Fatal(err)
}
swap := map[string]string{"0x0000000000001002": "0x00000000000000b2", "0x0000000000001102": "0x00000000000000b1"}
for i, l := range f.Links {
for _, host := range []*End{&f.Links[i].A, &f.Links[i].B} {
if leaf, ok := swap[host.GUID]; ok {
other := &f.Links[i].A
if host == other {
other = &f.Links[i].B
}
other.GUID = leaf
}
}
_ = l
}
v := Build(f, map[string]Host{
"0x0000000000001002": {Node: "n2", HCA: "mlx5_0"}, "0x0000000000001102": {Node: "n2", HCA: "mlx5_1"},
})
if v.Aligned {
t.Errorf("crossed rails read as aligned: %+v", v.Rails)
}
}
func TestNotAMapIsAnError(t *testing.T) {
if _, err := Parse("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port ((null):0)"); err == nil {
t.Error("expected an error")
}
}