Map every InfiniBand fabric, draw it, and place replicas by it
The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.
A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.
Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.
Static assets are named by their content, as the stylesheet already
was.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
23648fa19a
commit
5bbc619a31
23 files changed
+2171
-83
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -52,6 +53,14 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Fabrics())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := a.Topology(r.PathValue("id"))
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, t)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/models", a.listModels)
|
||||
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
|
||||
// Listing is a read and deleting is a separate verb, so a mistyped
|
||||
@@ -839,3 +848,114 @@ func (a *API) Fabrics() fabrics.Report {
|
||||
inv := a.ctl.Inventory()
|
||||
return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now())
|
||||
}
|
||||
|
||||
// TopologyView is one fabric's map, with the cluster's own adapters on it
|
||||
// live: their state, rate and traffic from the last heartbeat.
|
||||
type TopologyView struct {
|
||||
ID string `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Declared []string `json:"declared"`
|
||||
ScannedAt time.Time `json:"scanned_at"`
|
||||
ScannedBy string `json:"scanned_by,omitempty"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
Err string `json:"error,omitempty"`
|
||||
Map *topology.View `json:"map,omitempty"`
|
||||
Live map[string]Live `json:"live"`
|
||||
// Problems are the fabric's own, from the analysis of its identity and
|
||||
// subnet managers.
|
||||
Problems []fabrics.Problem `json:"problems,omitempty"`
|
||||
SM *fabrics.SM `json:"sm,omitempty"`
|
||||
SMs []fabrics.SM `json:"sms,omitempty"`
|
||||
}
|
||||
|
||||
// Live is one of the cluster's adapters as its node last reported it.
|
||||
type Live struct {
|
||||
Node string `json:"node"`
|
||||
HCA string `json:"hca"`
|
||||
State string `json:"state"`
|
||||
Phys string `json:"phys_state"`
|
||||
Gbps int `json:"gbps"`
|
||||
TxBps float64 `json:"tx_bps"`
|
||||
RxBps float64 `json:"rx_bps"`
|
||||
Errors map[string]uint64 `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// FabricID is a physical fabric's key as an address can carry it.
|
||||
func FabricID(key string) string {
|
||||
return strings.NewReplacer(":", "-", "/", "_").Replace(key)
|
||||
}
|
||||
|
||||
// Topologies lists every physical fabric with what is known of its map.
|
||||
func (a *API) Topologies() []TopologyView {
|
||||
rep := a.Fabrics()
|
||||
scans := a.ctl.Inventory().Scans()
|
||||
out := make([]TopologyView, 0, len(rep.Physical))
|
||||
for _, ph := range rep.Physical {
|
||||
t := TopologyView{ID: FabricID(ph.Key), Key: ph.Key, Label: ph.Label, Declared: ph.Declared}
|
||||
if sc, ok := scans[ph.Key]; ok {
|
||||
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Topology is one fabric's map, by the id Topologies gives it.
|
||||
func (a *API) Topology(id string) (TopologyView, bool) {
|
||||
rep := a.Fabrics()
|
||||
var ph *fabrics.Physical
|
||||
for i := range rep.Physical {
|
||||
if FabricID(rep.Physical[i].Key) == id {
|
||||
ph = &rep.Physical[i]
|
||||
}
|
||||
}
|
||||
if ph == nil {
|
||||
return TopologyView{}, false
|
||||
}
|
||||
t := TopologyView{ID: id, Key: ph.Key, Label: ph.Label, Declared: ph.Declared, Live: map[string]Live{},
|
||||
SM: ph.SM, SMs: ph.SMs}
|
||||
for _, p := range rep.Problems {
|
||||
for _, d := range ph.Declared {
|
||||
if p.Fabric == d {
|
||||
t.Problems = append(t.Problems, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
hosts := map[string]topology.Host{}
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.NodeGUID == "" {
|
||||
continue
|
||||
}
|
||||
hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA}
|
||||
l := Live{Node: n.ID, HCA: p.HCA, State: p.State, Phys: p.PhysState, Gbps: p.RateGbps,
|
||||
TxBps: p.TxBps, RxBps: p.RxBps}
|
||||
for name, c := range p.Counters {
|
||||
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
|
||||
if l.Errors == nil {
|
||||
l.Errors = map[string]uint64{}
|
||||
}
|
||||
l.Errors[name] = c
|
||||
}
|
||||
}
|
||||
t.Live[p.NodeGUID] = l
|
||||
}
|
||||
}
|
||||
sc, ok := a.ctl.Inventory().Scans()[ph.Key]
|
||||
if !ok {
|
||||
return t, true
|
||||
}
|
||||
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
|
||||
if sc.Output != "" {
|
||||
f, err := topology.Parse(sc.Output)
|
||||
if err != nil {
|
||||
t.Err = "the map could not be read: " + err.Error()
|
||||
return t, true
|
||||
}
|
||||
v := topology.Build(f, hosts)
|
||||
t.Map = &v
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -58,6 +59,8 @@ type Service struct {
|
||||
logMu sync.Mutex
|
||||
logWaits map[string]chan *llmgwv1.LogsReply
|
||||
logSeq uint64
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
}
|
||||
|
||||
// SetEngines sets the engines every node installs, by name and version.
|
||||
@@ -282,6 +285,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
"last_output", m.Shard.GetLastOutput())
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricScan:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
|
||||
delete(s.scanWaits, m.FabricScan.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.FabricScan
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Logs:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||
@@ -406,9 +418,13 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
}
|
||||
if ports := hb.GetIbPorts(); len(ports) > 0 {
|
||||
before := map[string]string{}
|
||||
prev := map[string]inventory.IBPort{}
|
||||
for _, p := range n.IBPorts {
|
||||
before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID
|
||||
prev[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p
|
||||
}
|
||||
now := time.Now()
|
||||
secs := now.Sub(n.IBAt).Seconds()
|
||||
n.IBPorts = nil
|
||||
for _, p := range ports {
|
||||
// A different master than last time is a failover, or a port
|
||||
@@ -427,7 +443,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
}
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{
|
||||
SMs: sms, SMsError: p.GetSubnetManagersError(),
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
TxBps: rate(prev[where].Counters["port_xmit_data"], p.GetCounters()["port_xmit_data"], secs),
|
||||
RxBps: rate(prev[where].Counters["port_rcv_data"], p.GetCounters()["port_rcv_data"], secs),
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
|
||||
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
|
||||
NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(),
|
||||
@@ -436,6 +454,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(hb.GetIbPorts()) > 0 {
|
||||
n.IBAt = time.Now()
|
||||
}
|
||||
// A port went down, or came back. Placement reads the fabrics, so this is
|
||||
// what stops a node being offered multi-node work it can no longer carry.
|
||||
if hb.GetFabricsChanged() {
|
||||
@@ -642,3 +663,102 @@ func (s *Service) Logs(ctx context.Context, nodeID, source string, lines int) ([
|
||||
return nil, nil, fmt.Errorf("node %s did not answer; an agent older than v0.8.4 cannot", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabric asks a node to map the fabric one of its ports is on.
|
||||
func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int) (*llmgwv1.FabricScanReply, error) {
|
||||
s.logMu.Lock()
|
||||
if s.scanWaits == nil {
|
||||
s.scanWaits = map[string]chan *llmgwv1.FabricScanReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("scan-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.FabricScanReply, 1)
|
||||
s.scanWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.scanWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricScan{
|
||||
FabricScan: &llmgwv1.FabricScanRequest{RequestId: id, Hca: hca, Port: int32(port)}, //nolint:gosec // a port number
|
||||
}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r, nil
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("node %s did not answer the fabric scan", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
|
||||
// one node on each: the first, by name, whose port on it is up. A fabric
|
||||
// looks the same from all its ports, so one is enough, and the rest of its
|
||||
// nodes are left to their work.
|
||||
func (s *Service) ScanFabrics(ctx context.Context, every time.Duration) {
|
||||
if every <= 0 {
|
||||
every = 5 * time.Minute
|
||||
}
|
||||
// Soon after a start, once agents have sent a heartbeat or two.
|
||||
timer := time.NewTimer(30 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
s.scanOnce(ctx)
|
||||
timer.Reset(every)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) scanOnce(ctx context.Context) {
|
||||
nodes := s.inv.Nodes()
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID })
|
||||
canon := inventory.CanonicalSMs(nodes)
|
||||
chosen := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
if n.State != inventory.Ready {
|
||||
continue
|
||||
}
|
||||
for _, p := range n.IBPorts {
|
||||
key := p.FabricKeyIn(canon)
|
||||
if key == "" || p.RoCE() || chosen[key] {
|
||||
continue
|
||||
}
|
||||
chosen[key] = true
|
||||
where := fmt.Sprintf("%s/%d", p.HCA, p.Port)
|
||||
r, err := s.ScanFabric(ctx, n.ID, p.HCA, p.Port)
|
||||
sc := inventory.FabricScan{Key: key, At: time.Now(), Node: n.ID, Port: where}
|
||||
switch {
|
||||
case err != nil:
|
||||
sc.Err = err.Error()
|
||||
case r.GetError() != "":
|
||||
sc.Err = r.GetError()
|
||||
default:
|
||||
sc.Output = r.GetOutput()
|
||||
}
|
||||
if r != nil {
|
||||
sc.Took = time.Duration(r.GetTookMs()) * time.Millisecond
|
||||
}
|
||||
if sc.Err != "" {
|
||||
s.log.Warn("could not map a fabric", "fabric", key, "node", n.ID, "port", where, "err", sc.Err)
|
||||
}
|
||||
s.inv.SetScan(sc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rate is a counter's increase a second, zero when there is no earlier
|
||||
// reading or the counter went backwards — an adapter reset starts it again.
|
||||
func rate(before, now uint64, secs float64) float64 {
|
||||
if before == 0 || now < before || secs <= 0 || secs > 120 {
|
||||
return 0
|
||||
}
|
||||
return float64(now-before) / secs
|
||||
}
|
||||
@@ -118,6 +118,8 @@ type Node struct {
|
||||
// IBPorts are the node's InfiniBand and RoCE ports at the last heartbeat,
|
||||
// with their counters. Not persisted, for the same reason as Load.
|
||||
IBPorts []IBPort `json:"-"`
|
||||
// IBAt is when IBPorts was read, for the rates.
|
||||
IBAt time.Time `json:"-"`
|
||||
}
|
||||
|
||||
// IBPort is one port's state, identity and cumulative counters.
|
||||
@@ -146,6 +148,9 @@ type IBPort struct {
|
||||
// on the one port per fabric that asked; SMsError says why it could not.
|
||||
SMs []SubnetManager
|
||||
SMsError string
|
||||
// TxBps and RxBps are bytes a second, from this heartbeat's data
|
||||
// counters against the last one's; zero on the first.
|
||||
TxBps, RxBps float64
|
||||
}
|
||||
|
||||
// SubnetManager is one subnet manager on a fabric.
|
||||
@@ -357,6 +362,8 @@ type Store struct {
|
||||
allocs map[string][]Allocation
|
||||
// smChanges are ports' changes of master subnet manager, newest last.
|
||||
smChanges []SMChange
|
||||
// scans are the latest ibnetdiscover of each physical fabric, by key.
|
||||
scans map[string]FabricScan
|
||||
}
|
||||
|
||||
// NewStore returns an empty inventory.
|
||||
@@ -378,6 +385,43 @@ func (s *Store) RecordSMChange(c SMChange) {
|
||||
}
|
||||
}
|
||||
|
||||
// FabricScan is one ibnetdiscover of a physical fabric: its text, and who
|
||||
// ran it when.
|
||||
type FabricScan struct {
|
||||
Key string
|
||||
At time.Time
|
||||
Node string
|
||||
Port string
|
||||
Output string
|
||||
Err string
|
||||
Took time.Duration
|
||||
}
|
||||
|
||||
// SetScan keeps a fabric's latest scan. A failed one does not replace a
|
||||
// good one's text: a map from ten minutes ago is worth more than none.
|
||||
func (s *Store) SetScan(sc FabricScan) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.scans == nil {
|
||||
s.scans = map[string]FabricScan{}
|
||||
}
|
||||
if sc.Err != "" {
|
||||
if old, ok := s.scans[sc.Key]; ok && old.Output != "" {
|
||||
old.Err = sc.Err
|
||||
s.scans[sc.Key] = old
|
||||
return
|
||||
}
|
||||
}
|
||||
s.scans[sc.Key] = sc
|
||||
}
|
||||
|
||||
// Scans returns the latest scan of every fabric.
|
||||
func (s *Store) Scans() map[string]FabricScan {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return maps.Clone(s.scans)
|
||||
}
|
||||
|
||||
// SMChanges returns the recorded changes, oldest first.
|
||||
func (s *Store) SMChanges() []SMChange {
|
||||
s.mu.Lock()
|
||||
|
||||
@@ -333,6 +333,9 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return len(candidates[i].free) > len(candidates[j].free)
|
||||
})
|
||||
// Then by the fabric's map, where there is one: one leaf's nodes
|
||||
// together, a degraded cable last.
|
||||
candidates = byLeaf(candidates, hints(s.inv))
|
||||
|
||||
assignments := make([]NodeAssignment, 0, p.Nodes)
|
||||
for rank, c := range candidates[:p.Nodes] {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
@@ -970,3 +971,50 @@ func TestAFailoverDoesNotSplitAFabricForPlacement(t *testing.T) {
|
||||
t.Fatalf("a failover split the fabric:\n%s", res.Explain())
|
||||
}
|
||||
}
|
||||
|
||||
// scanOf is ibnetdiscover's output for two leaves: leaf-a holding the first
|
||||
// two nodes' adapters, leaf-b the other two. degraded names a node whose
|
||||
// cable trained at half width.
|
||||
func scanOf(ids []string, degraded string) string {
|
||||
var b strings.Builder
|
||||
for li, leaf := range []string{"00000000000000a1", "00000000000000a2"} {
|
||||
fmt.Fprintf(&b, "Switch\t36 \"S-%s\"\t\t# \"MF0;leaf-%c:QM8700/U1\" enhanced port 0 lid %d lmc 0\n", leaf, 'a'+li, li+1)
|
||||
for k := 0; k < 2; k++ {
|
||||
id := ids[li*2+k]
|
||||
rate := "4xHDR"
|
||||
if id == degraded {
|
||||
rate = "2xHDR"
|
||||
}
|
||||
fmt.Fprintf(&b, "[%d]\t\"H-%s\"[1](%s) \t\t# \"%s mlx5_0\" lid %d %s\n", k+1, guidOf(id), guidOf(id), id, 10+li*2+k, rate)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func guidOf(id string) string { return fmt.Sprintf("%016x", []byte(id)[len(id)-1]) }
|
||||
|
||||
func TestAReplicaStaysUnderOneLeafAndOffDegradedCables(t *testing.T) {
|
||||
ids := []string{"n1", "n2", "n3", "n4"}
|
||||
var nodes []inventory.Node
|
||||
for _, id := range ids {
|
||||
n := onSM(node(id, "h100-80g", 8, "ib-a"), "0x01")
|
||||
n.IBPorts[0].NodeGUID = "0x" + guidOf(id)
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
st := store(nodes...)
|
||||
// n1 sits on leaf-a with n2, but its cable trained at half width: leaf-a
|
||||
// has one good node, leaf-b two.
|
||||
st.SetScan(inventory.FabricScan{Key: "sm:0x01", At: time.Now(), Output: scanOf(ids, "n1")})
|
||||
res := New(st).Place([]Profile{{Name: "pair", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
|
||||
if res.Placement == nil {
|
||||
t.Fatalf("no placement:\n%s", res.Explain())
|
||||
}
|
||||
var got []string
|
||||
for _, a := range res.Placement.Nodes {
|
||||
got = append(got, a.NodeID)
|
||||
}
|
||||
if !slices.Equal(got, []string{"n3", "n4"}) {
|
||||
t.Errorf("placed on %v, want the two good nodes under leaf-b", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package placement
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
)
|
||||
|
||||
// fabricHints is what a fabric's map says about placing on it: the leaf
|
||||
// switch each node's first adapter hangs off, and which nodes have a cable
|
||||
// that trained below the rest.
|
||||
type fabricHints struct {
|
||||
leaf map[string]string
|
||||
degraded map[string]bool
|
||||
}
|
||||
|
||||
// hintCache keeps a parsed map until its fabric is scanned again; parsing
|
||||
// thousands of lines on every placement would be the scheduler's main cost.
|
||||
var hintCache struct {
|
||||
sync.Mutex
|
||||
at map[string]time.Time
|
||||
hints map[string]fabricHints
|
||||
}
|
||||
|
||||
// hints reads every scanned fabric. Nodes on no mapped fabric are absent,
|
||||
// which leaves them placed as before.
|
||||
func hints(inv *inventory.Store) fabricHints {
|
||||
out := fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}}
|
||||
scans := inv.Scans()
|
||||
if len(scans) == 0 {
|
||||
return out
|
||||
}
|
||||
var hosts map[string]topology.Host
|
||||
hintCache.Lock()
|
||||
defer hintCache.Unlock()
|
||||
if hintCache.at == nil {
|
||||
hintCache.at, hintCache.hints = map[string]time.Time{}, map[string]fabricHints{}
|
||||
}
|
||||
for key, sc := range scans {
|
||||
if sc.Output == "" {
|
||||
continue
|
||||
}
|
||||
h, ok := hintCache.hints[key]
|
||||
if !ok || !hintCache.at[key].Equal(sc.At) {
|
||||
if hosts == nil {
|
||||
hosts = map[string]topology.Host{}
|
||||
for _, n := range inv.Nodes() {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.NodeGUID != "" {
|
||||
hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
f, err := topology.Parse(sc.Output)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
v := topology.Build(f, hosts)
|
||||
h = fabricHints{leaf: map[string]string{}, degraded: map[string]bool{}}
|
||||
guidNode := map[string]string{}
|
||||
for _, hv := range v.Hosts {
|
||||
if hv.Node == "" {
|
||||
continue
|
||||
}
|
||||
guidNode[hv.GUID] = hv.Node
|
||||
if hv.Rail == 0 {
|
||||
h.leaf[hv.Node] = hv.Leaf
|
||||
}
|
||||
}
|
||||
for _, l := range v.Links {
|
||||
if l.Host && l.Degraded {
|
||||
if n := guidNode[l.To]; n != "" {
|
||||
h.degraded[n] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
hintCache.hints[key], hintCache.at[key] = h, sc.At
|
||||
}
|
||||
for n, l := range h.leaf {
|
||||
out.leaf[n] = l
|
||||
}
|
||||
for n := range h.degraded {
|
||||
out.degraded[n] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// byLeaf orders candidates for a replica spanning nodes: those whose cables
|
||||
// all trained at their rate first, and among them the nodes of one leaf
|
||||
// together — the leaf with the most of them first — so a replica's traffic
|
||||
// stays under one switch when it can, and crosses the spines only when it
|
||||
// must. Within a leaf the order given (emptiest first) is kept.
|
||||
func byLeaf(cands []candidate, h fabricHints) []candidate {
|
||||
if len(h.leaf) == 0 && len(h.degraded) == 0 {
|
||||
return cands
|
||||
}
|
||||
count := map[string]int{}
|
||||
for _, c := range cands {
|
||||
if !h.degraded[c.node.ID] {
|
||||
count[h.leaf[c.node.ID]]++
|
||||
}
|
||||
}
|
||||
firstSeen := map[string]int{}
|
||||
for i, c := range cands {
|
||||
if _, ok := firstSeen[h.leaf[c.node.ID]]; !ok {
|
||||
firstSeen[h.leaf[c.node.ID]] = i
|
||||
}
|
||||
}
|
||||
out := make([]candidate, len(cands))
|
||||
copy(out, cands)
|
||||
rank := func(c candidate) (bad int, size int, first int) {
|
||||
if h.degraded[c.node.ID] {
|
||||
bad = 1
|
||||
}
|
||||
l := h.leaf[c.node.ID]
|
||||
return bad, count[l], firstSeen[l]
|
||||
}
|
||||
slices.SortStableFunc(out, func(a, b candidate) int {
|
||||
ba, sa, fa := rank(a)
|
||||
bb, sb, fb := rank(b)
|
||||
if ba != bb {
|
||||
return ba - bb
|
||||
}
|
||||
if sa != sb {
|
||||
return sb - sa
|
||||
}
|
||||
return fa - fb
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
// Package topology reads ibnetdiscover's map of a fabric and works out its
|
||||
// shape: which switches are leaves and which spines, which adapter belongs to
|
||||
// which node, which links trained below the rest, and whether the rails line
|
||||
// up.
|
||||
package topology
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Device is a switch, an adapter or a router on the fabric.
|
||||
type Device struct {
|
||||
GUID string `json:"guid"`
|
||||
Kind string `json:"kind"` // switch, ca, router
|
||||
Desc string `json:"desc"`
|
||||
Ports int `json:"ports"`
|
||||
LID int `json:"lid,omitempty"`
|
||||
}
|
||||
|
||||
// End is one side of a link.
|
||||
type End struct {
|
||||
GUID string `json:"guid"`
|
||||
Port int `json:"port"`
|
||||
}
|
||||
|
||||
// Link is one cable, once.
|
||||
type Link struct {
|
||||
A End `json:"a"`
|
||||
B End `json:"b"`
|
||||
Width string `json:"width"` // 4x
|
||||
Speed string `json:"speed"` // HDR
|
||||
Gbps int `json:"gbps"`
|
||||
}
|
||||
|
||||
// Fabric is a parsed ibnetdiscover.
|
||||
type Fabric struct {
|
||||
Devices map[string]*Device `json:"devices"`
|
||||
Links []Link `json:"links"`
|
||||
}
|
||||
|
||||
var (
|
||||
header = regexp.MustCompile(`^(Switch|Ca|Rt)\s+(\d+)\s+"([SHR])-([0-9a-fA-F]+)"\s*(?:#\s*"([^"]*)"(.*))?$`)
|
||||
portLn = regexp.MustCompile(`^\[(\d+)\](?:\([0-9a-fA-F]+\))?\s+"([SHR])-([0-9a-fA-F]+)"\[(\d+)\](?:\([0-9a-fA-F]+\))?\s*(?:#\s*(.*))?$`)
|
||||
lidRe = regexp.MustCompile(`\blid (\d+)`)
|
||||
rateRe = regexp.MustCompile(`\b(\d+)x([A-Z]+[0-9]*)\s*$`)
|
||||
)
|
||||
|
||||
// laneGbps is the signalling rate of one lane at each speed, as the link
|
||||
// reports it.
|
||||
var laneGbps = map[string]float64{
|
||||
"SDR": 2.5, "DDR": 5, "QDR": 10, "FDR10": 10, "FDR": 14, "EDR": 25, "HDR": 50, "NDR": 100, "XDR": 200,
|
||||
}
|
||||
|
||||
// Parse reads ibnetdiscover's default output.
|
||||
func Parse(out string) (*Fabric, error) {
|
||||
f := &Fabric{Devices: map[string]*Device{}}
|
||||
seen := map[string]bool{}
|
||||
var cur *Device
|
||||
for _, raw := range strings.Split(out, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
if m := header.FindStringSubmatch(line); m != nil {
|
||||
kind := map[string]string{"Switch": "switch", "Ca": "ca", "Rt": "router"}[m[1]]
|
||||
n, _ := strconv.Atoi(m[2])
|
||||
d := &Device{GUID: guid(m[4]), Kind: kind, Ports: n, Desc: m[5]}
|
||||
if l := lidRe.FindStringSubmatch(m[6]); l != nil {
|
||||
d.LID, _ = strconv.Atoi(l[1])
|
||||
}
|
||||
if old, ok := f.Devices[d.GUID]; ok && old.Desc != "" && d.Desc == "" {
|
||||
d.Desc = old.Desc
|
||||
}
|
||||
f.Devices[d.GUID] = d
|
||||
cur = d
|
||||
continue
|
||||
}
|
||||
m := portLn.FindStringSubmatch(line)
|
||||
if m == nil || cur == nil {
|
||||
continue
|
||||
}
|
||||
port, _ := strconv.Atoi(m[1])
|
||||
peer := guid(m[3])
|
||||
peerPort, _ := strconv.Atoi(m[4])
|
||||
comment := m[5]
|
||||
// The peer's own block may come later; its description is in the
|
||||
// comment, quoted, which is enough to name it until then.
|
||||
if _, ok := f.Devices[peer]; !ok {
|
||||
kind := map[string]string{"S": "switch", "H": "ca", "R": "router"}[m[2]]
|
||||
d := &Device{GUID: peer, Kind: kind}
|
||||
if q := strings.Index(comment, `"`); q >= 0 {
|
||||
if e := strings.Index(comment[q+1:], `"`); e >= 0 {
|
||||
d.Desc = comment[q+1 : q+1+e]
|
||||
}
|
||||
}
|
||||
f.Devices[peer] = d
|
||||
}
|
||||
a, b := End{cur.GUID, port}, End{peer, peerPort}
|
||||
key := linkKey(a, b)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
l := Link{A: a, B: b}
|
||||
if r := rateRe.FindStringSubmatch(comment); r != nil {
|
||||
l.Width, l.Speed = r[1]+"x", r[2]
|
||||
lanes, _ := strconv.Atoi(r[1])
|
||||
l.Gbps = int(float64(lanes) * laneGbps[r[2]])
|
||||
}
|
||||
f.Links = append(f.Links, l)
|
||||
}
|
||||
if len(f.Devices) == 0 {
|
||||
return nil, fmt.Errorf("no devices in ibnetdiscover output")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func guid(hex string) string {
|
||||
hex = strings.ToLower(hex)
|
||||
for len(hex) < 16 {
|
||||
hex = "0" + hex
|
||||
}
|
||||
return "0x" + hex
|
||||
}
|
||||
|
||||
func linkKey(a, b End) string {
|
||||
x, y := fmt.Sprintf("%s/%d", a.GUID, a.Port), fmt.Sprintf("%s/%d", b.GUID, b.Port)
|
||||
if x > y {
|
||||
x, y = y, x
|
||||
}
|
||||
return x + "-" + y
|
||||
}
|
||||
|
||||
// Host is an adapter the cluster knows: which node, which card.
|
||||
type Host struct {
|
||||
Node string
|
||||
HCA string
|
||||
}
|
||||
|
||||
// View is a fabric's shape, ready to draw.
|
||||
type View struct {
|
||||
Switches []SwitchView `json:"switches"`
|
||||
Hosts []HostView `json:"hosts"`
|
||||
Links []LinkView `json:"links"`
|
||||
// Tiers is how many switch levels: 1 for a single switch, 2 for
|
||||
// leaf and spine, 3 with a core.
|
||||
Tiers int `json:"tiers"`
|
||||
// Expected is the rate most links of each kind trained at; a link below
|
||||
// it is degraded.
|
||||
ExpectedHost int `json:"expected_host_gbps"`
|
||||
ExpectedSwitch int `json:"expected_switch_gbps"`
|
||||
Degraded int `json:"degraded"`
|
||||
// Rails is, per rail — an adapter's position on its node — the leaves it
|
||||
// reaches, and Aligned whether each rail has its own.
|
||||
Rails []Rail `json:"rails"`
|
||||
Aligned bool `json:"rail_aligned"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// SwitchView is a switch and its place in the tiers.
|
||||
type SwitchView struct {
|
||||
GUID string `json:"guid"`
|
||||
Name string `json:"name"`
|
||||
Tier int `json:"tier"` // 1 leaf, 2 spine, 3 core
|
||||
Ports int `json:"ports"`
|
||||
Used int `json:"used"`
|
||||
LID int `json:"lid,omitempty"`
|
||||
// Hosts counts adapters hanging off it.
|
||||
Hosts int `json:"hosts"`
|
||||
}
|
||||
|
||||
// HostView is an adapter, and the node it is in when the cluster knows it.
|
||||
type HostView struct {
|
||||
GUID string `json:"guid"`
|
||||
Desc string `json:"desc"`
|
||||
Node string `json:"node,omitempty"`
|
||||
HCA string `json:"hca,omitempty"`
|
||||
Rail int `json:"rail"`
|
||||
Leaf string `json:"leaf,omitempty"`
|
||||
}
|
||||
|
||||
// LinkView is a cable, with what it trained at and whether that is short of
|
||||
// its kind.
|
||||
type LinkView struct {
|
||||
From string `json:"from"`
|
||||
FromPort int `json:"from_port"`
|
||||
To string `json:"to"`
|
||||
ToPort int `json:"to_port"`
|
||||
Rate string `json:"rate"`
|
||||
Gbps int `json:"gbps"`
|
||||
Host bool `json:"host"`
|
||||
Degraded bool `json:"degraded"`
|
||||
}
|
||||
|
||||
// Rail is the leaves one rail's adapters reach.
|
||||
type Rail struct {
|
||||
Index int `json:"index"`
|
||||
Leaves []string `json:"leaves"`
|
||||
}
|
||||
|
||||
// Build works out the shape. hosts maps adapter GUIDs to the cluster's nodes.
|
||||
func Build(f *Fabric, hosts map[string]Host) View {
|
||||
var v View
|
||||
adj := map[string][]string{}
|
||||
for _, l := range f.Links {
|
||||
adj[l.A.GUID] = append(adj[l.A.GUID], l.B.GUID)
|
||||
adj[l.B.GUID] = append(adj[l.B.GUID], l.A.GUID)
|
||||
}
|
||||
// A switch's tier is its distance from the nearest adapter: leaves hold
|
||||
// adapters, spines hold leaves, cores hold spines.
|
||||
tier := map[string]int{}
|
||||
var frontier []string
|
||||
for g, d := range f.Devices {
|
||||
if d.Kind == "ca" {
|
||||
tier[g] = 0
|
||||
frontier = append(frontier, g)
|
||||
}
|
||||
}
|
||||
for len(frontier) > 0 {
|
||||
var next []string
|
||||
for _, g := range frontier {
|
||||
for _, p := range adj[g] {
|
||||
if _, ok := tier[p]; ok || f.Devices[p] == nil || f.Devices[p].Kind == "ca" {
|
||||
continue
|
||||
}
|
||||
tier[p] = tier[g] + 1
|
||||
next = append(next, p)
|
||||
}
|
||||
}
|
||||
frontier = next
|
||||
}
|
||||
|
||||
name := func(g string) string {
|
||||
d := f.Devices[g]
|
||||
if d == nil || d.Desc == "" {
|
||||
return g
|
||||
}
|
||||
desc := d.Desc
|
||||
// Mellanox switches say "MF0;leaf-01:MSB7700/U1": the name is between.
|
||||
if i := strings.Index(desc, ";"); i >= 0 {
|
||||
desc = desc[i+1:]
|
||||
if j := strings.Index(desc, ":"); j >= 0 {
|
||||
desc = desc[:j]
|
||||
}
|
||||
}
|
||||
return desc
|
||||
}
|
||||
|
||||
// Expected rate per kind of link: the most common.
|
||||
hostRates, swRates := map[int]int{}, map[int]int{}
|
||||
isHost := func(l Link) bool {
|
||||
return f.Devices[l.A.GUID].Kind == "ca" || f.Devices[l.B.GUID].Kind == "ca"
|
||||
}
|
||||
for _, l := range f.Links {
|
||||
if isHost(l) {
|
||||
hostRates[l.Gbps]++
|
||||
} else {
|
||||
swRates[l.Gbps]++
|
||||
}
|
||||
}
|
||||
v.ExpectedHost, v.ExpectedSwitch = mode(hostRates), mode(swRates)
|
||||
|
||||
used := map[string]int{}
|
||||
hostsOn := map[string]int{}
|
||||
leafOf := map[string]string{}
|
||||
for _, l := range f.Links {
|
||||
used[l.A.GUID]++
|
||||
used[l.B.GUID]++
|
||||
lv := LinkView{From: l.A.GUID, FromPort: l.A.Port, To: l.B.GUID, ToPort: l.B.Port,
|
||||
Rate: l.Width + l.Speed, Gbps: l.Gbps, Host: isHost(l)}
|
||||
// Keep adapters on the To side, so a drawing can go down the tiers.
|
||||
if f.Devices[lv.From].Kind == "ca" {
|
||||
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
|
||||
}
|
||||
if lv.Host {
|
||||
hostsOn[lv.From]++
|
||||
leafOf[lv.To] = lv.From
|
||||
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedHost
|
||||
} else {
|
||||
lv.Degraded = l.Gbps > 0 && l.Gbps < v.ExpectedSwitch
|
||||
if tier[lv.From] < tier[lv.To] {
|
||||
lv.From, lv.FromPort, lv.To, lv.ToPort = lv.To, lv.ToPort, lv.From, lv.FromPort
|
||||
}
|
||||
}
|
||||
if lv.Degraded {
|
||||
v.Degraded++
|
||||
}
|
||||
v.Links = append(v.Links, lv)
|
||||
}
|
||||
|
||||
for g, d := range f.Devices {
|
||||
switch d.Kind {
|
||||
case "switch":
|
||||
t := tier[g]
|
||||
if t == 0 {
|
||||
t = 1 // a switch with nothing below it is drawn with the leaves
|
||||
}
|
||||
v.Switches = append(v.Switches, SwitchView{GUID: g, Name: name(g), Tier: t, Ports: d.Ports,
|
||||
Used: used[g], LID: d.LID, Hosts: hostsOn[g]})
|
||||
v.Tiers = max(v.Tiers, t)
|
||||
case "ca":
|
||||
h := HostView{GUID: g, Desc: d.Desc, Leaf: leafOf[g]}
|
||||
if known, ok := hosts[g]; ok {
|
||||
h.Node, h.HCA = known.Node, known.HCA
|
||||
}
|
||||
v.Hosts = append(v.Hosts, h)
|
||||
}
|
||||
}
|
||||
sort.Slice(v.Switches, func(i, j int) bool {
|
||||
a, b := v.Switches[i], v.Switches[j]
|
||||
if a.Tier != b.Tier {
|
||||
return a.Tier > b.Tier
|
||||
}
|
||||
return a.Name < b.Name
|
||||
})
|
||||
|
||||
// Rails: an adapter's position among its node's adapters on this fabric.
|
||||
byNode := map[string][]int{}
|
||||
for i, h := range v.Hosts {
|
||||
key := h.Node
|
||||
if key == "" {
|
||||
key = strings.Fields(h.Desc + " ?")[0] // "node01 mlx5_0" names its host
|
||||
}
|
||||
byNode[key] = append(byNode[key], i)
|
||||
}
|
||||
railLeaves := map[int]map[string]bool{}
|
||||
for _, idx := range byNode {
|
||||
sort.Slice(idx, func(a, b int) bool {
|
||||
return hcaName(v.Hosts[idx[a]]) < hcaName(v.Hosts[idx[b]])
|
||||
})
|
||||
for r, i := range idx {
|
||||
v.Hosts[i].Rail = r
|
||||
if railLeaves[r] == nil {
|
||||
railLeaves[r] = map[string]bool{}
|
||||
}
|
||||
if v.Hosts[i].Leaf != "" {
|
||||
railLeaves[r][v.Hosts[i].Leaf] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(v.Hosts, func(i, j int) bool {
|
||||
a, b := v.Hosts[i], v.Hosts[j]
|
||||
if a.Leaf != b.Leaf {
|
||||
return a.Leaf < b.Leaf
|
||||
}
|
||||
return a.Desc < b.Desc
|
||||
})
|
||||
owner := map[string]int{}
|
||||
v.Aligned = len(railLeaves) > 1
|
||||
for r := 0; r < len(railLeaves); r++ {
|
||||
var leaves []string
|
||||
for l := range railLeaves[r] {
|
||||
leaves = append(leaves, name(l))
|
||||
if prev, ok := owner[l]; ok && prev != r {
|
||||
v.Aligned = false
|
||||
}
|
||||
owner[l] = r
|
||||
}
|
||||
sort.Strings(leaves)
|
||||
v.Rails = append(v.Rails, Rail{Index: r, Leaves: leaves})
|
||||
}
|
||||
if v.Degraded > 0 {
|
||||
v.Notes = append(v.Notes, fmt.Sprintf("%d link(s) trained below the rest of their kind", v.Degraded))
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func hcaName(h HostView) string {
|
||||
if h.HCA != "" {
|
||||
return h.HCA
|
||||
}
|
||||
return h.Desc
|
||||
}
|
||||
|
||||
func mode(counts map[int]int) int {
|
||||
best, n := 0, -1
|
||||
for v, c := range counts {
|
||||
if v == 0 {
|
||||
continue
|
||||
}
|
||||
if c > n || (c == n && v > best) {
|
||||
best, n = v, c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fatTree writes ibnetdiscover's default output for two spines, two leaves
|
||||
// and four nodes of two adapters each, rail-aligned: every node's mlx5_0 on
|
||||
// leaf-01 and mlx5_1 on leaf-02. node03's mlx5_1 trained at 2x instead of 4x.
|
||||
func fatTree() string {
|
||||
var b strings.Builder
|
||||
w := func(format string, a ...any) { fmt.Fprintf(&b, format+"\n", a...) }
|
||||
w("#\n# Topology file: generated on Mon Sep 29 10:00:00 2026\n#\n# Initiated from node 0000000000000a01 port 0000000000000a01\n")
|
||||
spine := []string{"00000000000000c1", "00000000000000c2"}
|
||||
leaf := []string{"00000000000000b1", "00000000000000b2"}
|
||||
hca := func(n, r int) string { return fmt.Sprintf("000000000000%d%02d", r+10, n) }
|
||||
for si, s := range spine {
|
||||
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", s, s, s)
|
||||
w("Switch\t64 \"S-%s\"\t\t# \"MF0;spine-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", s, si+1, 10+si)
|
||||
for li, l := range leaf {
|
||||
for k := 0; k < 2; k++ {
|
||||
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;leaf-0%d:MQM9700/U1\" lid %d 4xNDR", li*2+k+1, l, 33+si*2+k, li+1, 20+li)
|
||||
}
|
||||
}
|
||||
w("")
|
||||
}
|
||||
for li, l := range leaf {
|
||||
w("vendid=0x2c9\ndevid=0xd2f0\nsysimgguid=0x%s\nswitchguid=0x%s(%s)", l, l, l)
|
||||
w("Switch\t64 \"S-%s\"\t\t# \"MF0;leaf-0%d:MQM9700/U1\" enhanced port 0 lid %d lmc 0", l, li+1, 20+li)
|
||||
for n := 1; n <= 4; n++ {
|
||||
rate := "4xNDR"
|
||||
if n == 3 && li == 1 {
|
||||
rate = "2xNDR"
|
||||
}
|
||||
h := hca(n, li)
|
||||
w("[%d]\t\"H-%s\"[1](%s) \t\t# \"node0%d mlx5_%d\" lid %d %s", n, h, h, n, li, 100+n*2+li, rate)
|
||||
}
|
||||
for si, s := range spine {
|
||||
for k := 0; k < 2; k++ {
|
||||
w("[%d]\t\"S-%s\"[%d]\t\t# \"MF0;spine-0%d:MQM9700/U1\" lid %d 4xNDR", 33+si*2+k, s, li*2+k+1, si+1, 10+si)
|
||||
}
|
||||
}
|
||||
w("")
|
||||
}
|
||||
for n := 1; n <= 4; n++ {
|
||||
for li, l := range leaf {
|
||||
h := hca(n, li)
|
||||
rate := "4xNDR"
|
||||
if n == 3 && li == 1 {
|
||||
rate = "2xNDR"
|
||||
}
|
||||
w("vendid=0x2c9\ndevid=0x1021\nsysimgguid=0x%s\ncaguid=0x%s", h, h)
|
||||
w("Ca\t1 \"H-%s\"\t\t# \"node0%d mlx5_%d\"", h, n, li)
|
||||
w("[1](%s) \t\"S-%s\"[%d]\t\t# lid %d lmc 0 \"MF0;leaf-0%d:MQM9700/U1\" lid %d %s", h, l, n, 100+n*2+li, li+1, 20+li, rate)
|
||||
w("")
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func TestAFatTreeIsReadIntoTiers(t *testing.T) {
|
||||
f, err := Parse(fatTree())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// 2 spines + 2 leaves + 8 adapters; 8 spine-leaf cables + 8 host cables,
|
||||
// each once though both ends list it.
|
||||
if len(f.Devices) != 12 || len(f.Links) != 16 {
|
||||
t.Fatalf("%d devices, %d links", len(f.Devices), len(f.Links))
|
||||
}
|
||||
hosts := map[string]Host{}
|
||||
for n := 1; n <= 4; n++ {
|
||||
for r := 0; r < 2; r++ {
|
||||
hosts[fmt.Sprintf("0x000000000000%d%02d", r+10, n)] = Host{Node: fmt.Sprintf("n%d", n), HCA: fmt.Sprintf("mlx5_%d", r)}
|
||||
}
|
||||
}
|
||||
v := Build(f, hosts)
|
||||
if v.Tiers != 2 {
|
||||
t.Errorf("tiers = %d", v.Tiers)
|
||||
}
|
||||
tiers := map[string]int{}
|
||||
for _, s := range v.Switches {
|
||||
tiers[s.Name] = s.Tier
|
||||
}
|
||||
if tiers["spine-01"] != 2 || tiers["leaf-02"] != 1 {
|
||||
t.Errorf("tiers = %v", tiers)
|
||||
}
|
||||
if v.ExpectedHost != 400 || v.ExpectedSwitch != 400 {
|
||||
t.Errorf("expected rates %d / %d", v.ExpectedHost, v.ExpectedSwitch)
|
||||
}
|
||||
// node03's second adapter trained at half width.
|
||||
if v.Degraded != 1 {
|
||||
t.Errorf("degraded = %d", v.Degraded)
|
||||
}
|
||||
for _, l := range v.Links {
|
||||
if l.Degraded && (l.Rate != "2xNDR" || !l.Host) {
|
||||
t.Errorf("wrong link degraded: %+v", l)
|
||||
}
|
||||
}
|
||||
// Every node's first adapter on leaf-01, its second on leaf-02: aligned.
|
||||
if !v.Aligned || len(v.Rails) != 2 || v.Rails[0].Leaves[0] != "leaf-01" || v.Rails[1].Leaves[0] != "leaf-02" {
|
||||
t.Errorf("rails = %+v aligned=%v", v.Rails, v.Aligned)
|
||||
}
|
||||
mapped := 0
|
||||
for _, h := range v.Hosts {
|
||||
if h.Node != "" {
|
||||
mapped++
|
||||
}
|
||||
}
|
||||
if mapped != 8 {
|
||||
t.Errorf("%d adapters mapped to nodes", mapped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCrossedRailsAreNotAligned(t *testing.T) {
|
||||
// node02's two cables swapped at the leaves: its mlx5_0 on leaf-02 and
|
||||
// its mlx5_1 on leaf-01, so each leaf now carries both rails.
|
||||
f, err := Parse(fatTree())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap := map[string]string{"0x0000000000001002": "0x00000000000000b2", "0x0000000000001102": "0x00000000000000b1"}
|
||||
for i, l := range f.Links {
|
||||
for _, host := range []*End{&f.Links[i].A, &f.Links[i].B} {
|
||||
if leaf, ok := swap[host.GUID]; ok {
|
||||
other := &f.Links[i].A
|
||||
if host == other {
|
||||
other = &f.Links[i].B
|
||||
}
|
||||
other.GUID = leaf
|
||||
}
|
||||
}
|
||||
_ = l
|
||||
}
|
||||
v := Build(f, map[string]Host{
|
||||
"0x0000000000001002": {Node: "n2", HCA: "mlx5_0"}, "0x0000000000001102": {Node: "n2", HCA: "mlx5_1"},
|
||||
})
|
||||
if v.Aligned {
|
||||
t.Errorf("crossed rails read as aligned: %+v", v.Rails)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotAMapIsAnError(t *testing.T) {
|
||||
if _, err := Parse("ibwarn: [1234] mad_rpc_open_port: can't open UMAD port ((null):0)"); err == nil {
|
||||
t.Error("expected an error")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user