Map every InfiniBand fabric, draw it, and place replicas by it
The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.
A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.
Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.
Static assets are named by their content, as the stylesheet already
was.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
23648fa19a
commit
5bbc619a31
23 files changed
+2171
-83
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -58,6 +59,8 @@ type Service struct {
|
||||
logMu sync.Mutex
|
||||
logWaits map[string]chan *llmgwv1.LogsReply
|
||||
logSeq uint64
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
}
|
||||
|
||||
// SetEngines sets the engines every node installs, by name and version.
|
||||
@@ -282,6 +285,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
"last_output", m.Shard.GetLastOutput())
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricScan:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
|
||||
delete(s.scanWaits, m.FabricScan.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.FabricScan
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_Logs:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.logWaits[m.Logs.GetRequestId()]
|
||||
@@ -406,9 +418,13 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
}
|
||||
if ports := hb.GetIbPorts(); len(ports) > 0 {
|
||||
before := map[string]string{}
|
||||
prev := map[string]inventory.IBPort{}
|
||||
for _, p := range n.IBPorts {
|
||||
before[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p.SMGUID
|
||||
prev[fmt.Sprintf("%s/%d", p.HCA, p.Port)] = p
|
||||
}
|
||||
now := time.Now()
|
||||
secs := now.Sub(n.IBAt).Seconds()
|
||||
n.IBPorts = nil
|
||||
for _, p := range ports {
|
||||
// A different master than last time is a failover, or a port
|
||||
@@ -427,7 +443,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
}
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{
|
||||
SMs: sms, SMsError: p.GetSubnetManagersError(),
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
TxBps: rate(prev[where].Counters["port_xmit_data"], p.GetCounters()["port_xmit_data"], secs),
|
||||
RxBps: rate(prev[where].Counters["port_rcv_data"], p.GetCounters()["port_rcv_data"], secs),
|
||||
HCA: p.GetHca(), Port: int(p.GetPort()), State: p.GetState(),
|
||||
PhysState: p.GetPhysState(), RateGbps: int(p.GetRateGbps()),
|
||||
LinkLayer: p.GetLinkLayer(), Counters: maps.Clone(p.GetCounters()),
|
||||
NodeGUID: p.GetNodeGuid(), PortGUID: p.GetPortGuid(), GIDPrefix: p.GetGidPrefix(),
|
||||
@@ -436,6 +454,9 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(hb.GetIbPorts()) > 0 {
|
||||
n.IBAt = time.Now()
|
||||
}
|
||||
// A port went down, or came back. Placement reads the fabrics, so this is
|
||||
// what stops a node being offered multi-node work it can no longer carry.
|
||||
if hb.GetFabricsChanged() {
|
||||
@@ -642,3 +663,102 @@ func (s *Service) Logs(ctx context.Context, nodeID, source string, lines int) ([
|
||||
return nil, nil, fmt.Errorf("node %s did not answer; an agent older than v0.8.4 cannot", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabric asks a node to map the fabric one of its ports is on.
|
||||
func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int) (*llmgwv1.FabricScanReply, error) {
|
||||
s.logMu.Lock()
|
||||
if s.scanWaits == nil {
|
||||
s.scanWaits = map[string]chan *llmgwv1.FabricScanReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("scan-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.FabricScanReply, 1)
|
||||
s.scanWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.scanWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricScan{
|
||||
FabricScan: &llmgwv1.FabricScanRequest{RequestId: id, Hca: hca, Port: int32(port)}, //nolint:gosec // a port number
|
||||
}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r, nil
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("node %s did not answer the fabric scan", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
|
||||
// one node on each: the first, by name, whose port on it is up. A fabric
|
||||
// looks the same from all its ports, so one is enough, and the rest of its
|
||||
// nodes are left to their work.
|
||||
func (s *Service) ScanFabrics(ctx context.Context, every time.Duration) {
|
||||
if every <= 0 {
|
||||
every = 5 * time.Minute
|
||||
}
|
||||
// Soon after a start, once agents have sent a heartbeat or two.
|
||||
timer := time.NewTimer(30 * time.Second)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
}
|
||||
s.scanOnce(ctx)
|
||||
timer.Reset(every)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) scanOnce(ctx context.Context) {
|
||||
nodes := s.inv.Nodes()
|
||||
sort.Slice(nodes, func(i, j int) bool { return nodes[i].ID < nodes[j].ID })
|
||||
canon := inventory.CanonicalSMs(nodes)
|
||||
chosen := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
if n.State != inventory.Ready {
|
||||
continue
|
||||
}
|
||||
for _, p := range n.IBPorts {
|
||||
key := p.FabricKeyIn(canon)
|
||||
if key == "" || p.RoCE() || chosen[key] {
|
||||
continue
|
||||
}
|
||||
chosen[key] = true
|
||||
where := fmt.Sprintf("%s/%d", p.HCA, p.Port)
|
||||
r, err := s.ScanFabric(ctx, n.ID, p.HCA, p.Port)
|
||||
sc := inventory.FabricScan{Key: key, At: time.Now(), Node: n.ID, Port: where}
|
||||
switch {
|
||||
case err != nil:
|
||||
sc.Err = err.Error()
|
||||
case r.GetError() != "":
|
||||
sc.Err = r.GetError()
|
||||
default:
|
||||
sc.Output = r.GetOutput()
|
||||
}
|
||||
if r != nil {
|
||||
sc.Took = time.Duration(r.GetTookMs()) * time.Millisecond
|
||||
}
|
||||
if sc.Err != "" {
|
||||
s.log.Warn("could not map a fabric", "fabric", key, "node", n.ID, "port", where, "err", sc.Err)
|
||||
}
|
||||
s.inv.SetScan(sc)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rate is a counter's increase a second, zero when there is no earlier
|
||||
// reading or the counter went backwards — an adapter reset starts it again.
|
||||
func rate(before, now uint64, secs float64) float64 {
|
||||
if before == 0 || now < before || secs <= 0 || secs > 120 {
|
||||
return 0
|
||||
}
|
||||
return float64(now-before) / secs
|
||||
}
|
||||
Reference in new issue
Block a user