Map every InfiniBand fabric, draw it, and place replicas by it
The control plane asks one node on each physical fabric to run
ibnetdiscover every five minutes and parses the text itself, so a
format the parser has not met is a control plane upgrade, not one on
every node. From the map it works out the switch tiers (leaf, spine,
core, by distance from the adapters), which adapter is which node's,
which cables trained below the rate the rest of their kind run at, and
whether the rails line up: every node's Nth adapter on its own leaf.
A page per fabric (/admin/fabrics) draws it: switches in their tiers,
adapters fanned out under their leaves, cables coloured by rate and
bundled, degraded ones marching red with a marker, and the cluster's
own adapters live - down ports glowing red, and particles running each
cable as fast as its adapter is sending and receiving, from the
counters. It zooms, pans, fades what a hovered switch or adapter does
not touch, opens a panel of detail on a click, colours by rail, finds a
node, and refreshes every fifteen seconds. The data is also at
/api/v1/fabrics/{id}/topology.
Placement reads the map: a replica spanning nodes takes nodes of one
leaf together when it can, and nodes whose cable trained short last.
Static assets are named by their content, as the stylesheet already
was.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
23648fa19a
commit
5bbc619a31
23 files changed
+2171
-83
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
@@ -52,6 +53,14 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Fabrics())
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/fabrics/{id}/topology", func(w http.ResponseWriter, r *http.Request) {
|
||||
t, ok := a.Topology(r.PathValue("id"))
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, t)
|
||||
})
|
||||
mux.HandleFunc("GET /api/v1/models", a.listModels)
|
||||
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
|
||||
// Listing is a read and deleting is a separate verb, so a mistyped
|
||||
@@ -839,3 +848,114 @@ func (a *API) Fabrics() fabrics.Report {
|
||||
inv := a.ctl.Inventory()
|
||||
return fabrics.Analyze(inv.Nodes(), inv.SMChanges(), time.Now())
|
||||
}
|
||||
|
||||
// TopologyView is one fabric's map, with the cluster's own adapters on it
|
||||
// live: their state, rate and traffic from the last heartbeat.
|
||||
type TopologyView struct {
|
||||
ID string `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Label string `json:"label"`
|
||||
Declared []string `json:"declared"`
|
||||
ScannedAt time.Time `json:"scanned_at"`
|
||||
ScannedBy string `json:"scanned_by,omitempty"`
|
||||
TookMs int64 `json:"took_ms"`
|
||||
Err string `json:"error,omitempty"`
|
||||
Map *topology.View `json:"map,omitempty"`
|
||||
Live map[string]Live `json:"live"`
|
||||
// Problems are the fabric's own, from the analysis of its identity and
|
||||
// subnet managers.
|
||||
Problems []fabrics.Problem `json:"problems,omitempty"`
|
||||
SM *fabrics.SM `json:"sm,omitempty"`
|
||||
SMs []fabrics.SM `json:"sms,omitempty"`
|
||||
}
|
||||
|
||||
// Live is one of the cluster's adapters as its node last reported it.
|
||||
type Live struct {
|
||||
Node string `json:"node"`
|
||||
HCA string `json:"hca"`
|
||||
State string `json:"state"`
|
||||
Phys string `json:"phys_state"`
|
||||
Gbps int `json:"gbps"`
|
||||
TxBps float64 `json:"tx_bps"`
|
||||
RxBps float64 `json:"rx_bps"`
|
||||
Errors map[string]uint64 `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// FabricID is a physical fabric's key as an address can carry it.
|
||||
func FabricID(key string) string {
|
||||
return strings.NewReplacer(":", "-", "/", "_").Replace(key)
|
||||
}
|
||||
|
||||
// Topologies lists every physical fabric with what is known of its map.
|
||||
func (a *API) Topologies() []TopologyView {
|
||||
rep := a.Fabrics()
|
||||
scans := a.ctl.Inventory().Scans()
|
||||
out := make([]TopologyView, 0, len(rep.Physical))
|
||||
for _, ph := range rep.Physical {
|
||||
t := TopologyView{ID: FabricID(ph.Key), Key: ph.Key, Label: ph.Label, Declared: ph.Declared}
|
||||
if sc, ok := scans[ph.Key]; ok {
|
||||
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Topology is one fabric's map, by the id Topologies gives it.
|
||||
func (a *API) Topology(id string) (TopologyView, bool) {
|
||||
rep := a.Fabrics()
|
||||
var ph *fabrics.Physical
|
||||
for i := range rep.Physical {
|
||||
if FabricID(rep.Physical[i].Key) == id {
|
||||
ph = &rep.Physical[i]
|
||||
}
|
||||
}
|
||||
if ph == nil {
|
||||
return TopologyView{}, false
|
||||
}
|
||||
t := TopologyView{ID: id, Key: ph.Key, Label: ph.Label, Declared: ph.Declared, Live: map[string]Live{},
|
||||
SM: ph.SM, SMs: ph.SMs}
|
||||
for _, p := range rep.Problems {
|
||||
for _, d := range ph.Declared {
|
||||
if p.Fabric == d {
|
||||
t.Problems = append(t.Problems, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
hosts := map[string]topology.Host{}
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.NodeGUID == "" {
|
||||
continue
|
||||
}
|
||||
hosts[p.NodeGUID] = topology.Host{Node: n.ID, HCA: p.HCA}
|
||||
l := Live{Node: n.ID, HCA: p.HCA, State: p.State, Phys: p.PhysState, Gbps: p.RateGbps,
|
||||
TxBps: p.TxBps, RxBps: p.RxBps}
|
||||
for name, c := range p.Counters {
|
||||
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
|
||||
if l.Errors == nil {
|
||||
l.Errors = map[string]uint64{}
|
||||
}
|
||||
l.Errors[name] = c
|
||||
}
|
||||
}
|
||||
t.Live[p.NodeGUID] = l
|
||||
}
|
||||
}
|
||||
sc, ok := a.ctl.Inventory().Scans()[ph.Key]
|
||||
if !ok {
|
||||
return t, true
|
||||
}
|
||||
t.ScannedAt, t.ScannedBy, t.TookMs, t.Err = sc.At, sc.Node+" "+sc.Port, sc.Took.Milliseconds(), sc.Err
|
||||
if sc.Output != "" {
|
||||
f, err := topology.Parse(sc.Output)
|
||||
if err != nil {
|
||||
t.Err = "the map could not be read: " + err.Error()
|
||||
return t, true
|
||||
}
|
||||
v := topology.Build(f, hosts)
|
||||
t.Map = &v
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
Reference in new issue
Block a user