Tell independent fabrics apart by their subnet manager, not their name

A fabric was what agent.yaml called it. With several independent
InfiniBand fabrics at a site, two given one name would have had a
replica spread across both, starting and then hanging on its first
collective. Each active InfiniBand port now reports the master subnet
manager it sees (sminfo, asked once a minute), which every port on one
fabric shares and ports on separate fabrics cannot; a RoCE port reports
the IP network of its RoCE v2 address. Ports also carry their node and
port GUIDs.

Placement divides a declared fabric by that physical identity and forms
a replica on one part only; a node whose adapters for one name are on
two fabrics is refused, and a node that cannot say is kept apart from
those that can. /api/v1/fabrics and the cluster page list each physical
fabric with its subnet manager, and say when one name covers two
fabrics, two names one fabric, an adapter is up but undeclared, or a
node cannot be checked. doctor reports each port's subnet manager, and
an agent on a node with adapters installs infiniband-diags when sminfo
is missing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 07:29:16 +02:00
1 parent 51f1606b53
commit 77fe801892
21 files changed
+1130 -58

No files matched your search

+21 -1
View File
@@ -10,6 +10,7 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"io"
"net/http"
"slices"
@@ -48,6 +49,9 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/capacity", a.listCapacity)
mux.HandleFunc("GET /api/v1/usage", a.usageByScope)
mux.HandleFunc("GET /api/v1/nodes", a.listNodes)
mux.HandleFunc("GET /api/v1/fabrics", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.Fabrics())
})
mux.HandleFunc("GET /api/v1/models", a.listModels)
mux.HandleFunc("GET /api/v1/replicas", a.listReplicas)
// Listing is a read and deleting is a separate verb, so a mistyped
@@ -134,6 +138,10 @@ type IBPortView struct {
Active bool `json:"active"`
// Fabric is the declared fabric this port's adapter carries, if any.
Fabric string `json:"fabric,omitempty"`
// Physical is the fabric the port is physically on: its subnet manager,
// or its RoCE network. SMError says why that is not known.
Physical string `json:"physical,omitempty"`
SMError string `json:"sm_error,omitempty"`
// Errors are the error counters that are not zero, by name.
Errors map[string]uint64 `json:"errors,omitempty"`
TxBytes uint64 `json:"tx_bytes,omitempty"`
@@ -336,7 +344,13 @@ func (a *API) Nodes() []NodeView {
for _, p := range n.IBPorts {
pv := IBPortView{HCA: p.HCA, Port: p.Port, State: p.State, PhysState: p.PhysState,
RateGbps: p.RateGbps, LinkLayer: p.LinkLayer, Active: p.Active(), Fabric: fabricOf[p.HCA],
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"]}
TxBytes: p.Counters["port_xmit_data"], RxBytes: p.Counters["port_rcv_data"], SMError: p.SMError}
switch {
case p.RoCE() && p.Network != "":
pv.Physical = "RoCE " + p.Network
case p.SMGUID != "":
pv.Physical = fmt.Sprintf("SM %s (LID %d)", p.SMGUID, p.SMLID)
}
for name, c := range p.Counters {
if c > 0 && name != "port_xmit_data" && name != "port_rcv_data" {
if pv.Errors == nil {
@@ -818,3 +832,9 @@ func (a *API) profileChanged(model, profile, was string) bool {
}
return false
}
// Fabrics is the physical fabrics the adapters are on, against the names
// the nodes gave them.
func (a *API) Fabrics() fabrics.Report {
return fabrics.Analyze(a.ctl.Inventory().Nodes())
}