Measure two nodes' fabric rail by rail: llmgw ctl fabric-test

doctor checks a node alone, and every check can pass while two nodes still
cannot reach each other. ctl fabric-test A B asks B's agent for an
ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs
them, one rail at a time, and grades each against its link's rate. It goes
through the agents' streams, so there is no SSH between nodes; the agent
installs perftest the first time, and names the RoCE v2 GID itself.

The result is read from perftest's table by its heading, not a column
number. Agents now list the requests they answer beyond the usual
(Register.features), and the control plane refuses a test on one too old to
answer instead of waiting minutes for it. The forwarding dispatcher carries
the new capability, and its test says so: a missing forward would have made
the command quietly unavailable in production.

Not yet run over real InfiniBand or RoCE; this cluster has neither.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 21:29:23 +02:00
1 parent af6f8e023e
commit 7bb360fc78
21 files changed
+1265 -62

No files matched your search

+1
View File
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
}
+71
View File
@@ -0,0 +1,71 @@
package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
)
// FabricTestView is a bandwidth test between two nodes.
type FabricTestView struct {
A string `json:"a"`
B string `json:"b"`
Fabric string `json:"fabric"`
OK bool `json:"ok"`
Rails []RailView `json:"rails"`
Notes []string `json:"notes,omitempty"`
Took string `json:"took"`
}
// RailView is one rail's result.
type RailView struct {
A string `json:"a"`
B string `json:"b"`
RateGbps int `json:"rate_gbps"`
Gbps float64 `json:"gbps"`
Verdict string `json:"verdict"`
Detail string `json:"detail,omitempty"`
}
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
// sends traffic over the fabric for seconds a rail, so it is a POST and only
// ever run when asked.
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
var body struct {
A, B, Fabric string
Seconds int
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
ask := a.ctl.FabricTester()
if ask == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
return
}
inv := a.ctl.Inventory()
na, okA := inv.Get(body.A)
nb, okB := inv.Get(body.B)
if !okA || !okB {
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
return
}
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
start := time.Now()
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
Took: time.Since(start).Round(time.Second).String()}
for _, x := range rep.Results {
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
}
writeJSON(w, http.StatusOK, v)
}