Measure two nodes' fabric rail by rail: llmgw ctl fabric-test
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
af6f8e023e
commit
7bb360fc78
21 files changed
+1265
-62
No files matched your search
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
)
|
||||
|
||||
// FabricTestView is a bandwidth test between two nodes.
|
||||
type FabricTestView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Fabric string `json:"fabric"`
|
||||
OK bool `json:"ok"`
|
||||
Rails []RailView `json:"rails"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
Took string `json:"took"`
|
||||
}
|
||||
|
||||
// RailView is one rail's result.
|
||||
type RailView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
RateGbps int `json:"rate_gbps"`
|
||||
Gbps float64 `json:"gbps"`
|
||||
Verdict string `json:"verdict"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
|
||||
// sends traffic over the fabric for seconds a rail, so it is a POST and only
|
||||
// ever run when asked.
|
||||
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
A, B, Fabric string
|
||||
Seconds int
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
ask := a.ctl.FabricTester()
|
||||
if ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
|
||||
return
|
||||
}
|
||||
inv := a.ctl.Inventory()
|
||||
na, okA := inv.Get(body.A)
|
||||
nb, okB := inv.Get(body.B)
|
||||
if !okA || !okB {
|
||||
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
|
||||
return
|
||||
}
|
||||
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
|
||||
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
|
||||
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
|
||||
Took: time.Since(start).Round(time.Second).String()}
|
||||
for _, x := range rep.Results {
|
||||
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
Reference in new issue
Block a user