Ilker ManapandClaude Opus 5.5
7bb360fc78
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
72 lines
2.1 KiB
Go
72 lines
2.1 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
|
)
|
|
|
|
// FabricTestView is a bandwidth test between two nodes.
|
|
type FabricTestView struct {
|
|
A string `json:"a"`
|
|
B string `json:"b"`
|
|
Fabric string `json:"fabric"`
|
|
OK bool `json:"ok"`
|
|
Rails []RailView `json:"rails"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
Took string `json:"took"`
|
|
}
|
|
|
|
// RailView is one rail's result.
|
|
type RailView struct {
|
|
A string `json:"a"`
|
|
B string `json:"b"`
|
|
RateGbps int `json:"rate_gbps"`
|
|
Gbps float64 `json:"gbps"`
|
|
Verdict string `json:"verdict"`
|
|
Detail string `json:"detail,omitempty"`
|
|
}
|
|
|
|
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
|
|
// sends traffic over the fabric for seconds a rail, so it is a POST and only
|
|
// ever run when asked.
|
|
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
A, B, Fabric string
|
|
Seconds int
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
ask := a.ctl.FabricTester()
|
|
if ask == nil {
|
|
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
|
|
return
|
|
}
|
|
inv := a.ctl.Inventory()
|
|
na, okA := inv.Get(body.A)
|
|
nb, okB := inv.Get(body.B)
|
|
if !okA || !okB {
|
|
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
|
|
return
|
|
}
|
|
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
|
|
if err != nil {
|
|
writeError(w, http.StatusBadRequest, err.Error())
|
|
return
|
|
}
|
|
start := time.Now()
|
|
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
|
|
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
|
|
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
|
|
Took: time.Since(start).Round(time.Second).String()}
|
|
for _, x := range rep.Results {
|
|
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
|
|
}
|
|
writeJSON(w, http.StatusOK, v)
|
|
}
|