Measure two nodes' fabric rail by rail: llmgw ctl fabric-test

doctor checks a node alone, and every check can pass while two nodes still
cannot reach each other. ctl fabric-test A B asks B's agent for an
ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs
them, one rail at a time, and grades each against its link's rate. It goes
through the agents' streams, so there is no SSH between nodes; the agent
installs perftest the first time, and names the RoCE v2 GID itself.

The result is read from perftest's table by its heading, not a column
number. Agents now list the requests they answer beyond the usual
(Register.features), and the control plane refuses a test on one too old to
answer instead of waiting minutes for it. The forwarding dispatcher carries
the new capability, and its test says so: a missing forward would have made
the command quietly unavailable in production.

Not yet run over real InfiniBand or RoCE; this cluster has neither.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 21:29:23 +02:00
1 parent af6f8e023e
commit 7bb360fc78
21 files changed
+1265 -62

No files matched your search

+1
View File
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
}
+71
View File
@@ -0,0 +1,71 @@
package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
)
// FabricTestView is a bandwidth test between two nodes.
type FabricTestView struct {
A string `json:"a"`
B string `json:"b"`
Fabric string `json:"fabric"`
OK bool `json:"ok"`
Rails []RailView `json:"rails"`
Notes []string `json:"notes,omitempty"`
Took string `json:"took"`
}
// RailView is one rail's result.
type RailView struct {
A string `json:"a"`
B string `json:"b"`
RateGbps int `json:"rate_gbps"`
Gbps float64 `json:"gbps"`
Verdict string `json:"verdict"`
Detail string `json:"detail,omitempty"`
}
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
// sends traffic over the fabric for seconds a rail, so it is a POST and only
// ever run when asked.
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
var body struct {
A, B, Fabric string
Seconds int
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
ask := a.ctl.FabricTester()
if ask == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
return
}
inv := a.ctl.Inventory()
na, okA := inv.Get(body.A)
nb, okB := inv.Get(body.B)
if !okA || !okB {
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
return
}
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
start := time.Now()
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
Took: time.Since(start).Round(time.Second).String()}
for _, x := range rep.Results {
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
}
writeJSON(w, http.StatusOK, v)
}
+1
View File
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
Labels: r.GetLabels(),
State: inventory.Ready,
AgentVersion: r.GetAgentVersion(),
Features: r.GetFeatures(),
EngineVersions: r.GetEngineVersions(),
EnginesProbed: r.GetEnginesProbed(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
+47
View File
@@ -19,6 +19,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
@@ -61,6 +62,7 @@ type Service struct {
logSeq uint64
// scanWaits are the fabric scans awaiting a reply, by request id.
scanWaits map[string]chan *llmgwv1.FabricScanReply
testWaits map[string]chan *llmgwv1.FabricTestReply
}
// SetEngines sets the engines every node installs, by name and version.
@@ -289,6 +291,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
"last_output", m.Shard.GetLastOutput())
}
case *llmgwv1.AgentMessage_FabricTest:
s.logMu.Lock()
ch, ok := s.testWaits[m.FabricTest.GetRequestId()]
delete(s.testWaits, m.FabricTest.GetRequestId())
s.logMu.Unlock()
if ok {
ch <- m.FabricTest
}
case *llmgwv1.AgentMessage_FabricScan:
s.logMu.Lock()
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
@@ -717,6 +728,38 @@ func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int)
}
}
// FabricTest asks a node for one side of a bandwidth test, and waits for
// what it measured. The request's id is filled in here.
func (s *Service) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
s.logMu.Lock()
if s.testWaits == nil {
s.testWaits = map[string]chan *llmgwv1.FabricTestReply{}
}
s.logSeq++
id := fmt.Sprintf("test-%d", s.logSeq)
ch := make(chan *llmgwv1.FabricTestReply, 1)
s.testWaits[id] = ch
s.logMu.Unlock()
defer func() {
s.logMu.Lock()
delete(s.testWaits, id)
s.logMu.Unlock()
}()
req.RequestId = id
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricTest{FabricTest: req}}); err != nil {
return nil, err
}
// The test's own length, and a package install the first time.
ctx, cancel := context.WithTimeout(ctx, time.Duration(req.GetSeconds())*time.Second+5*time.Minute)
defer cancel()
select {
case r := <-ch:
return r, nil
case <-ctx.Done():
return nil, fmt.Errorf("node %s did not answer the bandwidth test", nodeID)
}
}
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
// one node on each: the first, by name, whose port on it is up. A fabric
// looks the same from all its ports, so one is enough, and the rest of its
@@ -795,3 +838,7 @@ func installable(reg *llmgwv1.Register) []string {
}
return []string{"llamacpp"}
}
// The admin API finds the bandwidth test by asserting this; a signature that
// drifted would make it quietly nil.
var _ fabrictest.Asker = (*Service)(nil)
+7
View File
@@ -11,6 +11,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
@@ -215,6 +216,12 @@ func (c *Controller) LogReader() LogReader {
return r
}
// FabricTester returns the dispatcher's bandwidth test capability, or nil.
func (c *Controller) FabricTester() fabrictest.Asker {
r, _ := c.disp.(fabrictest.Asker)
return r
}
// Start runs the reconciler and begins publishing endpoints.
func (c *Controller) Start(ctx context.Context) {
if c.lead != nil {
+235
View File
@@ -0,0 +1,235 @@
// Package fabrictest measures the fabric between two nodes, rail by rail.
//
// Every check on one node — ports up, a subnet manager, the right modules —
// can pass while the two nodes still cannot reach each other: a cable into the
// wrong switch, a partition key, a RoCE network with no route. A replica
// spanning them then starts, and hangs on its first all-reduce with nothing to
// say why. This runs what that all-reduce would ride on — an RDMA write from
// each adapter to its counterpart — and says how it went against the link's
// rate, before a replica does.
package fabrictest
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Asker sends one side of a test to a node and waits for its answer.
type Asker interface {
FabricTest(ctx context.Context, node string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error)
}
// Rail is one adapter on each node, paired by their position in the fabric's
// declaration: mlx5_0 with mlx5_0, the way NCCL pairs them.
type Rail struct {
A, B string // adapters
PortA, PortB int
// RateGbps is the slower of the two links.
RateGbps int
}
// Result is one rail's test.
type Result struct {
Rail
Gbps float64 `json:"gbps"`
// Verdict is "ok" at 80% of the link's rate or more, "slow" at 50%,
// "fail" below that or when it did not run.
Verdict string `json:"verdict"`
Detail string `json:"detail,omitempty"`
}
// Report is a whole test.
type Report struct {
Fabric string `json:"fabric"`
A, B string `json:"-"`
Results []Result `json:"results"`
// Notes say what the test could not cover.
Notes []string `json:"notes,omitempty"`
}
// OK reports whether every rail passed.
func (r Report) OK() bool {
for _, x := range r.Results {
if x.Verdict != "ok" {
return false
}
}
return len(r.Results) > 0
}
// Plan pairs the adapters two nodes have on a fabric. With no fabric named,
// the one fabric they share is used.
func Plan(a, b inventory.Node, fabric string) (string, []Rail, []string, error) {
if a.ID == b.ID {
return "", nil, nil, errors.New("give two different nodes")
}
if fabric == "" {
var common []string
for _, f := range a.FabricNames() {
if b.HasFabric(f) {
common = append(common, f)
}
}
switch len(common) {
case 0:
return "", nil, nil, fmt.Errorf("%s (%s) and %s (%s) declare no fabric in common",
a.ID, orNone(a.FabricNames()), b.ID, orNone(b.FabricNames()))
case 1:
fabric = common[0]
default:
return "", nil, nil, fmt.Errorf("%s and %s share %s: name one with --fabric", a.ID, b.ID, strings.Join(common, ", "))
}
}
for _, n := range []inventory.Node{a, b} {
if !slices.Contains(n.Features, "fabric-test") {
return "", nil, nil, fmt.Errorf("%s's agent (%s) is too old to run a bandwidth test: upgrade it", n.ID, orNone([]string{n.AgentVersion}))
}
}
ha, hb := a.HCAsFor(fabric), b.HCAsFor(fabric)
if len(ha) == 0 || len(hb) == 0 {
return "", nil, nil, fmt.Errorf("%s is not declared on both %s and %s", fabric, a.ID, b.ID)
}
var notes []string
if len(ha) != len(hb) {
notes = append(notes, fmt.Sprintf("%s has %d adapters on %s and %s has %d: only the first %d are paired",
a.ID, len(ha), fabric, b.ID, len(hb), min(len(ha), len(hb))))
}
var rails []Rail
for i := range min(len(ha), len(hb)) {
pa, ra := port(a, ha[i])
pb, rb := port(b, hb[i])
rails = append(rails, Rail{A: ha[i], B: hb[i], PortA: pa, PortB: pb, RateGbps: minPositive(ra, rb)})
}
return fabric, rails, notes, nil
}
// port is an adapter's first active port and its rate, or port 1 when the
// node reported none.
func port(n inventory.Node, hca string) (int, int) {
for _, p := range n.IBPorts {
if p.HCA == hca && strings.EqualFold(p.State, "ACTIVE") {
return p.Port, p.RateGbps
}
}
return 1, 0
}
// Options for a run.
type Options struct {
Seconds int
// BasePort is perftest's TCP exchange port for the first rail; each rail
// takes the next, so a rail whose server is still closing never meets the
// next one's.
BasePort int
// Settle is how long the server is given to listen before the client
// dials; the client retries besides.
Settle time.Duration
}
// Run tests every rail, one after another: rails at once would share the
// PCIe and memory bandwidth they are measuring.
func Run(ctx context.Context, ask Asker, a, b inventory.Node, fabric string, rails []Rail, o Options) Report {
if o.Seconds <= 0 {
o.Seconds = 3
}
if o.BasePort == 0 {
o.BasePort = 18515
}
rep := Report{Fabric: fabric, A: a.ID, B: b.ID}
for i, r := range rails {
rep.Results = append(rep.Results, runRail(ctx, ask, a, b, r, o, o.BasePort+i))
}
return rep
}
func runRail(ctx context.Context, ask Asker, a, b inventory.Node, r Rail, o Options, tcp int) Result {
res := Result{Rail: r, Verdict: "fail"}
if b.MgmtAddr == "" {
res.Detail = b.ID + " reported no management address for the client to dial"
return res
}
type answer struct {
r *llmgwv1.FabricTestReply
err error
}
server := make(chan answer, 1)
go func() {
rep, err := ask.FabricTest(ctx, b.ID, &llmgwv1.FabricTestRequest{Role: "server", Hca: r.B,
Port: int32(r.PortB), TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
server <- answer{rep, err}
}()
select {
case <-ctx.Done():
res.Detail = ctx.Err().Error()
return res
case <-time.After(o.Settle):
}
client, err := ask.FabricTest(ctx, a.ID, &llmgwv1.FabricTestRequest{Role: "client", Hca: r.A,
Port: int32(r.PortA), Peer: b.MgmtAddr, TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
var srv answer
select {
case srv = <-server:
case <-time.After(time.Duration(o.Seconds)*time.Second + time.Minute):
srv.err = errors.New("the server side did not finish")
}
switch {
case err != nil:
res.Detail = a.ID + ": " + err.Error()
case client.GetError() != "":
res.Detail = a.ID + ": " + client.GetError()
// The server's reason is often the better one: it saw no one arrive.
if srv.r != nil && srv.r.GetError() != "" {
res.Detail += "; " + b.ID + ": " + srv.r.GetError()
}
default:
res.Gbps = client.GetGbps()
res.Verdict, res.Detail = grade(res.Gbps, r.RateGbps)
}
return res
}
// grade compares a measurement with the link's rate. RDMA writes of 64 KiB
// reach 90–97% of the signalling rate on a healthy link; well under that is
// a degraded cable, a card in a narrow PCIe slot, or traffic crossing a
// socket.
func grade(gbps float64, rate int) (string, string) {
if gbps <= 0 {
return "fail", "no traffic got through"
}
if rate <= 0 {
return "ok", "the link's rate is not known to compare with"
}
pct := gbps / float64(rate) * 100
switch {
case pct >= 80:
return "ok", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
case pct >= 50:
return "slow", fmt.Sprintf("%.0f%% of %d Gb/s: check the cable, and the card's PCIe width and NUMA node", pct, rate)
default:
return "fail", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
}
}
func minPositive(a, b int) int {
switch {
case a <= 0:
return b
case b <= 0:
return a
}
return min(a, b)
}
func orNone(s []string) string {
if len(s) == 0 {
return "none"
}
return strings.Join(slices.Clone(s), ", ")
}
@@ -0,0 +1,100 @@
package fabrictest
import (
"context"
"strings"
"sync"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func node(id string, hcas ...string) inventory.Node {
n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"},
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}}
for _, h := range hcas {
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400})
}
return n
}
// fakeFabric answers as two agents would: the server waits for its client.
type fakeFabric struct {
mu sync.Mutex
gbps map[string]float64 // by client adapter
fails map[string]string
asked []string
}
func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca())
if req.GetRole() == "server" {
return &llmgwv1.FabricTestReply{}, nil
}
if e := f.fails[req.GetHca()]; e != "" {
return &llmgwv1.FabricTestReply{Error: e}, nil
}
return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil
}
func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) {
a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
fabric, rails, notes, err := Plan(a, b, "")
if err != nil {
t.Fatal(err)
}
if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 {
t.Errorf("got %s %+v", fabric, rails)
}
if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") {
t.Errorf("an unequal count went unmentioned: %v", notes)
}
}
func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) {
b := node("n2", "mlx5_0")
b.Fabrics[0].Fabric = "ib-b"
if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") {
t.Errorf("got %v", err)
}
}
func TestEachRailIsGradedAgainstItsLink(t *testing.T) {
a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
_, rails, _, _ := Plan(a, b, "ib-a")
f := &fakeFabric{
gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240},
fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"},
}
rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond})
want := []string{"ok", "slow", "fail"}
for i, r := range rep.Results {
if r.Verdict != want[i] {
t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i])
}
}
if rep.OK() {
t.Error("a report with a failed rail is OK")
}
if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") {
t.Errorf("the reason was lost: %q", rep.Results[2].Detail)
}
// Servers on the second node, clients on the first, one rail at a time.
if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" {
t.Errorf("asked %v", f.asked)
}
}
func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) {
// It would ignore the request, and the control plane would wait minutes.
old := node("n2", "mlx5_0")
old.Features, old.AgentVersion = nil, "v0.10.0"
_, _, _, err := Plan(node("n1", "mlx5_0"), old, "")
if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") {
t.Errorf("got %v", err)
}
}
+4 -1
View File
@@ -100,7 +100,10 @@ type Node struct {
LastSeen time.Time `json:"last_seen"`
// AgentVersion and EngineVersions are reported at registration. The engine
// version decides which metric names and CLI flags apply (plan §14).
AgentVersion string `json:"agent_version,omitempty"`
AgentVersion string `json:"agent_version,omitempty"`
// Features are the requests the agent answers beyond the ones every
// agent does.
Features []string `json:"features,omitempty"`
EngineVersions map[string]string `json:"engine_versions,omitempty"`
// EnginesProbed says EngineVersions is an answer, so an empty map means
// "has none" rather than "an agent too old to say".