Measure two nodes' fabric rail by rail: llmgw ctl fabric-test

doctor checks a node alone, and every check can pass while two nodes still
cannot reach each other. ctl fabric-test A B asks B's agent for an
ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs
them, one rail at a time, and grades each against its link's rate. It goes
through the agents' streams, so there is no SSH between nodes; the agent
installs perftest the first time, and names the RoCE v2 GID itself.

The result is read from perftest's table by its heading, not a column
number. Agents now list the requests they answer beyond the usual
(Register.features), and the control plane refuses a test on one too old to
answer instead of waiting minutes for it. The forwarding dispatcher carries
the new capability, and its test says so: a missing forward would have made
the command quietly unavailable in production.

Not yet run over real InfiniBand or RoCE; this cluster has neither.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 21:29:23 +02:00
1 parent af6f8e023e
commit 7bb360fc78
21 files changed
+1265 -62

No files matched your search

+75
View File
@@ -844,6 +844,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
reg.EngineVersions = a.engines(parent)
reg.EnginesProbed = true
reg.InstallableEngines = provision.Installable()
reg.Features = []string{FeatureFabricTest}
reg.RunningShards = a.runningShards()
a.mu.Lock()
h := a.prov.Host
@@ -1145,6 +1146,16 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
case <-ctx.Done():
}
}(m.FabricScan)
case *llmgwv1.ControlMessage_FabricTest:
// Seconds of traffic, and a package install the first time: off the
// receive loop too.
go func(req *llmgwv1.FabricTestRequest) {
reply := a.fabricTest(ctx, req)
select {
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricTest{FabricTest: reply}}:
case <-ctx.Done():
}
}(m.FabricTest)
case *llmgwv1.ControlMessage_Ack:
if !m.Ack.GetOk() {
a.log.Warn("control plane rejected a message",
@@ -1758,6 +1769,70 @@ func (a *Agent) ensureDiags(ctx context.Context) {
}
}
// FeatureFabricTest is the agent answering FabricTestRequest.
const FeatureFabricTest = "fabric-test"
// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
// installed; both replaced in tests.
var (
runWriteBW = discovery.RunWriteBW
haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
)
// fabricTest runs this node's side of a bandwidth test with another.
func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
start := time.Now()
reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
if req.GetRole() == "client" {
w.Peer = req.GetPeer()
}
a.mu.Lock()
found := false
for _, h := range a.inv.HCAs {
for _, p := range h.Ports {
if h.Name == w.HCA && p.Num == w.Port {
found = true
// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
w.GIDIndex = p.RoCEv2GID
}
}
}
}
a.mu.Unlock()
if !found {
reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
return reply
}
if !haveWriteBW() && a.prov != nil {
a.log.Info("installing perftest, for a bandwidth test with another node")
pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
reply.Error = "installing perftest: " + err.Error()
return reply
}
}
gbps, out, err := runWriteBW(ctx, w)
reply.Gbps = gbps
reply.Output = tailLines(out, 12)
if err != nil {
reply.Error = err.Error()
}
return reply
}
func tailLines(s string, n int) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
if len(lines) > n {
lines = lines[len(lines)-n:]
}
return strings.Join(lines, "\n")
}
// discoverGPUFabric and discoverIMEX are replaced in tests.
var (
discoverGPUFabric = discovery.DiscoverGPUFabric
+32
View File
@@ -23,6 +23,7 @@ import (
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
@@ -881,3 +882,34 @@ func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
t.Errorf("asked %q, got %+v", asked, r)
}
}
func TestABandwidthTestIsRunOnTheAdapterAskedFor(t *testing.T) {
oldRun, oldHave := runWriteBW, haveWriteBW
t.Cleanup(func() { runWriteBW, haveWriteBW = oldRun, oldHave })
haveWriteBW = func() bool { return true }
var got discovery.WriteBW
runWriteBW = func(_ context.Context, w discovery.WriteBW) (float64, string, error) {
got = w
return 388.2, " #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec]\n", nil
}
h := start(t, func(c *Config) { c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") })
r, err := h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{
Role: "client", Hca: "mlx5_0", Port: 1, Peer: "10.0.0.2", TcpPort: 18515, Seconds: 3})
if err != nil {
t.Fatal(err)
}
if r.GetGbps() != 388.2 || r.GetError() != "" {
t.Errorf("reply %+v", r)
}
if got.HCA != "mlx5_0" || got.Peer != "10.0.0.2" || got.TCPPort != 18515 || got.GIDIndex != -1 {
t.Errorf("ran %+v", got)
}
// An adapter the node does not have is said so, without running anything.
got = discovery.WriteBW{}
r, _ = h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{Role: "server", Hca: "mlx5_9", Port: 1, Seconds: 1})
if !strings.Contains(r.GetError(), "no port mlx5_9/1") || got.HCA != "" {
t.Errorf("reply %+v, ran %+v", r, got)
}
}
+133
View File
@@ -0,0 +1,133 @@
package discovery
import (
"context"
"errors"
"fmt"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// WriteBW is one side of an ib_write_bw run, from perftest: an RDMA write
// bandwidth test between two adapters, which is what a collective spanning
// nodes rides on.
type WriteBW struct {
HCA string
Port int
// GIDIndex is the RoCE v2 GID to use; negative on InfiniBand, where the
// port's LID is used.
GIDIndex int
// Peer is the server's address, for the client; empty makes the server.
Peer string
TCPPort int
Seconds int
}
// Args are the command line: a duration rather than a count, so the test
// takes as long on a slow link as a fast one, and results in Gb/s.
func (w WriteBW) Args() []string {
args := []string{"-d", w.HCA, "-i", strconv.Itoa(max(w.Port, 1)),
"-p", strconv.Itoa(w.TCPPort), "-D", strconv.Itoa(max(w.Seconds, 1)),
"-F", "--report_gbits"}
if w.GIDIndex >= 0 {
args = append(args, "-x", strconv.Itoa(w.GIDIndex))
}
if w.Peer != "" {
args = append(args, w.Peer)
}
return args
}
// RunWriteBW runs one side and returns the average bandwidth the output
// reports, in Gb/s, and the output. The client retries its connection for a
// while: the server is started first, but a few seconds may pass before it
// listens.
func RunWriteBW(ctx context.Context, w WriteBW) (float64, string, error) {
bin, err := exec.LookPath("ib_write_bw")
if err != nil {
return 0, "", errors.New("ib_write_bw is not installed (the perftest package)")
}
limit := time.Duration(w.Seconds+60) * time.Second
var out []byte
for attempt := 0; ; attempt++ {
cctx, cancel := context.WithTimeout(ctx, limit)
out, err = exec.CommandContext(cctx, bin, w.Args()...).CombinedOutput() //nolint:gosec // fixed tool
cancel()
if err == nil || w.Peer == "" || !refused(string(out)) || attempt >= 10 {
break
}
select {
case <-ctx.Done():
return 0, string(out), ctx.Err()
case <-time.After(time.Second):
}
}
text := string(out)
if err != nil {
return 0, text, fmt.Errorf("ib_write_bw: %v: %s", err, lastNonEmpty(text))
}
gbps, perr := ParseWriteBW(text)
if perr != nil && w.Peer != "" {
// Only the client is sure to print the result; a server that does not
// is not a failure.
return 0, text, perr
}
return gbps, text, nil
}
func refused(out string) bool {
for _, s := range []string{"Couldn't connect", "Unable to init the socket connection", "Connection refused"} {
if strings.Contains(out, s) {
return true
}
}
return false
}
var columnSplit = regexp.MustCompile(`\s{2,}`)
// ParseWriteBW finds the average bandwidth in perftest's result table:
//
// #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
// 65536 887036 0.00 96.93 0.184877
//
// by the heading's position rather than a fixed column, since versions add
// columns and the peak reads 0.00 when the test runs for a duration.
func ParseWriteBW(out string) (float64, error) {
lines := strings.Split(out, "\n")
for i, l := range lines {
if !strings.Contains(l, "BW average") {
continue
}
heads := columnSplit.Split(strings.TrimSpace(l), -1)
col := -1
for j, h := range heads {
if strings.HasPrefix(h, "BW average") {
col = j
}
}
for _, d := range lines[i+1:] {
f := strings.Fields(d)
if len(f) == 0 || strings.HasPrefix(f[0], "-") {
continue
}
if len(f) != len(heads) || col < 0 {
return 0, fmt.Errorf("perftest's result table has a shape this does not know: %q / %q", l, d)
}
v, err := strconv.ParseFloat(f[col], 64)
if err != nil {
return 0, fmt.Errorf("reading the average bandwidth %q: %w", f[col], err)
}
return v, nil
}
}
return 0, errors.New("perftest printed no result")
}
func lastNonEmpty(s string) string {
lines := strings.Split(strings.TrimSpace(s), "\n")
return strings.TrimSpace(lines[len(lines)-1])
}
+49
View File
@@ -0,0 +1,49 @@
package discovery
import (
"slices"
"testing"
)
// The client's output of perftest 24.x, with a duration.
const writeBWClient = `---------------------------------------------------------------------------------------
RDMA_Write BW Test
Dual-port : OFF Device : mlx5_0
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
TX depth : 128
CQ Moderation : 100
Mtu : 4096[B]
Link type : IB
Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
---------------------------------------------------------------------------------------
#bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
65536 1851214 0.00 388.21 0.740422
---------------------------------------------------------------------------------------
`
func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
got, err := ParseWriteBW(writeBWClient)
if err != nil || got != 388.21 {
t.Errorf("got %v, %v", got, err)
}
if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
t.Error("output with no result table gave a number")
}
}
func TestWriteBWArguments(t *testing.T) {
ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
if slices.Contains(ib, "-x") {
t.Errorf("InfiniBand was given a GID index: %v", ib)
}
roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
t.Errorf("RoCE client: %v", roce)
}
}
+1
View File
@@ -120,6 +120,7 @@ func newCtlCmd() *cobra.Command {
newModelsCmd(client, &asJSON),
newReplicasCmd(client, &asJSON),
newExplainCmd(client, &asJSON),
newFabricTestCmd(client, &asJSON),
newNodeStateCmd(client, "cordon", "cordoned",
"Stop placing new work on a node; what runs there keeps running"),
newNodeStateCmd(client, "uncordon", "ready",
+75
View File
@@ -0,0 +1,75 @@
package cli
import (
"encoding/json"
"fmt"
"text/tabwriter"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var (
fabric string
seconds int
)
c := &cobra.Command{
Use: "fabric-test NODE-A NODE-B",
Short: "Measure RDMA bandwidth between two nodes, rail by rail",
Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
one rail at a time, and grades each against its link's rate: ok at 80% or
more, slow at 50%, fail below or when nothing got through.
Every check on one node can pass while two nodes still cannot reach each
other: a cable into the wrong switch, a partition key, a RoCE network with no
route. A replica spanning them then hangs on its first all-reduce. Run this
before placing one.
It goes through the agents, so it needs no ssh between nodes. perftest is
installed on a node the first time it is asked. The traffic is real and
takes a few seconds a rail; run it when the fabric is quiet.`,
Example: ` llmgw ctl fabric-test gpu01 gpu02
llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
cl := client()
// Rails one after another, each for its seconds and a settle; the
// client's usual timeout is for questions, not measurements.
cl.http.Timeout = 30 * time.Minute
var v adminapi.FabricTestView
if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
return err
}
if *asJSON {
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
for _, r := range v.Rails {
link := "-"
if r.RateGbps > 0 {
link = fmt.Sprintf("%d Gb/s", r.RateGbps)
}
fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
}
_ = w.Flush()
for _, n := range v.Notes {
fmt.Fprintf(out, "\nnote: %s\n", n)
}
if !v.OK {
return fmt.Errorf("not every rail passed")
}
return nil
},
}
c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
return c
}
+8
View File
@@ -5,6 +5,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
)
func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
@@ -23,10 +24,17 @@ func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
}
if _, ok := ref.(fabrictest.Asker); !ok {
t.Error("dispatcherRef drops the bandwidth test, so ctl fabric-test cannot reach the agents")
}
// And the thing it forwards to must have it, or the forwarding has nothing
// to reach.
var svc any = &agentsvc.Service{}
if _, ok := svc.(control.WeightRemover); !ok {
t.Error("the agent service does not implement WeightRemover")
}
if _, ok := svc.(fabrictest.Asker); !ok {
t.Error("the agent service does not implement the bandwidth test")
}
}
+14
View File
@@ -21,11 +21,13 @@ import (
"golang.org/x/net/http2/h2c"
"golang.org/x/sync/errgroup"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -493,6 +495,18 @@ func (r *dispatcherRef) Logs(ctx context.Context, nodeID, source string, lines i
return lr.Logs(ctx, nodeID, source, lines)
}
func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
d, err := r.get()
if err != nil {
return nil, err
}
ft, ok := d.(fabrictest.Asker)
if !ok {
return nil, errors.New("this dispatcher cannot run a bandwidth test")
}
return ft.FabricTest(ctx, nodeID, req)
}
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
d, err := r.get()
if err != nil {
+1
View File
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
}
+71
View File
@@ -0,0 +1,71 @@
package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
)
// FabricTestView is a bandwidth test between two nodes.
type FabricTestView struct {
A string `json:"a"`
B string `json:"b"`
Fabric string `json:"fabric"`
OK bool `json:"ok"`
Rails []RailView `json:"rails"`
Notes []string `json:"notes,omitempty"`
Took string `json:"took"`
}
// RailView is one rail's result.
type RailView struct {
A string `json:"a"`
B string `json:"b"`
RateGbps int `json:"rate_gbps"`
Gbps float64 `json:"gbps"`
Verdict string `json:"verdict"`
Detail string `json:"detail,omitempty"`
}
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
// sends traffic over the fabric for seconds a rail, so it is a POST and only
// ever run when asked.
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
var body struct {
A, B, Fabric string
Seconds int
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
ask := a.ctl.FabricTester()
if ask == nil {
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
return
}
inv := a.ctl.Inventory()
na, okA := inv.Get(body.A)
nb, okB := inv.Get(body.B)
if !okA || !okB {
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
return
}
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
start := time.Now()
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
Took: time.Since(start).Round(time.Second).String()}
for _, x := range rep.Results {
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
}
writeJSON(w, http.StatusOK, v)
}
+1
View File
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
Labels: r.GetLabels(),
State: inventory.Ready,
AgentVersion: r.GetAgentVersion(),
Features: r.GetFeatures(),
EngineVersions: r.GetEngineVersions(),
EnginesProbed: r.GetEnginesProbed(),
MemoryTotalMB: int(r.GetMemoryTotalMb()),
+47
View File
@@ -19,6 +19,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
)
@@ -61,6 +62,7 @@ type Service struct {
logSeq uint64
// scanWaits are the fabric scans awaiting a reply, by request id.
scanWaits map[string]chan *llmgwv1.FabricScanReply
testWaits map[string]chan *llmgwv1.FabricTestReply
}
// SetEngines sets the engines every node installs, by name and version.
@@ -289,6 +291,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
"last_output", m.Shard.GetLastOutput())
}
case *llmgwv1.AgentMessage_FabricTest:
s.logMu.Lock()
ch, ok := s.testWaits[m.FabricTest.GetRequestId()]
delete(s.testWaits, m.FabricTest.GetRequestId())
s.logMu.Unlock()
if ok {
ch <- m.FabricTest
}
case *llmgwv1.AgentMessage_FabricScan:
s.logMu.Lock()
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
@@ -717,6 +728,38 @@ func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int)
}
}
// FabricTest asks a node for one side of a bandwidth test, and waits for
// what it measured. The request's id is filled in here.
func (s *Service) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
s.logMu.Lock()
if s.testWaits == nil {
s.testWaits = map[string]chan *llmgwv1.FabricTestReply{}
}
s.logSeq++
id := fmt.Sprintf("test-%d", s.logSeq)
ch := make(chan *llmgwv1.FabricTestReply, 1)
s.testWaits[id] = ch
s.logMu.Unlock()
defer func() {
s.logMu.Lock()
delete(s.testWaits, id)
s.logMu.Unlock()
}()
req.RequestId = id
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricTest{FabricTest: req}}); err != nil {
return nil, err
}
// The test's own length, and a package install the first time.
ctx, cancel := context.WithTimeout(ctx, time.Duration(req.GetSeconds())*time.Second+5*time.Minute)
defer cancel()
select {
case r := <-ch:
return r, nil
case <-ctx.Done():
return nil, fmt.Errorf("node %s did not answer the bandwidth test", nodeID)
}
}
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
// one node on each: the first, by name, whose port on it is up. A fabric
// looks the same from all its ports, so one is enough, and the rest of its
@@ -795,3 +838,7 @@ func installable(reg *llmgwv1.Register) []string {
}
return []string{"llamacpp"}
}
// The admin API finds the bandwidth test by asserting this; a signature that
// drifted would make it quietly nil.
var _ fabrictest.Asker = (*Service)(nil)
+7
View File
@@ -11,6 +11,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
@@ -215,6 +216,12 @@ func (c *Controller) LogReader() LogReader {
return r
}
// FabricTester returns the dispatcher's bandwidth test capability, or nil.
func (c *Controller) FabricTester() fabrictest.Asker {
r, _ := c.disp.(fabrictest.Asker)
return r
}
// Start runs the reconciler and begins publishing endpoints.
func (c *Controller) Start(ctx context.Context) {
if c.lead != nil {
+235
View File
@@ -0,0 +1,235 @@
// Package fabrictest measures the fabric between two nodes, rail by rail.
//
// Every check on one node — ports up, a subnet manager, the right modules —
// can pass while the two nodes still cannot reach each other: a cable into the
// wrong switch, a partition key, a RoCE network with no route. A replica
// spanning them then starts, and hangs on its first all-reduce with nothing to
// say why. This runs what that all-reduce would ride on — an RDMA write from
// each adapter to its counterpart — and says how it went against the link's
// rate, before a replica does.
package fabrictest
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// Asker sends one side of a test to a node and waits for its answer.
type Asker interface {
FabricTest(ctx context.Context, node string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error)
}
// Rail is one adapter on each node, paired by their position in the fabric's
// declaration: mlx5_0 with mlx5_0, the way NCCL pairs them.
type Rail struct {
A, B string // adapters
PortA, PortB int
// RateGbps is the slower of the two links.
RateGbps int
}
// Result is one rail's test.
type Result struct {
Rail
Gbps float64 `json:"gbps"`
// Verdict is "ok" at 80% of the link's rate or more, "slow" at 50%,
// "fail" below that or when it did not run.
Verdict string `json:"verdict"`
Detail string `json:"detail,omitempty"`
}
// Report is a whole test.
type Report struct {
Fabric string `json:"fabric"`
A, B string `json:"-"`
Results []Result `json:"results"`
// Notes say what the test could not cover.
Notes []string `json:"notes,omitempty"`
}
// OK reports whether every rail passed.
func (r Report) OK() bool {
for _, x := range r.Results {
if x.Verdict != "ok" {
return false
}
}
return len(r.Results) > 0
}
// Plan pairs the adapters two nodes have on a fabric. With no fabric named,
// the one fabric they share is used.
func Plan(a, b inventory.Node, fabric string) (string, []Rail, []string, error) {
if a.ID == b.ID {
return "", nil, nil, errors.New("give two different nodes")
}
if fabric == "" {
var common []string
for _, f := range a.FabricNames() {
if b.HasFabric(f) {
common = append(common, f)
}
}
switch len(common) {
case 0:
return "", nil, nil, fmt.Errorf("%s (%s) and %s (%s) declare no fabric in common",
a.ID, orNone(a.FabricNames()), b.ID, orNone(b.FabricNames()))
case 1:
fabric = common[0]
default:
return "", nil, nil, fmt.Errorf("%s and %s share %s: name one with --fabric", a.ID, b.ID, strings.Join(common, ", "))
}
}
for _, n := range []inventory.Node{a, b} {
if !slices.Contains(n.Features, "fabric-test") {
return "", nil, nil, fmt.Errorf("%s's agent (%s) is too old to run a bandwidth test: upgrade it", n.ID, orNone([]string{n.AgentVersion}))
}
}
ha, hb := a.HCAsFor(fabric), b.HCAsFor(fabric)
if len(ha) == 0 || len(hb) == 0 {
return "", nil, nil, fmt.Errorf("%s is not declared on both %s and %s", fabric, a.ID, b.ID)
}
var notes []string
if len(ha) != len(hb) {
notes = append(notes, fmt.Sprintf("%s has %d adapters on %s and %s has %d: only the first %d are paired",
a.ID, len(ha), fabric, b.ID, len(hb), min(len(ha), len(hb))))
}
var rails []Rail
for i := range min(len(ha), len(hb)) {
pa, ra := port(a, ha[i])
pb, rb := port(b, hb[i])
rails = append(rails, Rail{A: ha[i], B: hb[i], PortA: pa, PortB: pb, RateGbps: minPositive(ra, rb)})
}
return fabric, rails, notes, nil
}
// port is an adapter's first active port and its rate, or port 1 when the
// node reported none.
func port(n inventory.Node, hca string) (int, int) {
for _, p := range n.IBPorts {
if p.HCA == hca && strings.EqualFold(p.State, "ACTIVE") {
return p.Port, p.RateGbps
}
}
return 1, 0
}
// Options for a run.
type Options struct {
Seconds int
// BasePort is perftest's TCP exchange port for the first rail; each rail
// takes the next, so a rail whose server is still closing never meets the
// next one's.
BasePort int
// Settle is how long the server is given to listen before the client
// dials; the client retries besides.
Settle time.Duration
}
// Run tests every rail, one after another: rails at once would share the
// PCIe and memory bandwidth they are measuring.
func Run(ctx context.Context, ask Asker, a, b inventory.Node, fabric string, rails []Rail, o Options) Report {
if o.Seconds <= 0 {
o.Seconds = 3
}
if o.BasePort == 0 {
o.BasePort = 18515
}
rep := Report{Fabric: fabric, A: a.ID, B: b.ID}
for i, r := range rails {
rep.Results = append(rep.Results, runRail(ctx, ask, a, b, r, o, o.BasePort+i))
}
return rep
}
func runRail(ctx context.Context, ask Asker, a, b inventory.Node, r Rail, o Options, tcp int) Result {
res := Result{Rail: r, Verdict: "fail"}
if b.MgmtAddr == "" {
res.Detail = b.ID + " reported no management address for the client to dial"
return res
}
type answer struct {
r *llmgwv1.FabricTestReply
err error
}
server := make(chan answer, 1)
go func() {
rep, err := ask.FabricTest(ctx, b.ID, &llmgwv1.FabricTestRequest{Role: "server", Hca: r.B,
Port: int32(r.PortB), TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
server <- answer{rep, err}
}()
select {
case <-ctx.Done():
res.Detail = ctx.Err().Error()
return res
case <-time.After(o.Settle):
}
client, err := ask.FabricTest(ctx, a.ID, &llmgwv1.FabricTestRequest{Role: "client", Hca: r.A,
Port: int32(r.PortA), Peer: b.MgmtAddr, TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
var srv answer
select {
case srv = <-server:
case <-time.After(time.Duration(o.Seconds)*time.Second + time.Minute):
srv.err = errors.New("the server side did not finish")
}
switch {
case err != nil:
res.Detail = a.ID + ": " + err.Error()
case client.GetError() != "":
res.Detail = a.ID + ": " + client.GetError()
// The server's reason is often the better one: it saw no one arrive.
if srv.r != nil && srv.r.GetError() != "" {
res.Detail += "; " + b.ID + ": " + srv.r.GetError()
}
default:
res.Gbps = client.GetGbps()
res.Verdict, res.Detail = grade(res.Gbps, r.RateGbps)
}
return res
}
// grade compares a measurement with the link's rate. RDMA writes of 64 KiB
// reach 90–97% of the signalling rate on a healthy link; well under that is
// a degraded cable, a card in a narrow PCIe slot, or traffic crossing a
// socket.
func grade(gbps float64, rate int) (string, string) {
if gbps <= 0 {
return "fail", "no traffic got through"
}
if rate <= 0 {
return "ok", "the link's rate is not known to compare with"
}
pct := gbps / float64(rate) * 100
switch {
case pct >= 80:
return "ok", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
case pct >= 50:
return "slow", fmt.Sprintf("%.0f%% of %d Gb/s: check the cable, and the card's PCIe width and NUMA node", pct, rate)
default:
return "fail", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
}
}
func minPositive(a, b int) int {
switch {
case a <= 0:
return b
case b <= 0:
return a
}
return min(a, b)
}
func orNone(s []string) string {
if len(s) == 0 {
return "none"
}
return strings.Join(slices.Clone(s), ", ")
}
@@ -0,0 +1,100 @@
package fabrictest
import (
"context"
"strings"
"sync"
"testing"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func node(id string, hcas ...string) inventory.Node {
n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"},
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}}
for _, h := range hcas {
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400})
}
return n
}
// fakeFabric answers as two agents would: the server waits for its client.
type fakeFabric struct {
mu sync.Mutex
gbps map[string]float64 // by client adapter
fails map[string]string
asked []string
}
func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca())
if req.GetRole() == "server" {
return &llmgwv1.FabricTestReply{}, nil
}
if e := f.fails[req.GetHca()]; e != "" {
return &llmgwv1.FabricTestReply{Error: e}, nil
}
return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil
}
func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) {
a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
fabric, rails, notes, err := Plan(a, b, "")
if err != nil {
t.Fatal(err)
}
if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 {
t.Errorf("got %s %+v", fabric, rails)
}
if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") {
t.Errorf("an unequal count went unmentioned: %v", notes)
}
}
func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) {
b := node("n2", "mlx5_0")
b.Fabrics[0].Fabric = "ib-b"
if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") {
t.Errorf("got %v", err)
}
}
func TestEachRailIsGradedAgainstItsLink(t *testing.T) {
a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
_, rails, _, _ := Plan(a, b, "ib-a")
f := &fakeFabric{
gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240},
fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"},
}
rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond})
want := []string{"ok", "slow", "fail"}
for i, r := range rep.Results {
if r.Verdict != want[i] {
t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i])
}
}
if rep.OK() {
t.Error("a report with a failed rail is OK")
}
if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") {
t.Errorf("the reason was lost: %q", rep.Results[2].Detail)
}
// Servers on the second node, clients on the first, one rail at a time.
if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" {
t.Errorf("asked %v", f.asked)
}
}
func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) {
// It would ignore the request, and the control plane would wait minutes.
old := node("n2", "mlx5_0")
old.Features, old.AgentVersion = nil, "v0.10.0"
_, _, _, err := Plan(node("n1", "mlx5_0"), old, "")
if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") {
t.Errorf("got %v", err)
}
}
+4 -1
View File
@@ -100,7 +100,10 @@ type Node struct {
LastSeen time.Time `json:"last_seen"`
// AgentVersion and EngineVersions are reported at registration. The engine
// version decides which metric names and CLI flags apply (plan §14).
AgentVersion string `json:"agent_version,omitempty"`
AgentVersion string `json:"agent_version,omitempty"`
// Features are the requests the agent answers beyond the ones every
// agent does.
Features []string `json:"features,omitempty"`
EngineVersions map[string]string `json:"engine_versions,omitempty"`
// EnginesProbed says EngineVersions is an answer, so an empty map means
// "has none" rather than "an agent too old to say".