Measure two nodes' fabric rail by rail: llmgw ctl fabric-test
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
af6f8e023e
commit
7bb360fc78
21 files changed
+1265
-62
No files matched your search
@@ -120,6 +120,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newModelsCmd(client, &asJSON),
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
newFabricTestCmd(client, &asJSON),
|
||||
newNodeStateCmd(client, "cordon", "cordoned",
|
||||
"Stop placing new work on a node; what runs there keeps running"),
|
||||
newNodeStateCmd(client, "uncordon", "ready",
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var (
|
||||
fabric string
|
||||
seconds int
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "fabric-test NODE-A NODE-B",
|
||||
Short: "Measure RDMA bandwidth between two nodes, rail by rail",
|
||||
Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
|
||||
adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
|
||||
one rail at a time, and grades each against its link's rate: ok at 80% or
|
||||
more, slow at 50%, fail below or when nothing got through.
|
||||
|
||||
Every check on one node can pass while two nodes still cannot reach each
|
||||
other: a cable into the wrong switch, a partition key, a RoCE network with no
|
||||
route. A replica spanning them then hangs on its first all-reduce. Run this
|
||||
before placing one.
|
||||
|
||||
It goes through the agents, so it needs no ssh between nodes. perftest is
|
||||
installed on a node the first time it is asked. The traffic is real and
|
||||
takes a few seconds a rail; run it when the fabric is quiet.`,
|
||||
Example: ` llmgw ctl fabric-test gpu01 gpu02
|
||||
llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
|
||||
cl := client()
|
||||
// Rails one after another, each for its seconds and a settle; the
|
||||
// client's usual timeout is for questions, not measurements.
|
||||
cl.http.Timeout = 30 * time.Minute
|
||||
var v adminapi.FabricTestView
|
||||
if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
|
||||
for _, r := range v.Rails {
|
||||
link := "-"
|
||||
if r.RateGbps > 0 {
|
||||
link = fmt.Sprintf("%d Gb/s", r.RateGbps)
|
||||
}
|
||||
fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
|
||||
}
|
||||
_ = w.Flush()
|
||||
for _, n := range v.Notes {
|
||||
fmt.Fprintf(out, "\nnote: %s\n", n)
|
||||
}
|
||||
if !v.OK {
|
||||
return fmt.Errorf("not every rail passed")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
|
||||
c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
|
||||
return c
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
)
|
||||
|
||||
func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
||||
@@ -23,10 +24,17 @@ func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
||||
t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
|
||||
}
|
||||
|
||||
if _, ok := ref.(fabrictest.Asker); !ok {
|
||||
t.Error("dispatcherRef drops the bandwidth test, so ctl fabric-test cannot reach the agents")
|
||||
}
|
||||
|
||||
// And the thing it forwards to must have it, or the forwarding has nothing
|
||||
// to reach.
|
||||
var svc any = &agentsvc.Service{}
|
||||
if _, ok := svc.(control.WeightRemover); !ok {
|
||||
t.Error("the agent service does not implement WeightRemover")
|
||||
}
|
||||
if _, ok := svc.(fabrictest.Asker); !ok {
|
||||
t.Error("the agent service does not implement the bandwidth test")
|
||||
}
|
||||
}
|
||||
@@ -21,11 +21,13 @@ import (
|
||||
"golang.org/x/net/http2/h2c"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -493,6 +495,18 @@ func (r *dispatcherRef) Logs(ctx context.Context, nodeID, source string, lines i
|
||||
return lr.Logs(ctx, nodeID, source, lines)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ft, ok := d.(fabrictest.Asker)
|
||||
if !ok {
|
||||
return nil, errors.New("this dispatcher cannot run a bandwidth test")
|
||||
}
|
||||
return ft.FabricTest(ctx, nodeID, req)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user