Ilker ManapandClaude Opus 5.5
7bb360fc78
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
76 lines
2.6 KiB
Go
76 lines
2.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|
var (
|
|
fabric string
|
|
seconds int
|
|
)
|
|
c := &cobra.Command{
|
|
Use: "fabric-test NODE-A NODE-B",
|
|
Short: "Measure RDMA bandwidth between two nodes, rail by rail",
|
|
Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
|
|
adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
|
|
one rail at a time, and grades each against its link's rate: ok at 80% or
|
|
more, slow at 50%, fail below or when nothing got through.
|
|
|
|
Every check on one node can pass while two nodes still cannot reach each
|
|
other: a cable into the wrong switch, a partition key, a RoCE network with no
|
|
route. A replica spanning them then hangs on its first all-reduce. Run this
|
|
before placing one.
|
|
|
|
It goes through the agents, so it needs no ssh between nodes. perftest is
|
|
installed on a node the first time it is asked. The traffic is real and
|
|
takes a few seconds a rail; run it when the fabric is quiet.`,
|
|
Example: ` llmgw ctl fabric-test gpu01 gpu02
|
|
llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
|
|
Args: cobra.ExactArgs(2),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
|
|
cl := client()
|
|
// Rails one after another, each for its seconds and a settle; the
|
|
// client's usual timeout is for questions, not measurements.
|
|
cl.http.Timeout = 30 * time.Minute
|
|
var v adminapi.FabricTestView
|
|
if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
|
|
return err
|
|
}
|
|
if *asJSON {
|
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
|
|
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
|
fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
|
|
for _, r := range v.Rails {
|
|
link := "-"
|
|
if r.RateGbps > 0 {
|
|
link = fmt.Sprintf("%d Gb/s", r.RateGbps)
|
|
}
|
|
fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
|
|
}
|
|
_ = w.Flush()
|
|
for _, n := range v.Notes {
|
|
fmt.Fprintf(out, "\nnote: %s\n", n)
|
|
}
|
|
if !v.OK {
|
|
return fmt.Errorf("not every rail passed")
|
|
}
|
|
return nil
|
|
},
|
|
}
|
|
c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
|
|
c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
|
|
return c
|
|
}
|