package cli import ( "encoding/json" "fmt" "text/tabwriter" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var ( fabric string seconds int ) c := &cobra.Command{ Use: "fabric-test NODE-A NODE-B", Short: "Measure RDMA bandwidth between two nodes, rail by rail", Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them — one rail at a time, and grades each against its link's rate: ok at 80% or more, slow at 50%, fail below or when nothing got through. Every check on one node can pass while two nodes still cannot reach each other: a cable into the wrong switch, a partition key, a RoCE network with no route. A replica spanning them then hangs on its first all-reduce. Run this before placing one. It goes through the agents, so it needs no ssh between nodes. perftest is installed on a node the first time it is asked. The traffic is real and takes a few seconds a rail; run it when the fabric is quiet.`, Example: ` llmgw ctl fabric-test gpu01 gpu02 llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`, Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds}) cl := client() // Rails one after another, each for its seconds and a settle; the // client's usual timeout is for questions, not measurements. cl.http.Timeout = 30 * time.Minute var v adminapi.FabricTestView if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil { return err } if *asJSON { return json.NewEncoder(cmd.OutOrStdout()).Encode(v) } out := cmd.OutOrStdout() fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took) w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL") for _, r := range v.Rails { link := "-" if r.RateGbps > 0 { link = fmt.Sprintf("%d Gb/s", r.RateGbps) } fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail) } _ = w.Flush() for _, n := range v.Notes { fmt.Fprintf(out, "\nnote: %s\n", n) } if !v.OK { return fmt.Errorf("not every rail passed") } return nil }, } c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)") c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30") return c }