Measure two nodes' fabric rail by rail: llmgw ctl fabric-test
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
af6f8e023e
commit
7bb360fc78
21 files changed
+1265
-62
No files matched your search
@@ -844,6 +844,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
reg.EngineVersions = a.engines(parent)
|
||||
reg.EnginesProbed = true
|
||||
reg.InstallableEngines = provision.Installable()
|
||||
reg.Features = []string{FeatureFabricTest}
|
||||
reg.RunningShards = a.runningShards()
|
||||
a.mu.Lock()
|
||||
h := a.prov.Host
|
||||
@@ -1145,6 +1146,16 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(m.FabricScan)
|
||||
case *llmgwv1.ControlMessage_FabricTest:
|
||||
// Seconds of traffic, and a package install the first time: off the
|
||||
// receive loop too.
|
||||
go func(req *llmgwv1.FabricTestRequest) {
|
||||
reply := a.fabricTest(ctx, req)
|
||||
select {
|
||||
case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_FabricTest{FabricTest: reply}}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(m.FabricTest)
|
||||
case *llmgwv1.ControlMessage_Ack:
|
||||
if !m.Ack.GetOk() {
|
||||
a.log.Warn("control plane rejected a message",
|
||||
@@ -1758,6 +1769,70 @@ func (a *Agent) ensureDiags(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// FeatureFabricTest is the agent answering FabricTestRequest.
|
||||
const FeatureFabricTest = "fabric-test"
|
||||
|
||||
// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
|
||||
// installed; both replaced in tests.
|
||||
var (
|
||||
runWriteBW = discovery.RunWriteBW
|
||||
haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
|
||||
)
|
||||
|
||||
// fabricTest runs this node's side of a bandwidth test with another.
|
||||
func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
|
||||
start := time.Now()
|
||||
reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
|
||||
defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
|
||||
|
||||
w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
|
||||
TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
|
||||
if req.GetRole() == "client" {
|
||||
w.Peer = req.GetPeer()
|
||||
}
|
||||
a.mu.Lock()
|
||||
found := false
|
||||
for _, h := range a.inv.HCAs {
|
||||
for _, p := range h.Ports {
|
||||
if h.Name == w.HCA && p.Num == w.Port {
|
||||
found = true
|
||||
// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
|
||||
if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
|
||||
w.GIDIndex = p.RoCEv2GID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if !found {
|
||||
reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
|
||||
return reply
|
||||
}
|
||||
if !haveWriteBW() && a.prov != nil {
|
||||
a.log.Info("installing perftest, for a bandwidth test with another node")
|
||||
pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
|
||||
if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
|
||||
reply.Error = "installing perftest: " + err.Error()
|
||||
return reply
|
||||
}
|
||||
}
|
||||
gbps, out, err := runWriteBW(ctx, w)
|
||||
reply.Gbps = gbps
|
||||
reply.Output = tailLines(out, 12)
|
||||
if err != nil {
|
||||
reply.Error = err.Error()
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
func tailLines(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// discoverGPUFabric and discoverIMEX are replaced in tests.
|
||||
var (
|
||||
discoverGPUFabric = discovery.DiscoverGPUFabric
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
@@ -881,3 +882,34 @@ func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
|
||||
t.Errorf("asked %q, got %+v", asked, r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABandwidthTestIsRunOnTheAdapterAskedFor(t *testing.T) {
|
||||
oldRun, oldHave := runWriteBW, haveWriteBW
|
||||
t.Cleanup(func() { runWriteBW, haveWriteBW = oldRun, oldHave })
|
||||
haveWriteBW = func() bool { return true }
|
||||
var got discovery.WriteBW
|
||||
runWriteBW = func(_ context.Context, w discovery.WriteBW) (float64, string, error) {
|
||||
got = w
|
||||
return 388.2, " #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec]\n", nil
|
||||
}
|
||||
h := start(t, func(c *Config) { c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") })
|
||||
|
||||
r, err := h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{
|
||||
Role: "client", Hca: "mlx5_0", Port: 1, Peer: "10.0.0.2", TcpPort: 18515, Seconds: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.GetGbps() != 388.2 || r.GetError() != "" {
|
||||
t.Errorf("reply %+v", r)
|
||||
}
|
||||
if got.HCA != "mlx5_0" || got.Peer != "10.0.0.2" || got.TCPPort != 18515 || got.GIDIndex != -1 {
|
||||
t.Errorf("ran %+v", got)
|
||||
}
|
||||
|
||||
// An adapter the node does not have is said so, without running anything.
|
||||
got = discovery.WriteBW{}
|
||||
r, _ = h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{Role: "server", Hca: "mlx5_9", Port: 1, Seconds: 1})
|
||||
if !strings.Contains(r.GetError(), "no port mlx5_9/1") || got.HCA != "" {
|
||||
t.Errorf("reply %+v, ran %+v", r, got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// WriteBW is one side of an ib_write_bw run, from perftest: an RDMA write
|
||||
// bandwidth test between two adapters, which is what a collective spanning
|
||||
// nodes rides on.
|
||||
type WriteBW struct {
|
||||
HCA string
|
||||
Port int
|
||||
// GIDIndex is the RoCE v2 GID to use; negative on InfiniBand, where the
|
||||
// port's LID is used.
|
||||
GIDIndex int
|
||||
// Peer is the server's address, for the client; empty makes the server.
|
||||
Peer string
|
||||
TCPPort int
|
||||
Seconds int
|
||||
}
|
||||
|
||||
// Args are the command line: a duration rather than a count, so the test
|
||||
// takes as long on a slow link as a fast one, and results in Gb/s.
|
||||
func (w WriteBW) Args() []string {
|
||||
args := []string{"-d", w.HCA, "-i", strconv.Itoa(max(w.Port, 1)),
|
||||
"-p", strconv.Itoa(w.TCPPort), "-D", strconv.Itoa(max(w.Seconds, 1)),
|
||||
"-F", "--report_gbits"}
|
||||
if w.GIDIndex >= 0 {
|
||||
args = append(args, "-x", strconv.Itoa(w.GIDIndex))
|
||||
}
|
||||
if w.Peer != "" {
|
||||
args = append(args, w.Peer)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// RunWriteBW runs one side and returns the average bandwidth the output
|
||||
// reports, in Gb/s, and the output. The client retries its connection for a
|
||||
// while: the server is started first, but a few seconds may pass before it
|
||||
// listens.
|
||||
func RunWriteBW(ctx context.Context, w WriteBW) (float64, string, error) {
|
||||
bin, err := exec.LookPath("ib_write_bw")
|
||||
if err != nil {
|
||||
return 0, "", errors.New("ib_write_bw is not installed (the perftest package)")
|
||||
}
|
||||
limit := time.Duration(w.Seconds+60) * time.Second
|
||||
var out []byte
|
||||
for attempt := 0; ; attempt++ {
|
||||
cctx, cancel := context.WithTimeout(ctx, limit)
|
||||
out, err = exec.CommandContext(cctx, bin, w.Args()...).CombinedOutput() //nolint:gosec // fixed tool
|
||||
cancel()
|
||||
if err == nil || w.Peer == "" || !refused(string(out)) || attempt >= 10 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return 0, string(out), ctx.Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
text := string(out)
|
||||
if err != nil {
|
||||
return 0, text, fmt.Errorf("ib_write_bw: %v: %s", err, lastNonEmpty(text))
|
||||
}
|
||||
gbps, perr := ParseWriteBW(text)
|
||||
if perr != nil && w.Peer != "" {
|
||||
// Only the client is sure to print the result; a server that does not
|
||||
// is not a failure.
|
||||
return 0, text, perr
|
||||
}
|
||||
return gbps, text, nil
|
||||
}
|
||||
|
||||
func refused(out string) bool {
|
||||
for _, s := range []string{"Couldn't connect", "Unable to init the socket connection", "Connection refused"} {
|
||||
if strings.Contains(out, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var columnSplit = regexp.MustCompile(`\s{2,}`)
|
||||
|
||||
// ParseWriteBW finds the average bandwidth in perftest's result table:
|
||||
//
|
||||
// #bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
|
||||
// 65536 887036 0.00 96.93 0.184877
|
||||
//
|
||||
// by the heading's position rather than a fixed column, since versions add
|
||||
// columns and the peak reads 0.00 when the test runs for a duration.
|
||||
func ParseWriteBW(out string) (float64, error) {
|
||||
lines := strings.Split(out, "\n")
|
||||
for i, l := range lines {
|
||||
if !strings.Contains(l, "BW average") {
|
||||
continue
|
||||
}
|
||||
heads := columnSplit.Split(strings.TrimSpace(l), -1)
|
||||
col := -1
|
||||
for j, h := range heads {
|
||||
if strings.HasPrefix(h, "BW average") {
|
||||
col = j
|
||||
}
|
||||
}
|
||||
for _, d := range lines[i+1:] {
|
||||
f := strings.Fields(d)
|
||||
if len(f) == 0 || strings.HasPrefix(f[0], "-") {
|
||||
continue
|
||||
}
|
||||
if len(f) != len(heads) || col < 0 {
|
||||
return 0, fmt.Errorf("perftest's result table has a shape this does not know: %q / %q", l, d)
|
||||
}
|
||||
v, err := strconv.ParseFloat(f[col], 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("reading the average bandwidth %q: %w", f[col], err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("perftest printed no result")
|
||||
}
|
||||
|
||||
func lastNonEmpty(s string) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
return strings.TrimSpace(lines[len(lines)-1])
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The client's output of perftest 24.x, with a duration.
|
||||
const writeBWClient = `---------------------------------------------------------------------------------------
|
||||
RDMA_Write BW Test
|
||||
Dual-port : OFF Device : mlx5_0
|
||||
Number of qps : 1 Transport type : IB
|
||||
Connection type : RC Using SRQ : OFF
|
||||
TX depth : 128
|
||||
CQ Moderation : 100
|
||||
Mtu : 4096[B]
|
||||
Link type : IB
|
||||
Max inline data : 0[B]
|
||||
rdma_cm QPs : OFF
|
||||
Data ex. method : Ethernet
|
||||
---------------------------------------------------------------------------------------
|
||||
local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
|
||||
remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
|
||||
---------------------------------------------------------------------------------------
|
||||
#bytes #iterations BW peak[Gb/sec] BW average[Gb/sec] MsgRate[Mpps]
|
||||
65536 1851214 0.00 388.21 0.740422
|
||||
---------------------------------------------------------------------------------------
|
||||
`
|
||||
|
||||
func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
|
||||
got, err := ParseWriteBW(writeBWClient)
|
||||
if err != nil || got != 388.21 {
|
||||
t.Errorf("got %v, %v", got, err)
|
||||
}
|
||||
if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
|
||||
t.Error("output with no result table gave a number")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteBWArguments(t *testing.T) {
|
||||
ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
|
||||
if slices.Contains(ib, "-x") {
|
||||
t.Errorf("InfiniBand was given a GID index: %v", ib)
|
||||
}
|
||||
roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
|
||||
if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
|
||||
t.Errorf("RoCE client: %v", roce)
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,7 @@ func newCtlCmd() *cobra.Command {
|
||||
newModelsCmd(client, &asJSON),
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
newFabricTestCmd(client, &asJSON),
|
||||
newNodeStateCmd(client, "cordon", "cordoned",
|
||||
"Stop placing new work on a node; what runs there keeps running"),
|
||||
newNodeStateCmd(client, "uncordon", "ready",
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var (
|
||||
fabric string
|
||||
seconds int
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "fabric-test NODE-A NODE-B",
|
||||
Short: "Measure RDMA bandwidth between two nodes, rail by rail",
|
||||
Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
|
||||
adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
|
||||
one rail at a time, and grades each against its link's rate: ok at 80% or
|
||||
more, slow at 50%, fail below or when nothing got through.
|
||||
|
||||
Every check on one node can pass while two nodes still cannot reach each
|
||||
other: a cable into the wrong switch, a partition key, a RoCE network with no
|
||||
route. A replica spanning them then hangs on its first all-reduce. Run this
|
||||
before placing one.
|
||||
|
||||
It goes through the agents, so it needs no ssh between nodes. perftest is
|
||||
installed on a node the first time it is asked. The traffic is real and
|
||||
takes a few seconds a rail; run it when the fabric is quiet.`,
|
||||
Example: ` llmgw ctl fabric-test gpu01 gpu02
|
||||
llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
|
||||
Args: cobra.ExactArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
|
||||
cl := client()
|
||||
// Rails one after another, each for its seconds and a settle; the
|
||||
// client's usual timeout is for questions, not measurements.
|
||||
cl.http.Timeout = 30 * time.Minute
|
||||
var v adminapi.FabricTestView
|
||||
if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
|
||||
for _, r := range v.Rails {
|
||||
link := "-"
|
||||
if r.RateGbps > 0 {
|
||||
link = fmt.Sprintf("%d Gb/s", r.RateGbps)
|
||||
}
|
||||
fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
|
||||
}
|
||||
_ = w.Flush()
|
||||
for _, n := range v.Notes {
|
||||
fmt.Fprintf(out, "\nnote: %s\n", n)
|
||||
}
|
||||
if !v.OK {
|
||||
return fmt.Errorf("not every rail passed")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
|
||||
c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
|
||||
return c
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
)
|
||||
|
||||
func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
||||
@@ -23,10 +24,17 @@ func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
||||
t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
|
||||
}
|
||||
|
||||
if _, ok := ref.(fabrictest.Asker); !ok {
|
||||
t.Error("dispatcherRef drops the bandwidth test, so ctl fabric-test cannot reach the agents")
|
||||
}
|
||||
|
||||
// And the thing it forwards to must have it, or the forwarding has nothing
|
||||
// to reach.
|
||||
var svc any = &agentsvc.Service{}
|
||||
if _, ok := svc.(control.WeightRemover); !ok {
|
||||
t.Error("the agent service does not implement WeightRemover")
|
||||
}
|
||||
if _, ok := svc.(fabrictest.Asker); !ok {
|
||||
t.Error("the agent service does not implement the bandwidth test")
|
||||
}
|
||||
}
|
||||
@@ -21,11 +21,13 @@ import (
|
||||
"golang.org/x/net/http2/h2c"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -493,6 +495,18 @@ func (r *dispatcherRef) Logs(ctx context.Context, nodeID, source string, lines i
|
||||
return lr.Logs(ctx, nodeID, source, lines)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ft, ok := d.(fabrictest.Asker)
|
||||
if !ok {
|
||||
return nil, errors.New("this dispatcher cannot run a bandwidth test")
|
||||
}
|
||||
return ft.FabricTest(ctx, nodeID, req)
|
||||
}
|
||||
|
||||
func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
d, err := r.get()
|
||||
if err != nil {
|
||||
|
||||
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
)
|
||||
|
||||
// FabricTestView is a bandwidth test between two nodes.
|
||||
type FabricTestView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Fabric string `json:"fabric"`
|
||||
OK bool `json:"ok"`
|
||||
Rails []RailView `json:"rails"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
Took string `json:"took"`
|
||||
}
|
||||
|
||||
// RailView is one rail's result.
|
||||
type RailView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
RateGbps int `json:"rate_gbps"`
|
||||
Gbps float64 `json:"gbps"`
|
||||
Verdict string `json:"verdict"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
|
||||
// sends traffic over the fabric for seconds a rail, so it is a POST and only
|
||||
// ever run when asked.
|
||||
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
A, B, Fabric string
|
||||
Seconds int
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
ask := a.ctl.FabricTester()
|
||||
if ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
|
||||
return
|
||||
}
|
||||
inv := a.ctl.Inventory()
|
||||
na, okA := inv.Get(body.A)
|
||||
nb, okB := inv.Get(body.B)
|
||||
if !okA || !okB {
|
||||
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
|
||||
return
|
||||
}
|
||||
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
|
||||
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
|
||||
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
|
||||
Took: time.Since(start).Round(time.Second).String()}
|
||||
for _, x := range rep.Results {
|
||||
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
Labels: r.GetLabels(),
|
||||
State: inventory.Ready,
|
||||
AgentVersion: r.GetAgentVersion(),
|
||||
Features: r.GetFeatures(),
|
||||
EngineVersions: r.GetEngineVersions(),
|
||||
EnginesProbed: r.GetEnginesProbed(),
|
||||
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
@@ -61,6 +62,7 @@ type Service struct {
|
||||
logSeq uint64
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||
}
|
||||
|
||||
// SetEngines sets the engines every node installs, by name and version.
|
||||
@@ -289,6 +291,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
"last_output", m.Shard.GetLastOutput())
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricTest:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.testWaits[m.FabricTest.GetRequestId()]
|
||||
delete(s.testWaits, m.FabricTest.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.FabricTest
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricScan:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
|
||||
@@ -717,6 +728,38 @@ func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int)
|
||||
}
|
||||
}
|
||||
|
||||
// FabricTest asks a node for one side of a bandwidth test, and waits for
|
||||
// what it measured. The request's id is filled in here.
|
||||
func (s *Service) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
s.logMu.Lock()
|
||||
if s.testWaits == nil {
|
||||
s.testWaits = map[string]chan *llmgwv1.FabricTestReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("test-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.FabricTestReply, 1)
|
||||
s.testWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.testWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
req.RequestId = id
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricTest{FabricTest: req}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The test's own length, and a package install the first time.
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(req.GetSeconds())*time.Second+5*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r, nil
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("node %s did not answer the bandwidth test", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
|
||||
// one node on each: the first, by name, whose port on it is up. A fabric
|
||||
// looks the same from all its ports, so one is enough, and the rest of its
|
||||
@@ -795,3 +838,7 @@ func installable(reg *llmgwv1.Register) []string {
|
||||
}
|
||||
return []string{"llamacpp"}
|
||||
}
|
||||
|
||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||
// drifted would make it quietly nil.
|
||||
var _ fabrictest.Asker = (*Service)(nil)
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
@@ -215,6 +216,12 @@ func (c *Controller) LogReader() LogReader {
|
||||
return r
|
||||
}
|
||||
|
||||
// FabricTester returns the dispatcher's bandwidth test capability, or nil.
|
||||
func (c *Controller) FabricTester() fabrictest.Asker {
|
||||
r, _ := c.disp.(fabrictest.Asker)
|
||||
return r
|
||||
}
|
||||
|
||||
// Start runs the reconciler and begins publishing endpoints.
|
||||
func (c *Controller) Start(ctx context.Context) {
|
||||
if c.lead != nil {
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Package fabrictest measures the fabric between two nodes, rail by rail.
|
||||
//
|
||||
// Every check on one node — ports up, a subnet manager, the right modules —
|
||||
// can pass while the two nodes still cannot reach each other: a cable into the
|
||||
// wrong switch, a partition key, a RoCE network with no route. A replica
|
||||
// spanning them then starts, and hangs on its first all-reduce with nothing to
|
||||
// say why. This runs what that all-reduce would ride on — an RDMA write from
|
||||
// each adapter to its counterpart — and says how it went against the link's
|
||||
// rate, before a replica does.
|
||||
package fabrictest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Asker sends one side of a test to a node and waits for its answer.
|
||||
type Asker interface {
|
||||
FabricTest(ctx context.Context, node string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error)
|
||||
}
|
||||
|
||||
// Rail is one adapter on each node, paired by their position in the fabric's
|
||||
// declaration: mlx5_0 with mlx5_0, the way NCCL pairs them.
|
||||
type Rail struct {
|
||||
A, B string // adapters
|
||||
PortA, PortB int
|
||||
// RateGbps is the slower of the two links.
|
||||
RateGbps int
|
||||
}
|
||||
|
||||
// Result is one rail's test.
|
||||
type Result struct {
|
||||
Rail
|
||||
Gbps float64 `json:"gbps"`
|
||||
// Verdict is "ok" at 80% of the link's rate or more, "slow" at 50%,
|
||||
// "fail" below that or when it did not run.
|
||||
Verdict string `json:"verdict"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Report is a whole test.
|
||||
type Report struct {
|
||||
Fabric string `json:"fabric"`
|
||||
A, B string `json:"-"`
|
||||
Results []Result `json:"results"`
|
||||
// Notes say what the test could not cover.
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// OK reports whether every rail passed.
|
||||
func (r Report) OK() bool {
|
||||
for _, x := range r.Results {
|
||||
if x.Verdict != "ok" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(r.Results) > 0
|
||||
}
|
||||
|
||||
// Plan pairs the adapters two nodes have on a fabric. With no fabric named,
|
||||
// the one fabric they share is used.
|
||||
func Plan(a, b inventory.Node, fabric string) (string, []Rail, []string, error) {
|
||||
if a.ID == b.ID {
|
||||
return "", nil, nil, errors.New("give two different nodes")
|
||||
}
|
||||
if fabric == "" {
|
||||
var common []string
|
||||
for _, f := range a.FabricNames() {
|
||||
if b.HasFabric(f) {
|
||||
common = append(common, f)
|
||||
}
|
||||
}
|
||||
switch len(common) {
|
||||
case 0:
|
||||
return "", nil, nil, fmt.Errorf("%s (%s) and %s (%s) declare no fabric in common",
|
||||
a.ID, orNone(a.FabricNames()), b.ID, orNone(b.FabricNames()))
|
||||
case 1:
|
||||
fabric = common[0]
|
||||
default:
|
||||
return "", nil, nil, fmt.Errorf("%s and %s share %s: name one with --fabric", a.ID, b.ID, strings.Join(common, ", "))
|
||||
}
|
||||
}
|
||||
for _, n := range []inventory.Node{a, b} {
|
||||
if !slices.Contains(n.Features, "fabric-test") {
|
||||
return "", nil, nil, fmt.Errorf("%s's agent (%s) is too old to run a bandwidth test: upgrade it", n.ID, orNone([]string{n.AgentVersion}))
|
||||
}
|
||||
}
|
||||
ha, hb := a.HCAsFor(fabric), b.HCAsFor(fabric)
|
||||
if len(ha) == 0 || len(hb) == 0 {
|
||||
return "", nil, nil, fmt.Errorf("%s is not declared on both %s and %s", fabric, a.ID, b.ID)
|
||||
}
|
||||
var notes []string
|
||||
if len(ha) != len(hb) {
|
||||
notes = append(notes, fmt.Sprintf("%s has %d adapters on %s and %s has %d: only the first %d are paired",
|
||||
a.ID, len(ha), fabric, b.ID, len(hb), min(len(ha), len(hb))))
|
||||
}
|
||||
var rails []Rail
|
||||
for i := range min(len(ha), len(hb)) {
|
||||
pa, ra := port(a, ha[i])
|
||||
pb, rb := port(b, hb[i])
|
||||
rails = append(rails, Rail{A: ha[i], B: hb[i], PortA: pa, PortB: pb, RateGbps: minPositive(ra, rb)})
|
||||
}
|
||||
return fabric, rails, notes, nil
|
||||
}
|
||||
|
||||
// port is an adapter's first active port and its rate, or port 1 when the
|
||||
// node reported none.
|
||||
func port(n inventory.Node, hca string) (int, int) {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.HCA == hca && strings.EqualFold(p.State, "ACTIVE") {
|
||||
return p.Port, p.RateGbps
|
||||
}
|
||||
}
|
||||
return 1, 0
|
||||
}
|
||||
|
||||
// Options for a run.
|
||||
type Options struct {
|
||||
Seconds int
|
||||
// BasePort is perftest's TCP exchange port for the first rail; each rail
|
||||
// takes the next, so a rail whose server is still closing never meets the
|
||||
// next one's.
|
||||
BasePort int
|
||||
// Settle is how long the server is given to listen before the client
|
||||
// dials; the client retries besides.
|
||||
Settle time.Duration
|
||||
}
|
||||
|
||||
// Run tests every rail, one after another: rails at once would share the
|
||||
// PCIe and memory bandwidth they are measuring.
|
||||
func Run(ctx context.Context, ask Asker, a, b inventory.Node, fabric string, rails []Rail, o Options) Report {
|
||||
if o.Seconds <= 0 {
|
||||
o.Seconds = 3
|
||||
}
|
||||
if o.BasePort == 0 {
|
||||
o.BasePort = 18515
|
||||
}
|
||||
rep := Report{Fabric: fabric, A: a.ID, B: b.ID}
|
||||
for i, r := range rails {
|
||||
rep.Results = append(rep.Results, runRail(ctx, ask, a, b, r, o, o.BasePort+i))
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
func runRail(ctx context.Context, ask Asker, a, b inventory.Node, r Rail, o Options, tcp int) Result {
|
||||
res := Result{Rail: r, Verdict: "fail"}
|
||||
if b.MgmtAddr == "" {
|
||||
res.Detail = b.ID + " reported no management address for the client to dial"
|
||||
return res
|
||||
}
|
||||
type answer struct {
|
||||
r *llmgwv1.FabricTestReply
|
||||
err error
|
||||
}
|
||||
server := make(chan answer, 1)
|
||||
go func() {
|
||||
rep, err := ask.FabricTest(ctx, b.ID, &llmgwv1.FabricTestRequest{Role: "server", Hca: r.B,
|
||||
Port: int32(r.PortB), TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
|
||||
server <- answer{rep, err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
res.Detail = ctx.Err().Error()
|
||||
return res
|
||||
case <-time.After(o.Settle):
|
||||
}
|
||||
client, err := ask.FabricTest(ctx, a.ID, &llmgwv1.FabricTestRequest{Role: "client", Hca: r.A,
|
||||
Port: int32(r.PortA), Peer: b.MgmtAddr, TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
|
||||
var srv answer
|
||||
select {
|
||||
case srv = <-server:
|
||||
case <-time.After(time.Duration(o.Seconds)*time.Second + time.Minute):
|
||||
srv.err = errors.New("the server side did not finish")
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
res.Detail = a.ID + ": " + err.Error()
|
||||
case client.GetError() != "":
|
||||
res.Detail = a.ID + ": " + client.GetError()
|
||||
// The server's reason is often the better one: it saw no one arrive.
|
||||
if srv.r != nil && srv.r.GetError() != "" {
|
||||
res.Detail += "; " + b.ID + ": " + srv.r.GetError()
|
||||
}
|
||||
default:
|
||||
res.Gbps = client.GetGbps()
|
||||
res.Verdict, res.Detail = grade(res.Gbps, r.RateGbps)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// grade compares a measurement with the link's rate. RDMA writes of 64 KiB
|
||||
// reach 90–97% of the signalling rate on a healthy link; well under that is
|
||||
// a degraded cable, a card in a narrow PCIe slot, or traffic crossing a
|
||||
// socket.
|
||||
func grade(gbps float64, rate int) (string, string) {
|
||||
if gbps <= 0 {
|
||||
return "fail", "no traffic got through"
|
||||
}
|
||||
if rate <= 0 {
|
||||
return "ok", "the link's rate is not known to compare with"
|
||||
}
|
||||
pct := gbps / float64(rate) * 100
|
||||
switch {
|
||||
case pct >= 80:
|
||||
return "ok", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
|
||||
case pct >= 50:
|
||||
return "slow", fmt.Sprintf("%.0f%% of %d Gb/s: check the cable, and the card's PCIe width and NUMA node", pct, rate)
|
||||
default:
|
||||
return "fail", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
|
||||
}
|
||||
}
|
||||
|
||||
func minPositive(a, b int) int {
|
||||
switch {
|
||||
case a <= 0:
|
||||
return b
|
||||
case b <= 0:
|
||||
return a
|
||||
}
|
||||
return min(a, b)
|
||||
}
|
||||
|
||||
func orNone(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(slices.Clone(s), ", ")
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package fabrictest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func node(id string, hcas ...string) inventory.Node {
|
||||
n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"},
|
||||
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}}
|
||||
for _, h := range hcas {
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// fakeFabric answers as two agents would: the server waits for its client.
|
||||
type fakeFabric struct {
|
||||
mu sync.Mutex
|
||||
gbps map[string]float64 // by client adapter
|
||||
fails map[string]string
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca())
|
||||
if req.GetRole() == "server" {
|
||||
return &llmgwv1.FabricTestReply{}, nil
|
||||
}
|
||||
if e := f.fails[req.GetHca()]; e != "" {
|
||||
return &llmgwv1.FabricTestReply{Error: e}, nil
|
||||
}
|
||||
return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil
|
||||
}
|
||||
|
||||
func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) {
|
||||
a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
||||
fabric, rails, notes, err := Plan(a, b, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 {
|
||||
t.Errorf("got %s %+v", fabric, rails)
|
||||
}
|
||||
if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") {
|
||||
t.Errorf("an unequal count went unmentioned: %v", notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) {
|
||||
b := node("n2", "mlx5_0")
|
||||
b.Fabrics[0].Fabric = "ib-b"
|
||||
if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachRailIsGradedAgainstItsLink(t *testing.T) {
|
||||
a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
||||
_, rails, _, _ := Plan(a, b, "ib-a")
|
||||
f := &fakeFabric{
|
||||
gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240},
|
||||
fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"},
|
||||
}
|
||||
rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond})
|
||||
want := []string{"ok", "slow", "fail"}
|
||||
for i, r := range rep.Results {
|
||||
if r.Verdict != want[i] {
|
||||
t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i])
|
||||
}
|
||||
}
|
||||
if rep.OK() {
|
||||
t.Error("a report with a failed rail is OK")
|
||||
}
|
||||
if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") {
|
||||
t.Errorf("the reason was lost: %q", rep.Results[2].Detail)
|
||||
}
|
||||
// Servers on the second node, clients on the first, one rail at a time.
|
||||
if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" {
|
||||
t.Errorf("asked %v", f.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) {
|
||||
// It would ignore the request, and the control plane would wait minutes.
|
||||
old := node("n2", "mlx5_0")
|
||||
old.Features, old.AgentVersion = nil, "v0.10.0"
|
||||
_, _, _, err := Plan(node("n1", "mlx5_0"), old, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -100,7 +100,10 @@ type Node struct {
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
// AgentVersion and EngineVersions are reported at registration. The engine
|
||||
// version decides which metric names and CLI flags apply (plan §14).
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
// Features are the requests the agent answers beyond the ones every
|
||||
// agent does.
|
||||
Features []string `json:"features,omitempty"`
|
||||
EngineVersions map[string]string `json:"engine_versions,omitempty"`
|
||||
// EnginesProbed says EngineVersions is an answer, so an empty map means
|
||||
// "has none" rather than "an agent too old to say".
|
||||
|
||||
Reference in new issue
Block a user