Measure two nodes' fabric rail by rail: llmgw ctl fabric-test
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
af6f8e023e
commit
7bb360fc78
21 files changed
+1265
-62
No files matched your search
@@ -75,6 +75,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/explain-placement", a.explainPlacement)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
)
|
||||
|
||||
// FabricTestView is a bandwidth test between two nodes.
|
||||
type FabricTestView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
Fabric string `json:"fabric"`
|
||||
OK bool `json:"ok"`
|
||||
Rails []RailView `json:"rails"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
Took string `json:"took"`
|
||||
}
|
||||
|
||||
// RailView is one rail's result.
|
||||
type RailView struct {
|
||||
A string `json:"a"`
|
||||
B string `json:"b"`
|
||||
RateGbps int `json:"rate_gbps"`
|
||||
Gbps float64 `json:"gbps"`
|
||||
Verdict string `json:"verdict"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// fabricTest measures RDMA bandwidth between two nodes, rail by rail. It
|
||||
// sends traffic over the fabric for seconds a rail, so it is a POST and only
|
||||
// ever run when asked.
|
||||
func (a *API) fabricTest(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
A, B, Fabric string
|
||||
Seconds int
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
ask := a.ctl.FabricTester()
|
||||
if ask == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane cannot reach its nodes' agents")
|
||||
return
|
||||
}
|
||||
inv := a.ctl.Inventory()
|
||||
na, okA := inv.Get(body.A)
|
||||
nb, okB := inv.Get(body.B)
|
||||
if !okA || !okB {
|
||||
writeError(w, http.StatusNotFound, fmt.Sprintf("no node %q", map[bool]string{true: body.B, false: body.A}[okA]))
|
||||
return
|
||||
}
|
||||
fabric, rails, notes, err := fabrictest.Plan(na, nb, body.Fabric)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
rep := fabrictest.Run(r.Context(), ask, na, nb, fabric, rails,
|
||||
fabrictest.Options{Seconds: min(max(body.Seconds, 1), 30), Settle: 2 * time.Second})
|
||||
v := FabricTestView{A: na.ID, B: nb.ID, Fabric: fabric, OK: rep.OK(), Notes: append(notes, rep.Notes...),
|
||||
Took: time.Since(start).Round(time.Second).String()}
|
||||
for _, x := range rep.Results {
|
||||
v.Rails = append(v.Rails, RailView{x.A, x.B, x.RateGbps, x.Gbps, x.Verdict, x.Detail})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -23,6 +23,7 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
|
||||
Labels: r.GetLabels(),
|
||||
State: inventory.Ready,
|
||||
AgentVersion: r.GetAgentVersion(),
|
||||
Features: r.GetFeatures(),
|
||||
EngineVersions: r.GetEngineVersions(),
|
||||
EnginesProbed: r.GetEnginesProbed(),
|
||||
MemoryTotalMB: int(r.GetMemoryTotalMb()),
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
@@ -61,6 +62,7 @@ type Service struct {
|
||||
logSeq uint64
|
||||
// scanWaits are the fabric scans awaiting a reply, by request id.
|
||||
scanWaits map[string]chan *llmgwv1.FabricScanReply
|
||||
testWaits map[string]chan *llmgwv1.FabricTestReply
|
||||
}
|
||||
|
||||
// SetEngines sets the engines every node installs, by name and version.
|
||||
@@ -289,6 +291,15 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) {
|
||||
"last_output", m.Shard.GetLastOutput())
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricTest:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.testWaits[m.FabricTest.GetRequestId()]
|
||||
delete(s.testWaits, m.FabricTest.GetRequestId())
|
||||
s.logMu.Unlock()
|
||||
if ok {
|
||||
ch <- m.FabricTest
|
||||
}
|
||||
|
||||
case *llmgwv1.AgentMessage_FabricScan:
|
||||
s.logMu.Lock()
|
||||
ch, ok := s.scanWaits[m.FabricScan.GetRequestId()]
|
||||
@@ -717,6 +728,38 @@ func (s *Service) ScanFabric(ctx context.Context, nodeID, hca string, port int)
|
||||
}
|
||||
}
|
||||
|
||||
// FabricTest asks a node for one side of a bandwidth test, and waits for
|
||||
// what it measured. The request's id is filled in here.
|
||||
func (s *Service) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
s.logMu.Lock()
|
||||
if s.testWaits == nil {
|
||||
s.testWaits = map[string]chan *llmgwv1.FabricTestReply{}
|
||||
}
|
||||
s.logSeq++
|
||||
id := fmt.Sprintf("test-%d", s.logSeq)
|
||||
ch := make(chan *llmgwv1.FabricTestReply, 1)
|
||||
s.testWaits[id] = ch
|
||||
s.logMu.Unlock()
|
||||
defer func() {
|
||||
s.logMu.Lock()
|
||||
delete(s.testWaits, id)
|
||||
s.logMu.Unlock()
|
||||
}()
|
||||
req.RequestId = id
|
||||
if err := s.send(nodeID, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_FabricTest{FabricTest: req}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The test's own length, and a package install the first time.
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Duration(req.GetSeconds())*time.Second+5*time.Minute)
|
||||
defer cancel()
|
||||
select {
|
||||
case r := <-ch:
|
||||
return r, nil
|
||||
case <-ctx.Done():
|
||||
return nil, fmt.Errorf("node %s did not answer the bandwidth test", nodeID)
|
||||
}
|
||||
}
|
||||
|
||||
// ScanFabrics maps every physical InfiniBand fabric every so often, asking
|
||||
// one node on each: the first, by name, whose port on it is up. A fabric
|
||||
// looks the same from all its ports, so one is enough, and the rest of its
|
||||
@@ -795,3 +838,7 @@ func installable(reg *llmgwv1.Register) []string {
|
||||
}
|
||||
return []string{"llamacpp"}
|
||||
}
|
||||
|
||||
// The admin API finds the bandwidth test by asserting this; a signature that
|
||||
// drifted would make it quietly nil.
|
||||
var _ fabrictest.Asker = (*Service)(nil)
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
@@ -215,6 +216,12 @@ func (c *Controller) LogReader() LogReader {
|
||||
return r
|
||||
}
|
||||
|
||||
// FabricTester returns the dispatcher's bandwidth test capability, or nil.
|
||||
func (c *Controller) FabricTester() fabrictest.Asker {
|
||||
r, _ := c.disp.(fabrictest.Asker)
|
||||
return r
|
||||
}
|
||||
|
||||
// Start runs the reconciler and begins publishing endpoints.
|
||||
func (c *Controller) Start(ctx context.Context) {
|
||||
if c.lead != nil {
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Package fabrictest measures the fabric between two nodes, rail by rail.
|
||||
//
|
||||
// Every check on one node — ports up, a subnet manager, the right modules —
|
||||
// can pass while the two nodes still cannot reach each other: a cable into the
|
||||
// wrong switch, a partition key, a RoCE network with no route. A replica
|
||||
// spanning them then starts, and hangs on its first all-reduce with nothing to
|
||||
// say why. This runs what that all-reduce would ride on — an RDMA write from
|
||||
// each adapter to its counterpart — and says how it went against the link's
|
||||
// rate, before a replica does.
|
||||
package fabrictest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// Asker sends one side of a test to a node and waits for its answer.
|
||||
type Asker interface {
|
||||
FabricTest(ctx context.Context, node string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error)
|
||||
}
|
||||
|
||||
// Rail is one adapter on each node, paired by their position in the fabric's
|
||||
// declaration: mlx5_0 with mlx5_0, the way NCCL pairs them.
|
||||
type Rail struct {
|
||||
A, B string // adapters
|
||||
PortA, PortB int
|
||||
// RateGbps is the slower of the two links.
|
||||
RateGbps int
|
||||
}
|
||||
|
||||
// Result is one rail's test.
|
||||
type Result struct {
|
||||
Rail
|
||||
Gbps float64 `json:"gbps"`
|
||||
// Verdict is "ok" at 80% of the link's rate or more, "slow" at 50%,
|
||||
// "fail" below that or when it did not run.
|
||||
Verdict string `json:"verdict"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Report is a whole test.
|
||||
type Report struct {
|
||||
Fabric string `json:"fabric"`
|
||||
A, B string `json:"-"`
|
||||
Results []Result `json:"results"`
|
||||
// Notes say what the test could not cover.
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// OK reports whether every rail passed.
|
||||
func (r Report) OK() bool {
|
||||
for _, x := range r.Results {
|
||||
if x.Verdict != "ok" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(r.Results) > 0
|
||||
}
|
||||
|
||||
// Plan pairs the adapters two nodes have on a fabric. With no fabric named,
|
||||
// the one fabric they share is used.
|
||||
func Plan(a, b inventory.Node, fabric string) (string, []Rail, []string, error) {
|
||||
if a.ID == b.ID {
|
||||
return "", nil, nil, errors.New("give two different nodes")
|
||||
}
|
||||
if fabric == "" {
|
||||
var common []string
|
||||
for _, f := range a.FabricNames() {
|
||||
if b.HasFabric(f) {
|
||||
common = append(common, f)
|
||||
}
|
||||
}
|
||||
switch len(common) {
|
||||
case 0:
|
||||
return "", nil, nil, fmt.Errorf("%s (%s) and %s (%s) declare no fabric in common",
|
||||
a.ID, orNone(a.FabricNames()), b.ID, orNone(b.FabricNames()))
|
||||
case 1:
|
||||
fabric = common[0]
|
||||
default:
|
||||
return "", nil, nil, fmt.Errorf("%s and %s share %s: name one with --fabric", a.ID, b.ID, strings.Join(common, ", "))
|
||||
}
|
||||
}
|
||||
for _, n := range []inventory.Node{a, b} {
|
||||
if !slices.Contains(n.Features, "fabric-test") {
|
||||
return "", nil, nil, fmt.Errorf("%s's agent (%s) is too old to run a bandwidth test: upgrade it", n.ID, orNone([]string{n.AgentVersion}))
|
||||
}
|
||||
}
|
||||
ha, hb := a.HCAsFor(fabric), b.HCAsFor(fabric)
|
||||
if len(ha) == 0 || len(hb) == 0 {
|
||||
return "", nil, nil, fmt.Errorf("%s is not declared on both %s and %s", fabric, a.ID, b.ID)
|
||||
}
|
||||
var notes []string
|
||||
if len(ha) != len(hb) {
|
||||
notes = append(notes, fmt.Sprintf("%s has %d adapters on %s and %s has %d: only the first %d are paired",
|
||||
a.ID, len(ha), fabric, b.ID, len(hb), min(len(ha), len(hb))))
|
||||
}
|
||||
var rails []Rail
|
||||
for i := range min(len(ha), len(hb)) {
|
||||
pa, ra := port(a, ha[i])
|
||||
pb, rb := port(b, hb[i])
|
||||
rails = append(rails, Rail{A: ha[i], B: hb[i], PortA: pa, PortB: pb, RateGbps: minPositive(ra, rb)})
|
||||
}
|
||||
return fabric, rails, notes, nil
|
||||
}
|
||||
|
||||
// port is an adapter's first active port and its rate, or port 1 when the
|
||||
// node reported none.
|
||||
func port(n inventory.Node, hca string) (int, int) {
|
||||
for _, p := range n.IBPorts {
|
||||
if p.HCA == hca && strings.EqualFold(p.State, "ACTIVE") {
|
||||
return p.Port, p.RateGbps
|
||||
}
|
||||
}
|
||||
return 1, 0
|
||||
}
|
||||
|
||||
// Options for a run.
|
||||
type Options struct {
|
||||
Seconds int
|
||||
// BasePort is perftest's TCP exchange port for the first rail; each rail
|
||||
// takes the next, so a rail whose server is still closing never meets the
|
||||
// next one's.
|
||||
BasePort int
|
||||
// Settle is how long the server is given to listen before the client
|
||||
// dials; the client retries besides.
|
||||
Settle time.Duration
|
||||
}
|
||||
|
||||
// Run tests every rail, one after another: rails at once would share the
|
||||
// PCIe and memory bandwidth they are measuring.
|
||||
func Run(ctx context.Context, ask Asker, a, b inventory.Node, fabric string, rails []Rail, o Options) Report {
|
||||
if o.Seconds <= 0 {
|
||||
o.Seconds = 3
|
||||
}
|
||||
if o.BasePort == 0 {
|
||||
o.BasePort = 18515
|
||||
}
|
||||
rep := Report{Fabric: fabric, A: a.ID, B: b.ID}
|
||||
for i, r := range rails {
|
||||
rep.Results = append(rep.Results, runRail(ctx, ask, a, b, r, o, o.BasePort+i))
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
func runRail(ctx context.Context, ask Asker, a, b inventory.Node, r Rail, o Options, tcp int) Result {
|
||||
res := Result{Rail: r, Verdict: "fail"}
|
||||
if b.MgmtAddr == "" {
|
||||
res.Detail = b.ID + " reported no management address for the client to dial"
|
||||
return res
|
||||
}
|
||||
type answer struct {
|
||||
r *llmgwv1.FabricTestReply
|
||||
err error
|
||||
}
|
||||
server := make(chan answer, 1)
|
||||
go func() {
|
||||
rep, err := ask.FabricTest(ctx, b.ID, &llmgwv1.FabricTestRequest{Role: "server", Hca: r.B,
|
||||
Port: int32(r.PortB), TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
|
||||
server <- answer{rep, err}
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
res.Detail = ctx.Err().Error()
|
||||
return res
|
||||
case <-time.After(o.Settle):
|
||||
}
|
||||
client, err := ask.FabricTest(ctx, a.ID, &llmgwv1.FabricTestRequest{Role: "client", Hca: r.A,
|
||||
Port: int32(r.PortA), Peer: b.MgmtAddr, TcpPort: int32(tcp), Seconds: int32(o.Seconds)}) //nolint:gosec // small numbers
|
||||
var srv answer
|
||||
select {
|
||||
case srv = <-server:
|
||||
case <-time.After(time.Duration(o.Seconds)*time.Second + time.Minute):
|
||||
srv.err = errors.New("the server side did not finish")
|
||||
}
|
||||
switch {
|
||||
case err != nil:
|
||||
res.Detail = a.ID + ": " + err.Error()
|
||||
case client.GetError() != "":
|
||||
res.Detail = a.ID + ": " + client.GetError()
|
||||
// The server's reason is often the better one: it saw no one arrive.
|
||||
if srv.r != nil && srv.r.GetError() != "" {
|
||||
res.Detail += "; " + b.ID + ": " + srv.r.GetError()
|
||||
}
|
||||
default:
|
||||
res.Gbps = client.GetGbps()
|
||||
res.Verdict, res.Detail = grade(res.Gbps, r.RateGbps)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// grade compares a measurement with the link's rate. RDMA writes of 64 KiB
|
||||
// reach 90–97% of the signalling rate on a healthy link; well under that is
|
||||
// a degraded cable, a card in a narrow PCIe slot, or traffic crossing a
|
||||
// socket.
|
||||
func grade(gbps float64, rate int) (string, string) {
|
||||
if gbps <= 0 {
|
||||
return "fail", "no traffic got through"
|
||||
}
|
||||
if rate <= 0 {
|
||||
return "ok", "the link's rate is not known to compare with"
|
||||
}
|
||||
pct := gbps / float64(rate) * 100
|
||||
switch {
|
||||
case pct >= 80:
|
||||
return "ok", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
|
||||
case pct >= 50:
|
||||
return "slow", fmt.Sprintf("%.0f%% of %d Gb/s: check the cable, and the card's PCIe width and NUMA node", pct, rate)
|
||||
default:
|
||||
return "fail", fmt.Sprintf("%.0f%% of %d Gb/s", pct, rate)
|
||||
}
|
||||
}
|
||||
|
||||
func minPositive(a, b int) int {
|
||||
switch {
|
||||
case a <= 0:
|
||||
return b
|
||||
case b <= 0:
|
||||
return a
|
||||
}
|
||||
return min(a, b)
|
||||
}
|
||||
|
||||
func orNone(s []string) string {
|
||||
if len(s) == 0 {
|
||||
return "none"
|
||||
}
|
||||
return strings.Join(slices.Clone(s), ", ")
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package fabrictest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func node(id string, hcas ...string) inventory.Node {
|
||||
n := inventory.Node{ID: id, MgmtAddr: "10.0.0." + id[len(id)-1:], Features: []string{"fabric-test"},
|
||||
Fabrics: []inventory.FabricAttachment{{Fabric: "ib-a", HCAs: hcas}}}
|
||||
for _, h := range hcas {
|
||||
n.IBPorts = append(n.IBPorts, inventory.IBPort{HCA: h, Port: 1, State: "ACTIVE", RateGbps: 400})
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// fakeFabric answers as two agents would: the server waits for its client.
|
||||
type fakeFabric struct {
|
||||
mu sync.Mutex
|
||||
gbps map[string]float64 // by client adapter
|
||||
fails map[string]string
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeFabric) FabricTest(_ context.Context, n string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.asked = append(f.asked, n+" "+req.GetRole()+" "+req.GetHca())
|
||||
if req.GetRole() == "server" {
|
||||
return &llmgwv1.FabricTestReply{}, nil
|
||||
}
|
||||
if e := f.fails[req.GetHca()]; e != "" {
|
||||
return &llmgwv1.FabricTestReply{Error: e}, nil
|
||||
}
|
||||
return &llmgwv1.FabricTestReply{Gbps: f.gbps[req.GetHca()]}, nil
|
||||
}
|
||||
|
||||
func TestRailsArePairedByPositionOnTheSharedFabric(t *testing.T) {
|
||||
a, b := node("n1", "mlx5_0", "mlx5_1"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
||||
fabric, rails, notes, err := Plan(a, b, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fabric != "ib-a" || len(rails) != 2 || rails[1].A != "mlx5_1" || rails[1].B != "mlx5_1" || rails[0].RateGbps != 400 {
|
||||
t.Errorf("got %s %+v", fabric, rails)
|
||||
}
|
||||
if len(notes) != 1 || !strings.Contains(notes[0], "only the first 2") {
|
||||
t.Errorf("an unequal count went unmentioned: %v", notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesWithNoFabricInCommonAreSaidSo(t *testing.T) {
|
||||
b := node("n2", "mlx5_0")
|
||||
b.Fabrics[0].Fabric = "ib-b"
|
||||
if _, _, _, err := Plan(node("n1", "mlx5_0"), b, ""); err == nil || !strings.Contains(err.Error(), "no fabric in common") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachRailIsGradedAgainstItsLink(t *testing.T) {
|
||||
a, b := node("n1", "mlx5_0", "mlx5_1", "mlx5_2"), node("n2", "mlx5_0", "mlx5_1", "mlx5_2")
|
||||
_, rails, _, _ := Plan(a, b, "ib-a")
|
||||
f := &fakeFabric{
|
||||
gbps: map[string]float64{"mlx5_0": 388, "mlx5_1": 240},
|
||||
fails: map[string]string{"mlx5_2": "ib_write_bw: Couldn't connect to 10.0.0.2:18517"},
|
||||
}
|
||||
rep := Run(context.Background(), f, a, b, "ib-a", rails, Options{Seconds: 1, Settle: time.Millisecond})
|
||||
want := []string{"ok", "slow", "fail"}
|
||||
for i, r := range rep.Results {
|
||||
if r.Verdict != want[i] {
|
||||
t.Errorf("rail %d: %s (%s), want %s", i, r.Verdict, r.Detail, want[i])
|
||||
}
|
||||
}
|
||||
if rep.OK() {
|
||||
t.Error("a report with a failed rail is OK")
|
||||
}
|
||||
if !strings.Contains(rep.Results[2].Detail, "Couldn't connect") {
|
||||
t.Errorf("the reason was lost: %q", rep.Results[2].Detail)
|
||||
}
|
||||
// Servers on the second node, clients on the first, one rail at a time.
|
||||
if f.asked[0] != "n2 server mlx5_0" || f.asked[1] != "n1 client mlx5_0" {
|
||||
t.Errorf("asked %v", f.asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAgentTooOldToAnswerIsRefusedAtOnce(t *testing.T) {
|
||||
// It would ignore the request, and the control plane would wait minutes.
|
||||
old := node("n2", "mlx5_0")
|
||||
old.Features, old.AgentVersion = nil, "v0.10.0"
|
||||
_, _, _, err := Plan(node("n1", "mlx5_0"), old, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "n2's agent (v0.10.0) is too old") {
|
||||
t.Errorf("got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -100,7 +100,10 @@ type Node struct {
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
// AgentVersion and EngineVersions are reported at registration. The engine
|
||||
// version decides which metric names and CLI flags apply (plan §14).
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
// Features are the requests the agent answers beyond the ones every
|
||||
// agent does.
|
||||
Features []string `json:"features,omitempty"`
|
||||
EngineVersions map[string]string `json:"engine_versions,omitempty"`
|
||||
// EnginesProbed says EngineVersions is an answer, so an empty map means
|
||||
// "has none" rather than "an agent too old to say".
|
||||
|
||||
Reference in new issue
Block a user