diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 4d670bb..ced26b7 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -47,6 +47,31 @@ jobs: (cd dist && sha256sum "${name}" >> checksums.txt) done + # An NVML build for nodes with NVLink, which the portable binary + # cannot serve. + # + # NVML is the only source of NVLink topology, its bindings need cgo, + # and cgo is exactly what CGO_ENABLED=0 above exists to avoid. So the + # portable binary silently falls back to taking the lowest free GPU + # indices — right on an SXM box with NVSwitch, and wrong on a PCIe + # box where NVLink bridges pair specific cards: a tensor group + # straddling two bridges runs several times slower and nothing says + # so. Shipping only the portable build meant nobody could have that + # topology without building it themselves. + # + # Dynamically linked against the host's libc by necessity, so it is + # published beside the portable one rather than instead of it. + if command -v gcc >/dev/null; then + nvml_name="llmgw-${version}-linux-amd64-nvml" + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build \ + -tags nvml -trimpath \ + -ldflags "-s -w -X gitea.mynodes.xyz/ilker/llmgw/internal/version.Version=${version}" \ + -o "dist/${nvml_name}" ./cmd/llmgw + (cd dist && sha256sum "${nvml_name}" >> checksums.txt) + else + echo "no C toolchain; skipping the nvml build" >&2 + fi + # Proof the artifact runs at all before it is published. It cannot # test the cross-compiled ones, which is why the native build is # checked rather than assumed. diff --git a/docs/install.md b/docs/install.md index 843c1e4..a4b33ec 100644 --- a/docs/install.md +++ b/docs/install.md @@ -222,6 +222,35 @@ A node with no GPU at all is fine and is not an error. It reports zero cards and can host CPU profiles, which is a good way to try llmgw before committing hardware to it. +### NVLink, if your GPUs have it + +The published binary cannot see NVLink topology. NVML is the only source of it, +its bindings need cgo, and the portable binary is built without cgo on purpose +so it runs on a node whose libc is older than the builder's. + +Without that topology, a tensor-parallel group takes the lowest free GPU +indices. On an SXM board with NVSwitch that is correct, because every card is +equidistant. On a PCIe box where NVLink bridges pair specific cards, a group +straddling two bridges falls back to PCIe and runs several times slower — and +nothing reports it, because from the outside it is simply a replica that is +slow. + +For those nodes, install the `-nvml` build published beside the portable one: + +``` +sudo install -m0755 llmgw-$VERSION-linux-amd64-nvml /usr/local/bin/llmgw +``` + +It is dynamically linked, so it needs a libc no older than the builder's and the +NVIDIA driver present. Check it took effect with: + +``` +llmgw ctl nodes --json | grep -c nvlink_peers # 0 means the topology is unknown +``` + +`llmgw_gpu_nvlink_peers` in the metrics says the same thing across a fleet: a +flat zero everywhere means the topology is unknown rather than absent. + ### InfiniBand, if you have it ``` @@ -230,6 +259,12 @@ ibstat # must list your HCAs with State: Active ls /sys/class/infiniband ``` +The agent checks what you declare against what the node has. An adapter that is +not there, or that has no ACTIVE port — physically linked but with no subnet +manager, which survives a reboot and looks like a software problem — takes that +fabric out of service on this node, loudly, in the log. Without that check the +replica starts, passes its health checks, and hangs on its first all-reduce. + Fabric membership is configuration, not discovery, and there is no way around that: OpenSM's default subnet prefix is the same on every fabric, so a matching prefix proves nothing and a wrong guess produces a replica that starts, passes diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d3178a1..458139d 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -344,6 +344,38 @@ func (a *Agent) settleAdopted(sh *supervisor.Shard, o supervisor.Orphan) { "shard", o.ShardID, "replica", o.ReplicaID) } +// usableFabrics reports the fabrics this node can actually carry traffic on. +// +// Loud about what it drops. A fabric silently missing from a registration looks +// identical to one never configured, and the operator who typed the adapter +// name is the only person who can fix it — so the reason is logged here, on the +// machine where both halves are visible, rather than inferred later from a +// replica that will not converge. +func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment { + declared := map[string][]string{} + for _, f := range a.cfg.Fabrics { + declared[f.Fabric] = f.HCAs + } + + var out []inventory.FabricAttachment + for _, c := range discovery.CheckFabrics(declared, inv.HCAs) { + for _, p := range c.Problems { + a.log.Error("a declared fabric adapter cannot carry traffic", + "fabric", c.Fabric, "problem", p) + } + if !c.OK() { + a.log.Error("fabric not advertised; no declared adapter on it can carry traffic", + "fabric", c.Fabric, + "consequence", "no multi-node replica will be placed here") + continue + } + out = append(out, inventory.FabricAttachment{ + Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps, + }) + } + return out +} + // weights builds the fetcher lazily, so an agent that never starts a shard // never needs a cache directory. func (a *Agent) weights() *weights.Fetcher { @@ -445,9 +477,14 @@ func (a *Agent) session(ctx context.Context) error { inv := a.inv a.mu.Unlock() + // Declared fabric membership against what the node actually has. A fabric + // whose adapters are absent or have no active port is not advertised at + // all, so the control plane will not place a multi-node replica on it — + // rather than placing one that starts, passes health checks and hangs on + // its first all-reduce. reg := agentsvc.RegisterFromInventory( a.cfg.NodeID, a.cfg.Hostname, a.cfg.MgmtAddr, a.cfg.Version, - a.cfg.Labels, inv, a.cfg.Fabrics) + a.cfg.Labels, inv, a.usableFabrics(inv)) // Every registration reports what is already running, not just the first // one: a reconnect may be a control plane restart, and it has to recover // the same way (plan §6.5). diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 8e9e2e4..9ca4e00 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -337,9 +337,15 @@ func freePort(t *testing.T) int { } func TestRegistrationCarriesInventory(t *testing.T) { + // The node must actually have the adapters it declares. It used to declare + // mlx5_0 and mlx5_1 on a node whose sysfs held neither, which passed + // because nothing compared the two — and that is the gap a real cluster + // would have found as a replica hanging on its first all-reduce. + ibRoot := ibTreeWithActivePorts(t, "mlx5_0", "mlx5_1") h := start(t, func(c *Config) { c.Hostname = "gpu-a-01" c.Labels = map[string]string{"rack": "r12"} + c.Discovery.IBRoot = ibRoot c.Fabrics = []inventory.FabricAttachment{ {Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}, RateGbps: 200}, } @@ -648,3 +654,85 @@ func TestTheCacheDefaultsToTheFirstModelDirectory(t *testing.T) { t.Errorf("cache = %q, want the first model directory %q", got, dir) } } + +// ibTreeWithActivePorts builds a sysfs InfiniBand tree with the named adapters, +// each with one active port, shaped like the real thing. +func ibTreeWithActivePorts(t *testing.T, names ...string) string { + t.Helper() + root := t.TempDir() + for _, name := range names { + dir := filepath.Join(root, name, "ports", "1") + if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil { + t.Fatal(err) + } + for file, content := range map[string]string{ + "state": "4: ACTIVE", + "phys_state": "5: LinkUp", + "rate": "200 Gb/sec (4X HDR)", + "link_layer": "InfiniBand", + "lid": "0x1a", + "gids/0": "fec0:0000:0000:000a:0002:c903:00a0:1234", + } { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + } + return root +} + +func TestAFabricWhoseAdaptersAreAbsentIsNotAdvertised(t *testing.T) { + // The failure this prevents: a typo, or a card that enumerated differently + // after a reboot, produced NCCL_IB_HCA naming an adapter that is not there. + // The replica starts, passes its health checks, and hangs on its first + // all-reduce — the exact case the fabric invariant exists to prevent, + // arriving through a door nobody was watching. + h := start(t, func(c *Config) { + c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") + c.Fabrics = []inventory.FabricAttachment{ + {Fabric: "ib-typo", HCAs: []string{"mlx5_9"}, RateGbps: 200}, + } + }) + + n, ok := h.inv.Get("n1") + if !ok { + t.Fatal("node missing from inventory") + } + if n.HasFabric("ib-typo") { + t.Errorf("a fabric with no usable adapter was advertised: %+v", n.Fabrics) + } +} + +func TestAFabricRateIsTheSlowestActiveLink(t *testing.T) { + // A collective runs at the speed of its slowest link, so reporting the + // fastest would have the scheduler believe a fabric is quicker than it is. + root := t.TempDir() + for name, rate := range map[string]string{"mlx5_0": "200", "mlx5_1": "100"} { + dir := filepath.Join(root, name, "ports", "1") + if err := os.MkdirAll(filepath.Join(dir, "gids"), 0o755); err != nil { + t.Fatal(err) + } + for file, content := range map[string]string{ + "state": "4: ACTIVE", "phys_state": "5: LinkUp", + "rate": rate + " Gb/sec (4X HDR)", "link_layer": "InfiniBand", + "lid": "0x1a", "gids/0": "fec0::1", + } { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } + } + h := start(t, func(c *Config) { + c.Discovery.IBRoot = root + c.Fabrics = []inventory.FabricAttachment{ + {Fabric: "ib-a", HCAs: []string{"mlx5_0", "mlx5_1"}}, + } + }) + + n, _ := h.inv.Get("n1") + for _, f := range n.Fabrics { + if f.Fabric == "ib-a" && f.RateGbps != 100 { + t.Errorf("fabric rate = %d, want the slower link's 100", f.RateGbps) + } + } +} diff --git a/internal/agent/discovery/fabrics.go b/internal/agent/discovery/fabrics.go new file mode 100644 index 0000000..27d699d --- /dev/null +++ b/internal/agent/discovery/fabrics.go @@ -0,0 +1,116 @@ +package discovery + +import ( + "fmt" + "sort" + "strings" +) + +// FabricCheck is one declared fabric measured against what the node actually +// has. +type FabricCheck struct { + Fabric string + // Usable are the declared adapters that exist and have an active port. + Usable []string + // RateGbps is the slowest active port among them, because a collective + // runs at the speed of its slowest link. + RateGbps int + // Problems are the declared adapters that cannot carry traffic, and why. + Problems []string +} + +// OK reports whether this fabric can carry a collective at all. +func (c FabricCheck) OK() bool { return len(c.Usable) > 0 } + +// CheckFabrics measures declared fabric membership against discovered adapters. +// +// Fabric membership has to be configuration — OpenSM's default subnet prefix is +// the same on every fabric, so a matching prefix proves nothing — but taking it +// entirely on trust was a mistake of a different kind. A typo, or a card that +// enumerated differently after a reboot, produced NCCL_IB_HCA naming an adapter +// that does not exist; and an adapter whose cable is out, or whose subnet +// manager is down, is present in sysfs and carries nothing. +// +// Either way the replica starts, passes its health checks, and hangs on its +// first all-reduce — which is the failure the whole fabric invariant exists to +// prevent, arriving through a door nobody was watching. Declared and real are +// now compared where both are visible, which is only here. +func CheckFabrics(declared map[string][]string, discovered []HCA) []FabricCheck { + byName := make(map[string]HCA, len(discovered)) + for _, h := range discovered { + byName[h.Name] = h + } + + names := make([]string, 0, len(declared)) + for f := range declared { + names = append(names, f) + } + sort.Strings(names) + + out := make([]FabricCheck, 0, len(names)) + for _, fabric := range names { + c := FabricCheck{Fabric: fabric} + for _, want := range declared[fabric] { + hca, found := byName[want] + if !found { + c.Problems = append(c.Problems, fmt.Sprintf( + "%s is not on this node (found: %s)", want, listNames(discovered))) + continue + } + rate, active := activeRate(hca) + if !active { + c.Problems = append(c.Problems, fmt.Sprintf( + "%s has no active port (%s)", want, portStates(hca))) + continue + } + c.Usable = append(c.Usable, want) + if c.RateGbps == 0 || rate < c.RateGbps { + c.RateGbps = rate + } + } + out = append(out, c) + } + return out +} + +// activeRate returns the fastest active port's rate on an adapter. +// +// Only ACTIVE ports carry traffic. A port that is physically linked but has no +// subnet manager sits in INIT, looks healthy to anything counting adapters, and +// passes no data — which is precisely the case worth catching, because it +// survives a reboot and looks like a software problem. +func activeRate(h HCA) (int, bool) { + best, ok := 0, false + for _, p := range h.Ports { + if !strings.EqualFold(p.State, "ACTIVE") { + continue + } + ok = true + if p.RateGbps > best { + best = p.RateGbps + } + } + return best, ok +} + +func portStates(h HCA) string { + if len(h.Ports) == 0 { + return "no ports" + } + var parts []string + for _, p := range h.Ports { + parts = append(parts, fmt.Sprintf("port %d %s/%s", p.Num, p.State, p.PhysState)) + } + return strings.Join(parts, ", ") +} + +func listNames(hcas []HCA) string { + if len(hcas) == 0 { + return "none" + } + var names []string + for _, h := range hcas { + names = append(names, h.Name) + } + return strings.Join(names, ", ") +} diff --git a/internal/agent/discovery/fabrics_test.go b/internal/agent/discovery/fabrics_test.go new file mode 100644 index 0000000..4336c5c --- /dev/null +++ b/internal/agent/discovery/fabrics_test.go @@ -0,0 +1,114 @@ +package discovery + +import ( + "strings" + "testing" +) + +func hca(name string, ports ...Port) HCA { return HCA{Name: name, Ports: ports} } + +func active(rate int) Port { + return Port{Num: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: rate, LinkLayer: LinkInfiniBand} +} + +func TestADeclaredAdapterThatIsNotThereIsReported(t *testing.T) { + // A typo, or a card that enumerated differently after a reboot. Taken on + // trust it becomes NCCL_IB_HCA naming an adapter that does not exist, and + // the replica hangs on its first all-reduce. + got := CheckFabrics( + map[string][]string{"ib-a": {"mlx5_0", "mlx5_9"}}, + []HCA{hca("mlx5_0", active(200))}, + ) + if len(got) != 1 { + t.Fatalf("got %d checks", len(got)) + } + c := got[0] + if len(c.Usable) != 1 || c.Usable[0] != "mlx5_0" { + t.Errorf("usable = %v, want only the adapter that exists", c.Usable) + } + if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "mlx5_9") { + t.Errorf("problems = %v", c.Problems) + } + // The message names what is actually present, because the operator is + // looking at a typo and cannot see the node. + if !strings.Contains(c.Problems[0], "mlx5_0") { + t.Errorf("the problem does not say what was found: %q", c.Problems[0]) + } +} + +func TestAnAdapterWithNoActivePortCannotCarryTraffic(t *testing.T) { + // A port that is physically linked but has no subnet manager sits in INIT. + // It looks healthy to anything counting adapters and passes no data, which + // is the case worth catching: it survives a reboot and looks like software. + got := CheckFabrics( + map[string][]string{"ib-a": {"mlx5_0"}}, + []HCA{hca("mlx5_0", Port{Num: 1, State: "INIT", PhysState: "LinkUp", RateGbps: 200})}, + ) + c := got[0] + if c.OK() { + t.Error("a fabric with no active port was reported usable") + } + if len(c.Problems) != 1 || !strings.Contains(c.Problems[0], "INIT") { + t.Errorf("the problem does not say what state the port is in: %v", c.Problems) + } +} + +func TestTheRateIsTheSlowestActiveLink(t *testing.T) { + // A collective runs at the speed of its slowest link, so reporting the + // fastest would have the scheduler believe a fabric is quicker than it is. + got := CheckFabrics( + map[string][]string{"ib-a": {"fast", "slow"}}, + []HCA{hca("fast", active(400)), hca("slow", active(100))}, + ) + if got[0].RateGbps != 100 { + t.Errorf("rate = %d, want 100", got[0].RateGbps) + } +} + +func TestAnAdapterWithSeveralPortsUsesItsBestActiveOne(t *testing.T) { + // A dual-port card with one cable in is a perfectly good adapter, and + // judging it by the dead port would take a working fabric out of service. + got := CheckFabrics( + map[string][]string{"ib-a": {"mlx5_0"}}, + []HCA{hca("mlx5_0", + Port{Num: 1, State: "DOWN", PhysState: "Disabled"}, + active(200), + )}, + ) + if !got[0].OK() { + t.Fatalf("a card with one live port was rejected: %+v", got[0]) + } + if got[0].RateGbps != 200 { + t.Errorf("rate = %d, want the active port's 200", got[0].RateGbps) + } +} + +func TestAFabricWithNothingUsableIsNotOK(t *testing.T) { + got := CheckFabrics( + map[string][]string{"ib-dead": {"mlx5_0"}}, + []HCA{hca("mlx5_0", Port{Num: 1, State: "DOWN", PhysState: "Polling"})}, + ) + if got[0].OK() { + t.Error("a fabric whose only adapter is down was reported usable") + } +} + +func TestANodeWithNoFabricsDeclaredProducesNoChecks(t *testing.T) { + // Every node without InfiniBand, which must stay entirely unaffected. + if got := CheckFabrics(nil, []HCA{hca("mlx5_0", active(200))}); len(got) != 0 { + t.Errorf("got %d checks for a node that declared none", len(got)) + } +} + +func TestChecksComeBackInAStableOrder(t *testing.T) { + // Map iteration would otherwise reorder log lines between restarts, which + // makes a diff of two registrations unreadable. + declared := map[string][]string{"ib-c": {"m"}, "ib-a": {"m"}, "ib-b": {"m"}} + got := CheckFabrics(declared, nil) + if len(got) != 3 { + t.Fatalf("got %d", len(got)) + } + if got[0].Fabric != "ib-a" || got[1].Fabric != "ib-b" || got[2].Fabric != "ib-c" { + t.Errorf("order = %s, %s, %s", got[0].Fabric, got[1].Fabric, got[2].Fabric) + } +} diff --git a/internal/control/fleetmetrics/fleetmetrics.go b/internal/control/fleetmetrics/fleetmetrics.go index c8a0ec3..a9b39bf 100644 --- a/internal/control/fleetmetrics/fleetmetrics.go +++ b/internal/control/fleetmetrics/fleetmetrics.go @@ -51,6 +51,7 @@ type Collector struct { gpuUtil *prometheus.Desc gpuTemp *prometheus.Desc gpuHealthy *prometheus.Desc + gpuNVLink *prometheus.Desc gpuECC *prometheus.Desc replicaState *prometheus.Desc @@ -88,6 +89,11 @@ func New(src Source) *Collector { "A GPU's reported temperature.", gpu, nil), gpuHealthy: prometheus.NewDesc("llmgw_gpu_healthy", "1 when a GPU is eligible for placement, 0 when a fault keeps it out.", gpu, nil), + gpuNVLink: prometheus.NewDesc("llmgw_gpu_nvlink_peers", + "GPUs reachable over NVLink from this one. Always 0 on an agent built without "+ + "the nvml tag, which is every portable binary — so a flat zero across a fleet "+ + "means the topology is unknown rather than absent, and tensor groups are being "+ + "chosen by index.", gpu, nil), gpuECC: prometheus.NewDesc("llmgw_gpu_ecc_errors", "Uncorrected ECC errors on a GPU. Non-zero is why a card stops being scheduled.", gpu, nil), @@ -105,7 +111,8 @@ func New(src Source) *Collector { func (c *Collector) Describe(ch chan<- *prometheus.Desc) { for _, d := range []*prometheus.Desc{ c.nodeUp, c.nodeGPUs, c.nodeGPUsFree, c.nodeMemory, c.nodeSeen, - c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, c.gpuECC, + c.gpuMemoryTotal, c.gpuMemoryUsed, c.gpuUtil, c.gpuTemp, c.gpuHealthy, + c.gpuNVLink, c.gpuECC, c.replicaState, c.replicaReady, c.replicaIdle, } { ch <- d @@ -163,6 +170,8 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) { healthy = 1 } ch <- prometheus.MustNewConstMetric(c.gpuHealthy, prometheus.GaugeValue, healthy, labels...) + ch <- prometheus.MustNewConstMetric(c.gpuNVLink, prometheus.GaugeValue, + float64(len(g.NVLinkPeers)), labels...) ch <- prometheus.MustNewConstMetric(c.gpuECC, prometheus.GaugeValue, float64(g.Health.ECCErrors), labels...) }