From 9ee21dc8142ecca9f835bd669f0e29af835b4b32 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 00:17:31 +0200 Subject: [PATCH] Fix what a multi-node launch would have got wrong on real hardware vLLM's mp launch gave workers no rank and the leader's address as their own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank, --master-addr and --master-port, and the workers --headless, as vLLM documents it. NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one interface name for every node; the agent already worked out its own - the one holding its management address - and now uses it. On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts, which does not route. The agent finds the RoCE v2 entry holding an IPv4 address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none. A worker whose only process needs the leader - a Ray join - started without waiting for it, since the wait was honoured for sidecars alone; it found no head, exited and failed the replica. It now waits. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/agent/agent.go | 35 +++++++++++++++++ internal/agent/agent_test.go | 26 +++++++++++++ internal/agent/discovery/ib.go | 29 ++++++++++++++ internal/agent/discovery/ib_test.go | 30 ++++++++++++++- internal/agent/engines/engine.go | 8 ++++ internal/agent/engines/engines_test.go | 40 +++++++++++++++++--- internal/agent/engines/vllm.go | 17 +++++++-- internal/agent/supervisor/supervisor.go | 10 +++++ internal/agent/supervisor/supervisor_test.go | 33 ++++++++++++++++ 9 files changed, 218 insertions(+), 10 deletions(-) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 35b2a31..a0f4bdb 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1161,6 +1161,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan< } spec, timeout := agentsvc.ShardSpecFromStart(m) + a.localizeSpec(&spec) eng, err := a.reg.Get(spec.Profile.Engine) if err != nil { @@ -1553,3 +1554,37 @@ func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply { } return out } + +// localizeSpec fills in what only this node knows. +// +// The interface NCCL bootstraps over is this node's, not the cluster's: the +// control plane sends one name for every node, and a node whose management +// network is on ens3 rather than eth0 would have NCCL bind nothing. And on +// RoCE, which GID entry routes is a property of these adapters. +func (a *Agent) localizeSpec(spec *engines.ShardSpec) { + if a.cfg.MgmtInterface != "" { + spec.MgmtInterface = a.cfg.MgmtInterface + } + if len(spec.HCAs) == 0 { + return + } + a.mu.Lock() + hcas := a.inv.HCAs + a.mu.Unlock() + want := map[string]bool{} + for _, h := range spec.HCAs { + want[h] = true + } + for _, h := range hcas { + if !want[h.Name] { + continue + } + for _, p := range h.Ports { + if p.Active() && p.RoCEv2GID >= 0 { + idx := p.RoCEv2GID + spec.GIDIndex = &idx + return + } + } + } +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index f71d29b..9ca6a39 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -22,6 +22,7 @@ import ( "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor" "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc" @@ -787,3 +788,28 @@ func TestAPortGoingDownIsToldToTheControlPlaneOnce(t *testing.T) { t.Error("the change should be sent once, not on every heartbeat") } } + +func TestAShardUsesThisNodesInterfaceAndRoCEIndex(t *testing.T) { + // The control plane sends one interface name for every node; a node whose + // management network is on another had NCCL bind nothing. And which RoCE + // GID entry routes is a property of this node's adapters. + a := &Agent{cfg: Config{MgmtInterface: "ens3"}} + a.inv.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{ + {Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkEthernet, RoCEv2GID: 3}}}} + spec := engines.ShardSpec{MgmtInterface: "eth0", HCAs: []string{"mlx5_0"}} + a.localizeSpec(&spec) + if spec.MgmtInterface != "ens3" { + t.Errorf("interface = %q", spec.MgmtInterface) + } + if spec.GIDIndex == nil || *spec.GIDIndex != 3 { + t.Errorf("GID index = %v", spec.GIDIndex) + } + + ib := engines.ShardSpec{HCAs: []string{"mlx5_1"}} + a.inv.HCAs = append(a.inv.HCAs, discovery.HCA{Name: "mlx5_1", Ports: []discovery.Port{ + {Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkInfiniBand, RoCEv2GID: -1}}}) + a.localizeSpec(&ib) + if ib.GIDIndex != nil { + t.Error("an InfiniBand shard was given a GID index") + } +} diff --git a/internal/agent/discovery/ib.go b/internal/agent/discovery/ib.go index b54d1fd..7e8fc14 100644 --- a/internal/agent/discovery/ib.go +++ b/internal/agent/discovery/ib.go @@ -46,6 +46,12 @@ type Port struct { // names, as far as the driver has them. The data counters are converted // to bytes; sysfs counts them in four-byte words. Counters map[string]uint64 `json:"counters,omitempty"` + // RoCEv2GID is the GID table index NCCL should use on a RoCE port: the + // first RoCE v2 entry holding an IPv4 address, else the first RoCE v2 + // entry. -1 on InfiniBand, and on a RoCE port with no v2 entry. NCCL's + // default of 0 is a RoCE v1 link-local entry on most setups, which does + // not route, and a job on it hangs. + RoCEv2GID int `json:"rocev2_gid"` } // PortCounters are the counters worth watching, from each port's counters/ @@ -156,6 +162,10 @@ func readPort(dir string, num int) Port { LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))), SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))), } + p.RoCEv2GID = -1 + if p.LinkLayer == LinkEthernet { + p.RoCEv2GID = rocev2GID(dir) + } for _, name := range PortCounters { raw := readTrimmed(filepath.Join(dir, "counters", name)) v, err := strconv.ParseUint(raw, 10, 64) @@ -233,3 +243,22 @@ func readTrimmed(path string) string { } return strings.TrimSpace(string(b)) } + +// rocev2GID finds the GID index a RoCE port's traffic should use. +func rocev2GID(dir string) int { + first := -1 + for i := 0; i < 64; i++ { + typ := readTrimmed(filepath.Join(dir, "gid_attrs", "types", strconv.Itoa(i))) + if !strings.EqualFold(typ, "RoCE v2") { + continue + } + gid := strings.ToLower(readTrimmed(filepath.Join(dir, "gids", strconv.Itoa(i)))) + if strings.HasPrefix(gid, "0000:0000:0000:0000:0000:ffff:") { + return i + } + if first < 0 { + first = i + } + } + return first +} diff --git a/internal/agent/discovery/ib_test.go b/internal/agent/discovery/ib_test.go index fe02209..884ef66 100644 --- a/internal/agent/discovery/ib_test.go +++ b/internal/agent/discovery/ib_test.go @@ -49,7 +49,12 @@ func fakeSysfs(t *testing.T) string { write("ib/mlx5_2/ports/1/state", "4: ACTIVE") write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)") write("ib/mlx5_2/ports/1/link_layer", "Ethernet") - write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001") + write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001") + write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1") + write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001") + write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2") + write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001") + write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2") // A PCI device with a NUMA node, linked the way the driver does it. pci := filepath.Join(root, "pci", "0000:0b:00.0") @@ -232,3 +237,26 @@ func TestAPortsCountersAreRead(t *testing.T) { t.Error("a counter the driver does not have should be absent, not zero") } } + +func TestRoCEFindsItsRoutableGID(t *testing.T) { + // NCCL's default GID index is a v1 link-local entry on most hosts, which + // does not route: the job hangs. The v2 entry holding an IPv4 address is + // the one to use, over an earlier v2 link-local one. + hcas, err := DiscoverIB(fakeSysfs(t)) + if err != nil { + t.Fatal(err) + } + for _, h := range hcas { + p := h.Ports[0] + switch h.Name { + case "mlx5_2": + if p.RoCEv2GID != 3 { + t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID) + } + default: + if p.RoCEv2GID != -1 { + t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID) + } + } + } +} diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index 145d948..3d9bbe6 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -63,6 +63,9 @@ type ShardSpec struct { HCAs []string // MgmtInterface is the ethernet interface used for bootstrap traffic. MgmtInterface string + // GIDIndex is the RoCE v2 GID index for the HCAs, set by the agent when + // they are RoCE; nil on InfiniBand, where NCCL needs none. + GIDIndex *int HTTPPort int Profile Profile @@ -215,6 +218,11 @@ func baseEnv(s ShardSpec) map[string]string { if len(s.HCAs) > 0 { env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",") env["NCCL_IB_DISABLE"] = "0" + // RoCE: the index of the routable RoCE v2 entry. NCCL's default is a + // v1 link-local one on most hosts, and a job on it hangs. + if s.GIDIndex != nil { + env["NCCL_IB_GID_INDEX"] = strconv.Itoa(*s.GIDIndex) + } } else { // Single node, or a node with no fabric. Either way IB is not the path. env["NCCL_IB_DISABLE"] = "1" diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index ef8cb89..6189d68 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -3,6 +3,7 @@ package engines import ( "fmt" "slices" + "strconv" "strings" "testing" ) @@ -243,14 +244,41 @@ func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) { } func TestVLLMMPMode(t *testing.T) { + // vLLM's multi-node launch: every node the same serve with its rank and + // the leader's address, workers headless. Workers used to be given the + // leader's address as their own VLLM_HOST_IP, and no rank at all. v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}} - cmd := buildOne(t, v, multiNodeSpec(0)) - if argValue(cmd.Args, "--distributed-executor-backend") != "mp" { - t.Errorf("backend not set: %v", cmd.Args) + for _, rank := range []int{0, 1} { + cmd := buildOne(t, v, multiNodeSpec(rank)) + for flag, want := range map[string]string{ + "--distributed-executor-backend": "mp", "--nnodes": "2", + "--node-rank": strconv.Itoa(rank), "--master-addr": "10.0.0.10", "--master-port": "20001", + } { + if got := argValue(cmd.Args, flag); got != want { + t.Errorf("rank %d: %s = %q, want %q (%v)", rank, flag, got, want, cmd.Args) + } + } + if headless := slices.Contains(cmd.Args, "--headless"); headless != (rank > 0) { + t.Errorf("rank %d: headless = %v", rank, headless) + } + if ip, ok := envMap(cmd.Env)["VLLM_HOST_IP"]; ok && ip == "10.0.0.10" && rank > 0 { + t.Errorf("a worker was told the leader's address is its own") + } } - env := envMap(cmd.Env) - if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" { - t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"]) +} + +func TestRoCEGetsItsGIDIndex(t *testing.T) { + s := multiNodeSpec(0) + idx := 3 + s.GIDIndex = &idx + env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env) + if env["NCCL_IB_GID_INDEX"] != "3" { + t.Errorf("NCCL_IB_GID_INDEX = %q", env["NCCL_IB_GID_INDEX"]) + } + // InfiniBand needs none, and must not be given one. + env = envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env) + if _, ok := env["NCCL_IB_GID_INDEX"]; ok { + t.Error("an InfiniBand shard was given a GID index") } } diff --git a/internal/agent/engines/vllm.go b/internal/agent/engines/vllm.go index 9eb8eef..76ebd9b 100644 --- a/internal/agent/engines/vllm.go +++ b/internal/agent/engines/vllm.go @@ -107,9 +107,20 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort) args = append(args, "--distributed-executor-backend", "ray") case "mp": - env["VLLM_HOST_IP"] = s.LeaderAddr - env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort) - args = append(args, "--distributed-executor-backend", "mp") + // vLLM's own multi-node launch: every node runs the same serve + // with its rank and the leader's address, and the workers run + // headless — no API server, only their GPUs. VLLM_HOST_IP is + // this node's address for vLLM to find itself, so it is left + // to vLLM; the leader's, which workers used to be given, made + // every worker announce itself as the leader. + args = append(args, "--distributed-executor-backend", "mp", + "--nnodes", strconv.Itoa(s.NNodes), + "--node-rank", strconv.Itoa(s.Rank), + "--master-addr", s.LeaderAddr, + "--master-port", strconv.Itoa(s.LeaderPort)) + if !s.Leader() { + args = append(args, "--headless") + } default: return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode) } diff --git a/internal/agent/supervisor/supervisor.go b/internal/agent/supervisor/supervisor.go index f64d815..b010e59 100644 --- a/internal/agent/supervisor/supervisor.go +++ b/internal/agent/supervisor/supervisor.go @@ -243,6 +243,16 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma } cmd := cmds[len(cmds)-1] + // The main process can need the rendezvous point too: a Ray worker's + // join is its only process, and started before the head listens it + // exits, failing the shard and with it the replica. + if cmd.WaitForLeaderPort && !sh.Spec.Leader() { + addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort) + if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil { + cancel() + return fmt.Errorf("waiting for %s (%s): %w", addr, cmd.Name, err) + } + } c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...) c.Env = cmd.Env diff --git a/internal/agent/supervisor/supervisor_test.go b/internal/agent/supervisor/supervisor_test.go index 8bc5587..b6ce756 100644 --- a/internal/agent/supervisor/supervisor_test.go +++ b/internal/agent/supervisor/supervisor_test.go @@ -794,3 +794,36 @@ func TestAnAdoptedShardCanBecomeReady(t *testing.T) { } _ = s2.Stop(t.Context(), "a") } + +func TestAWorkersOnlyProcessWaitsForTheLeader(t *testing.T) { + // A Ray worker's join is its only process. The wait was honoured only for + // sidecars, so the join started at once, found no head, exited, and took + // the replica down with it. + s, _ := newSupervisor(t) + s.cfg.RendezvousTimeout = 5 * time.Second + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + _ = ln.Close() + go func() { + time.Sleep(400 * time.Millisecond) + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err == nil { + t.Cleanup(func() { _ = l.Close() }) + } + }() + + spec := engines.ShardSpec{Rank: 1, LeaderAddr: "127.0.0.1", LeaderPort: port} + cmds := []engines.Command{{Name: "join", Path: "/bin/sh", Args: []string{"-c", "sleep 30"}, + Env: os.Environ(), WaitForLeaderPort: true}} + start := time.Now() + if _, err := s.Start(t.Context(), "w", spec, cmds); err != nil { + t.Fatal(err) + } + if d := time.Since(start); d < 300*time.Millisecond { + t.Errorf("the worker started after %v, before the leader was listening", d) + } +}