Fix what a multi-node launch would have got wrong on real hardware
vLLM's mp launch gave workers no rank and the leader's address as their own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank, --master-addr and --master-port, and the workers --headless, as vLLM documents it. NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one interface name for every node; the agent already worked out its own - the one holding its management address - and now uses it. On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts, which does not route. The agent finds the RoCE v2 entry holding an IPv4 address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none. A worker whose only process needs the leader - a Ray join - started without waiting for it, since the wait was honoured for sidecars alone; it found no head, exited and failed the replica. It now waits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
93bd16f00e
commit
9ee21dc814
9 files changed
+218
-10
No files matched your search
@@ -1161,6 +1161,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
|
||||
spec, timeout := agentsvc.ShardSpecFromStart(m)
|
||||
a.localizeSpec(&spec)
|
||||
|
||||
eng, err := a.reg.Get(spec.Profile.Engine)
|
||||
if err != nil {
|
||||
@@ -1553,3 +1554,37 @@ func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// localizeSpec fills in what only this node knows.
|
||||
//
|
||||
// The interface NCCL bootstraps over is this node's, not the cluster's: the
|
||||
// control plane sends one name for every node, and a node whose management
|
||||
// network is on ens3 rather than eth0 would have NCCL bind nothing. And on
|
||||
// RoCE, which GID entry routes is a property of these adapters.
|
||||
func (a *Agent) localizeSpec(spec *engines.ShardSpec) {
|
||||
if a.cfg.MgmtInterface != "" {
|
||||
spec.MgmtInterface = a.cfg.MgmtInterface
|
||||
}
|
||||
if len(spec.HCAs) == 0 {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
hcas := a.inv.HCAs
|
||||
a.mu.Unlock()
|
||||
want := map[string]bool{}
|
||||
for _, h := range spec.HCAs {
|
||||
want[h] = true
|
||||
}
|
||||
for _, h := range hcas {
|
||||
if !want[h.Name] {
|
||||
continue
|
||||
}
|
||||
for _, p := range h.Ports {
|
||||
if p.Active() && p.RoCEv2GID >= 0 {
|
||||
idx := p.RoCEv2GID
|
||||
spec.GIDIndex = &idx
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"golang.org/x/net/http2"
|
||||
"golang.org/x/net/http2/h2c"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
@@ -787,3 +788,28 @@ func TestAPortGoingDownIsToldToTheControlPlaneOnce(t *testing.T) {
|
||||
t.Error("the change should be sent once, not on every heartbeat")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShardUsesThisNodesInterfaceAndRoCEIndex(t *testing.T) {
|
||||
// The control plane sends one interface name for every node; a node whose
|
||||
// management network is on another had NCCL bind nothing. And which RoCE
|
||||
// GID entry routes is a property of this node's adapters.
|
||||
a := &Agent{cfg: Config{MgmtInterface: "ens3"}}
|
||||
a.inv.HCAs = []discovery.HCA{{Name: "mlx5_0", Ports: []discovery.Port{
|
||||
{Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkEthernet, RoCEv2GID: 3}}}}
|
||||
spec := engines.ShardSpec{MgmtInterface: "eth0", HCAs: []string{"mlx5_0"}}
|
||||
a.localizeSpec(&spec)
|
||||
if spec.MgmtInterface != "ens3" {
|
||||
t.Errorf("interface = %q", spec.MgmtInterface)
|
||||
}
|
||||
if spec.GIDIndex == nil || *spec.GIDIndex != 3 {
|
||||
t.Errorf("GID index = %v", spec.GIDIndex)
|
||||
}
|
||||
|
||||
ib := engines.ShardSpec{HCAs: []string{"mlx5_1"}}
|
||||
a.inv.HCAs = append(a.inv.HCAs, discovery.HCA{Name: "mlx5_1", Ports: []discovery.Port{
|
||||
{Num: 1, State: "ACTIVE", LinkLayer: discovery.LinkInfiniBand, RoCEv2GID: -1}}})
|
||||
a.localizeSpec(&ib)
|
||||
if ib.GIDIndex != nil {
|
||||
t.Error("an InfiniBand shard was given a GID index")
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,12 @@ type Port struct {
|
||||
// names, as far as the driver has them. The data counters are converted
|
||||
// to bytes; sysfs counts them in four-byte words.
|
||||
Counters map[string]uint64 `json:"counters,omitempty"`
|
||||
// RoCEv2GID is the GID table index NCCL should use on a RoCE port: the
|
||||
// first RoCE v2 entry holding an IPv4 address, else the first RoCE v2
|
||||
// entry. -1 on InfiniBand, and on a RoCE port with no v2 entry. NCCL's
|
||||
// default of 0 is a RoCE v1 link-local entry on most setups, which does
|
||||
// not route, and a job on it hangs.
|
||||
RoCEv2GID int `json:"rocev2_gid"`
|
||||
}
|
||||
|
||||
// PortCounters are the counters worth watching, from each port's counters/
|
||||
@@ -156,6 +162,10 @@ func readPort(dir string, num int) Port {
|
||||
LID: parseMaybeHex(readTrimmed(filepath.Join(dir, "lid"))),
|
||||
SMLID: parseMaybeHex(readTrimmed(filepath.Join(dir, "sm_lid"))),
|
||||
}
|
||||
p.RoCEv2GID = -1
|
||||
if p.LinkLayer == LinkEthernet {
|
||||
p.RoCEv2GID = rocev2GID(dir)
|
||||
}
|
||||
for _, name := range PortCounters {
|
||||
raw := readTrimmed(filepath.Join(dir, "counters", name))
|
||||
v, err := strconv.ParseUint(raw, 10, 64)
|
||||
@@ -233,3 +243,22 @@ func readTrimmed(path string) string {
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// rocev2GID finds the GID index a RoCE port's traffic should use.
|
||||
func rocev2GID(dir string) int {
|
||||
first := -1
|
||||
for i := 0; i < 64; i++ {
|
||||
typ := readTrimmed(filepath.Join(dir, "gid_attrs", "types", strconv.Itoa(i)))
|
||||
if !strings.EqualFold(typ, "RoCE v2") {
|
||||
continue
|
||||
}
|
||||
gid := strings.ToLower(readTrimmed(filepath.Join(dir, "gids", strconv.Itoa(i))))
|
||||
if strings.HasPrefix(gid, "0000:0000:0000:0000:0000:ffff:") {
|
||||
return i
|
||||
}
|
||||
if first < 0 {
|
||||
first = i
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
@@ -49,7 +49,12 @@ func fakeSysfs(t *testing.T) string {
|
||||
write("ib/mlx5_2/ports/1/state", "4: ACTIVE")
|
||||
write("ib/mlx5_2/ports/1/rate", "100 Gb/sec (4X EDR)")
|
||||
write("ib/mlx5_2/ports/1/link_layer", "Ethernet")
|
||||
write("ib/mlx5_2/ports/1/gids/0", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
||||
write("ib/mlx5_2/ports/1/gids/0", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/types/0", "IB/RoCE v1")
|
||||
write("ib/mlx5_2/ports/1/gids/1", "fe80:0000:0000:0000:0000:0000:0000:0001")
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/types/1", "RoCE v2")
|
||||
write("ib/mlx5_2/ports/1/gids/3", "0000:0000:0000:0000:0000:ffff:0a00:0001")
|
||||
write("ib/mlx5_2/ports/1/gid_attrs/types/3", "RoCE v2")
|
||||
|
||||
// A PCI device with a NUMA node, linked the way the driver does it.
|
||||
pci := filepath.Join(root, "pci", "0000:0b:00.0")
|
||||
@@ -232,3 +237,26 @@ func TestAPortsCountersAreRead(t *testing.T) {
|
||||
t.Error("a counter the driver does not have should be absent, not zero")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoCEFindsItsRoutableGID(t *testing.T) {
|
||||
// NCCL's default GID index is a v1 link-local entry on most hosts, which
|
||||
// does not route: the job hangs. The v2 entry holding an IPv4 address is
|
||||
// the one to use, over an earlier v2 link-local one.
|
||||
hcas, err := DiscoverIB(fakeSysfs(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hcas {
|
||||
p := h.Ports[0]
|
||||
switch h.Name {
|
||||
case "mlx5_2":
|
||||
if p.RoCEv2GID != 3 {
|
||||
t.Errorf("RoCE GID index = %d, want 3", p.RoCEv2GID)
|
||||
}
|
||||
default:
|
||||
if p.RoCEv2GID != -1 {
|
||||
t.Errorf("%s is InfiniBand and has a GID index %d", h.Name, p.RoCEv2GID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,9 @@ type ShardSpec struct {
|
||||
HCAs []string
|
||||
// MgmtInterface is the ethernet interface used for bootstrap traffic.
|
||||
MgmtInterface string
|
||||
// GIDIndex is the RoCE v2 GID index for the HCAs, set by the agent when
|
||||
// they are RoCE; nil on InfiniBand, where NCCL needs none.
|
||||
GIDIndex *int
|
||||
|
||||
HTTPPort int
|
||||
Profile Profile
|
||||
@@ -215,6 +218,11 @@ func baseEnv(s ShardSpec) map[string]string {
|
||||
if len(s.HCAs) > 0 {
|
||||
env["NCCL_IB_HCA"] = strings.Join(s.HCAs, ",")
|
||||
env["NCCL_IB_DISABLE"] = "0"
|
||||
// RoCE: the index of the routable RoCE v2 entry. NCCL's default is a
|
||||
// v1 link-local one on most hosts, and a job on it hangs.
|
||||
if s.GIDIndex != nil {
|
||||
env["NCCL_IB_GID_INDEX"] = strconv.Itoa(*s.GIDIndex)
|
||||
}
|
||||
} else {
|
||||
// Single node, or a node with no fabric. Either way IB is not the path.
|
||||
env["NCCL_IB_DISABLE"] = "1"
|
||||
|
||||
@@ -3,6 +3,7 @@ package engines
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -243,14 +244,41 @@ func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestVLLMMPMode(t *testing.T) {
|
||||
// vLLM's multi-node launch: every node the same serve with its rank and
|
||||
// the leader's address, workers headless. Workers used to be given the
|
||||
// leader's address as their own VLLM_HOST_IP, and no rank at all.
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
||||
cmd := buildOne(t, v, multiNodeSpec(0))
|
||||
if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
|
||||
t.Errorf("backend not set: %v", cmd.Args)
|
||||
for _, rank := range []int{0, 1} {
|
||||
cmd := buildOne(t, v, multiNodeSpec(rank))
|
||||
for flag, want := range map[string]string{
|
||||
"--distributed-executor-backend": "mp", "--nnodes": "2",
|
||||
"--node-rank": strconv.Itoa(rank), "--master-addr": "10.0.0.10", "--master-port": "20001",
|
||||
} {
|
||||
if got := argValue(cmd.Args, flag); got != want {
|
||||
t.Errorf("rank %d: %s = %q, want %q (%v)", rank, flag, got, want, cmd.Args)
|
||||
}
|
||||
}
|
||||
if headless := slices.Contains(cmd.Args, "--headless"); headless != (rank > 0) {
|
||||
t.Errorf("rank %d: headless = %v", rank, headless)
|
||||
}
|
||||
if ip, ok := envMap(cmd.Env)["VLLM_HOST_IP"]; ok && ip == "10.0.0.10" && rank > 0 {
|
||||
t.Errorf("a worker was told the leader's address is its own")
|
||||
}
|
||||
}
|
||||
env := envMap(cmd.Env)
|
||||
if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
|
||||
t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
|
||||
}
|
||||
|
||||
func TestRoCEGetsItsGIDIndex(t *testing.T) {
|
||||
s := multiNodeSpec(0)
|
||||
idx := 3
|
||||
s.GIDIndex = &idx
|
||||
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
|
||||
if env["NCCL_IB_GID_INDEX"] != "3" {
|
||||
t.Errorf("NCCL_IB_GID_INDEX = %q", env["NCCL_IB_GID_INDEX"])
|
||||
}
|
||||
// InfiniBand needs none, and must not be given one.
|
||||
env = envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env)
|
||||
if _, ok := env["NCCL_IB_GID_INDEX"]; ok {
|
||||
t.Error("an InfiniBand shard was given a GID index")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -107,9 +107,20 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
||||
args = append(args, "--distributed-executor-backend", "ray")
|
||||
case "mp":
|
||||
env["VLLM_HOST_IP"] = s.LeaderAddr
|
||||
env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort)
|
||||
args = append(args, "--distributed-executor-backend", "mp")
|
||||
// vLLM's own multi-node launch: every node runs the same serve
|
||||
// with its rank and the leader's address, and the workers run
|
||||
// headless — no API server, only their GPUs. VLLM_HOST_IP is
|
||||
// this node's address for vLLM to find itself, so it is left
|
||||
// to vLLM; the leader's, which workers used to be given, made
|
||||
// every worker announce itself as the leader.
|
||||
args = append(args, "--distributed-executor-backend", "mp",
|
||||
"--nnodes", strconv.Itoa(s.NNodes),
|
||||
"--node-rank", strconv.Itoa(s.Rank),
|
||||
"--master-addr", s.LeaderAddr,
|
||||
"--master-port", strconv.Itoa(s.LeaderPort))
|
||||
if !s.Leader() {
|
||||
args = append(args, "--headless")
|
||||
}
|
||||
default:
|
||||
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
|
||||
}
|
||||
|
||||
@@ -243,6 +243,16 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
}
|
||||
|
||||
cmd := cmds[len(cmds)-1]
|
||||
// The main process can need the rendezvous point too: a Ray worker's
|
||||
// join is its only process, and started before the head listens it
|
||||
// exits, failing the shard and with it the replica.
|
||||
if cmd.WaitForLeaderPort && !sh.Spec.Leader() {
|
||||
addr := fmt.Sprintf("%s:%d", sh.Spec.LeaderAddr, sh.Spec.LeaderPort)
|
||||
if err := waitForPort(runCtx, addr, s.cfg.RendezvousTimeout); err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("waiting for %s (%s): %w", addr, cmd.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
c := exec.CommandContext(runCtx, cmd.Path, cmd.Args...)
|
||||
c.Env = cmd.Env
|
||||
|
||||
@@ -794,3 +794,36 @@ func TestAnAdoptedShardCanBecomeReady(t *testing.T) {
|
||||
}
|
||||
_ = s2.Stop(t.Context(), "a")
|
||||
}
|
||||
|
||||
func TestAWorkersOnlyProcessWaitsForTheLeader(t *testing.T) {
|
||||
// A Ray worker's join is its only process. The wait was honoured only for
|
||||
// sidecars, so the join started at once, found no head, exited, and took
|
||||
// the replica down with it.
|
||||
s, _ := newSupervisor(t)
|
||||
s.cfg.RendezvousTimeout = 5 * time.Second
|
||||
|
||||
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
port := ln.Addr().(*net.TCPAddr).Port
|
||||
_ = ln.Close()
|
||||
go func() {
|
||||
time.Sleep(400 * time.Millisecond)
|
||||
l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
|
||||
if err == nil {
|
||||
t.Cleanup(func() { _ = l.Close() })
|
||||
}
|
||||
}()
|
||||
|
||||
spec := engines.ShardSpec{Rank: 1, LeaderAddr: "127.0.0.1", LeaderPort: port}
|
||||
cmds := []engines.Command{{Name: "join", Path: "/bin/sh", Args: []string{"-c", "sleep 30"},
|
||||
Env: os.Environ(), WaitForLeaderPort: true}}
|
||||
start := time.Now()
|
||||
if _, err := s.Start(t.Context(), "w", spec, cmds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d := time.Since(start); d < 300*time.Millisecond {
|
||||
t.Errorf("the worker started after %v, before the leader was listening", d)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user