Placement and environment generation had landed early, which made multi-node look further along than it was. A replica spanning nodes could not come up at all, and all three reasons came down to assumptions that only hold on one node. Not every rank serves HTTP. The agent health-checked every shard at its own port, but only the leader answers; workers timed out, reported Failed and took the replica with them. Engine.ServesHTTP now says which ranks serve, and a worker is judged by staying alive. That is deliberately weaker evidence: the replica's real readiness is the leader answering, which only happens once every worker has joined. A shard is not always one process. Engine.Commands returns a sequence — every command but the last is a sidecar kept alive for the shard's lifetime and torn down with it, and the last one's exit is the shard's. vLLM under Ray is why: the leader runs a Ray head and then the server against it, while a worker runs only the join and no vLLM process at all. RayCommand had been written and tested but never called, so launch_mode: ray would have run vllm serve against a cluster that did not exist. Sidecars are killed with the shard, because a Ray head left behind holds the port the next replica needs. Start order was backwards. Workers went first, on the theory that the head should wait for them. Both engines put the rendezvous point on rank 0, so a worker started first has nothing to connect to. vLLM's server blocks until the Ray cluster has the GPUs it needs, so the leader can start immediately and waits on its own. A command can now also require that rendezvous point to be listening before the next starts, bounded by a timeout so a head that never appears fails the shard instead of hanging it. Stop was also killing only the engine, leaving sidecars behind. None of this is verified against real engines — it is tested with stub processes. The Ray sequencing, and which launch mode a given vLLM version wants, can only be confirmed on hardware. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
75a8e76706
commit
1d1a6aebb0
11 files changed
+647
-125
No files matched your search
@@ -83,6 +83,14 @@ type Command struct {
|
||||
Args []string
|
||||
// Env is the complete environment, not a delta.
|
||||
Env []string
|
||||
// Name distinguishes the processes of a multi-process shard in logs.
|
||||
Name string
|
||||
// WaitForLeaderPort makes the supervisor hold the next command until
|
||||
// LeaderAddr:LeaderPort accepts a connection.
|
||||
//
|
||||
// It exists for Ray: the head has to be listening before a worker can join,
|
||||
// and both sides express the same wait because both dial the same address.
|
||||
WaitForLeaderPort bool
|
||||
}
|
||||
|
||||
// Engine builds launch commands for one inference engine.
|
||||
@@ -92,8 +100,24 @@ type Engine interface {
|
||||
// Validate rejects a profile that cannot work on these GPUs, before
|
||||
// anything is started.
|
||||
Validate(p Profile, gpus []GPUInfo) error
|
||||
// BuildCommand produces the launch command for a shard.
|
||||
BuildCommand(s ShardSpec) (Command, error)
|
||||
|
||||
// Commands returns the processes that make up one shard, in start order.
|
||||
//
|
||||
// Most shards are a single process. vLLM under Ray is not: the leader runs
|
||||
// a Ray head and then the server against it, and a worker runs only the Ray
|
||||
// join. Returning a sequence rather than one command is what lets the
|
||||
// supervisor treat both the same way — every command but the last is a
|
||||
// sidecar kept alive for the shard's lifetime, and the last one's exit is
|
||||
// the shard's exit.
|
||||
Commands(s ShardSpec) ([]Command, error)
|
||||
|
||||
// ServesHTTP reports whether this rank answers the OpenAI API.
|
||||
//
|
||||
// In a multi-node replica only the leader does. Health-checking a worker's
|
||||
// port would time out and fail a replica that is coming up perfectly well,
|
||||
// which is exactly what happened before this existed.
|
||||
ServesHTTP(s ShardSpec) bool
|
||||
|
||||
// HealthPath and MetricsPath are appended to the shard's base URL.
|
||||
HealthPath() string
|
||||
MetricsPath() string
|
||||
|
||||
@@ -1,11 +1,56 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// buildOne returns the single command a shard produces, failing if it produced
|
||||
// a different number.
|
||||
func buildOne(t *testing.T, e Engine, s ShardSpec) Command {
|
||||
t.Helper()
|
||||
cmds := build(t, e, s)
|
||||
if len(cmds) != 1 {
|
||||
t.Fatalf("expected 1 command, got %d: %s", len(cmds), summarize(cmds))
|
||||
}
|
||||
return cmds[0]
|
||||
}
|
||||
|
||||
// buildEngine returns the last command, which is the engine itself. The ones
|
||||
// before it are sidecars.
|
||||
func buildEngine(t *testing.T, e Engine, s ShardSpec) Command {
|
||||
t.Helper()
|
||||
cmds := build(t, e, s)
|
||||
return cmds[len(cmds)-1]
|
||||
}
|
||||
|
||||
func build(t *testing.T, e Engine, s ShardSpec) []Command {
|
||||
t.Helper()
|
||||
cmds, err := e.Commands(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cmds) == 0 {
|
||||
t.Fatal("no commands produced")
|
||||
}
|
||||
return cmds
|
||||
}
|
||||
|
||||
// summarize keeps failure output readable: a Command carries the whole
|
||||
// environment, and printing it buries the actual problem.
|
||||
func summarize(cmds []Command) string {
|
||||
var b strings.Builder
|
||||
for i, c := range cmds {
|
||||
if i > 0 {
|
||||
b.WriteString(" | ")
|
||||
}
|
||||
fmt.Fprintf(&b, "%s: %s %s", c.Name, c.Path, strings.Join(c.Args, " "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// envMap turns exec-form environment back into a map for assertions.
|
||||
func envMap(env []string) map[string]string {
|
||||
m := map[string]string{}
|
||||
@@ -61,10 +106,7 @@ func multiNodeSpec(rank int) ShardSpec {
|
||||
// --- environment generation: the part that breaks multi-node runs ---
|
||||
|
||||
func TestSingleNodeDisablesIB(t *testing.T) {
|
||||
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||
env := envMap(cmd.Env)
|
||||
|
||||
if env["NCCL_IB_DISABLE"] != "1" {
|
||||
@@ -79,10 +121,7 @@ func TestSingleNodeDisablesIB(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMultiNodePinsFabric(t *testing.T) {
|
||||
cmd, err := (&VLLM{}).BuildCommand(multiNodeSpec(0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildEngine(t, &VLLM{}, multiNodeSpec(0))
|
||||
env := envMap(cmd.Env)
|
||||
|
||||
// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
|
||||
@@ -108,10 +147,7 @@ func TestProfileEnvOverridesGenerated(t *testing.T) {
|
||||
"NCCL_IB_HCA": "mlx5_9",
|
||||
"NCCL_NET_GDR_LEVEL": "SYS",
|
||||
}
|
||||
cmd, err := (&VLLM{}).BuildCommand(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildEngine(t, &VLLM{}, s)
|
||||
env := envMap(cmd.Env)
|
||||
|
||||
if env["NCCL_IB_HCA"] != "mlx5_9" {
|
||||
@@ -126,17 +162,14 @@ func TestProcessEnvironmentIsInherited(t *testing.T) {
|
||||
// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
|
||||
// file set up.
|
||||
t.Setenv("LLMGW_TEST_MARKER", "present")
|
||||
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||
if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
|
||||
t.Error("the process environment was not inherited")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvIsSorted(t *testing.T) {
|
||||
cmd, _ := (&VLLM{}).BuildCommand(singleNodeSpec())
|
||||
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
|
||||
if !slices.IsSorted(cmd.Env) {
|
||||
t.Error("environment is not sorted; launches would not be reproducible in logs")
|
||||
}
|
||||
@@ -145,10 +178,7 @@ func TestEnvIsSorted(t *testing.T) {
|
||||
// --- vLLM ---
|
||||
|
||||
func TestVLLMSingleNodeArgs(t *testing.T) {
|
||||
cmd, err := (&VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}).BuildCommand(singleNodeSpec())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildOne(t, &VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}, singleNodeSpec())
|
||||
if cmd.Path != "/opt/vllm/bin/vllm" {
|
||||
t.Errorf("Path=%q", cmd.Path)
|
||||
}
|
||||
@@ -167,26 +197,54 @@ func TestVLLMSingleNodeArgs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMRayMode(t *testing.T) {
|
||||
func TestVLLMRayLeaderRunsTwoProcesses(t *testing.T) {
|
||||
// The leader starts a Ray head and then the server against it. A single
|
||||
// command would mean running vLLM against a cluster that does not exist.
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||
cmd, err := v.BuildCommand(multiNodeSpec(1))
|
||||
cmds, err := v.Commands(multiNodeSpec(0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if argValue(cmd.Args, "--distributed-executor-backend") != "ray" {
|
||||
t.Errorf("backend not set: %v", cmd.Args)
|
||||
if len(cmds) != 2 {
|
||||
t.Fatalf("expected 2 commands for the Ray leader, got %d", len(cmds))
|
||||
}
|
||||
if got := envMap(cmd.Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
|
||||
if cmds[0].Name != "ray" || !slices.Contains(cmds[0].Args, "--head") {
|
||||
t.Errorf("first command is not the Ray head: %+v", cmds[0])
|
||||
}
|
||||
if !cmds[0].WaitForLeaderPort {
|
||||
t.Error("the server would start before the head is listening")
|
||||
}
|
||||
if cmds[1].Name != "vllm" || argValue(cmds[1].Args, "--distributed-executor-backend") != "ray" {
|
||||
t.Errorf("second command is not the vLLM server: %+v", cmds[1])
|
||||
}
|
||||
if got := envMap(cmds[1].Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
|
||||
t.Errorf("RAY_ADDRESS=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
|
||||
// A Ray worker runs no vLLM process: the cluster hands its GPUs to the
|
||||
// leader. Starting a second server there would bind a port nobody uses and
|
||||
// compete for the same cards.
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||
cmds, err := v.Commands(multiNodeSpec(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cmds) != 1 {
|
||||
t.Fatalf("expected 1 command for a Ray worker, got %d: %+v", len(cmds), cmds)
|
||||
}
|
||||
if cmds[0].Name != "ray" || slices.Contains(cmds[0].Args, "--head") {
|
||||
t.Errorf("worker command wrong: %+v", cmds[0])
|
||||
}
|
||||
if argValue(cmds[0].Args, "--address") != "10.0.0.10:20001" {
|
||||
t.Errorf("worker is not joining the leader: %v", cmds[0].Args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMMPMode(t *testing.T) {
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
||||
cmd, err := v.BuildCommand(multiNodeSpec(0))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildOne(t, v, multiNodeSpec(0))
|
||||
if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
|
||||
t.Errorf("backend not set: %v", cmd.Args)
|
||||
}
|
||||
@@ -198,51 +256,42 @@ func TestVLLMMPMode(t *testing.T) {
|
||||
|
||||
func TestVLLMUnknownLaunchMode(t *testing.T) {
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
|
||||
if _, err := v.BuildCommand(multiNodeSpec(0)); err == nil {
|
||||
if _, err := v.Commands(multiNodeSpec(0)); err == nil {
|
||||
t.Error("an unknown launch mode was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMRayCommand(t *testing.T) {
|
||||
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||
func TestOnlyLeaderServesHTTP(t *testing.T) {
|
||||
// The fix for the bug that stopped multi-node replicas coming up at all:
|
||||
// health-checking a worker's port times out and fails a healthy replica.
|
||||
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
|
||||
if !e.ServesHTTP(multiNodeSpec(0)) {
|
||||
t.Errorf("%s: the leader must serve HTTP", e.Name())
|
||||
}
|
||||
if e.ServesHTTP(multiNodeSpec(1)) {
|
||||
t.Errorf("%s: a worker must not be expected to serve HTTP", e.Name())
|
||||
}
|
||||
if !e.ServesHTTP(singleNodeSpec()) {
|
||||
t.Errorf("%s: a single-node shard must serve HTTP", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
head, ok := v.RayCommand(multiNodeSpec(0))
|
||||
if !ok {
|
||||
t.Fatal("no ray command for the leader")
|
||||
func TestNoRayForSingleNodeOrMPMode(t *testing.T) {
|
||||
ray := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
||||
if cmds := buildOne(t, ray, singleNodeSpec()); cmds.Name != "vllm" {
|
||||
t.Errorf("a single-node profile was given a Ray process: %+v", cmds)
|
||||
}
|
||||
if !slices.Contains(head.Args, "--head") {
|
||||
t.Errorf("rank 0 must start the head: %v", head.Args)
|
||||
}
|
||||
|
||||
worker, ok := v.RayCommand(multiNodeSpec(1))
|
||||
if !ok {
|
||||
t.Fatal("no ray command for the worker")
|
||||
}
|
||||
if slices.Contains(worker.Args, "--head") {
|
||||
t.Errorf("rank 1 must not start a head: %v", worker.Args)
|
||||
}
|
||||
if argValue(worker.Args, "--address") != "10.0.0.10:20001" {
|
||||
t.Errorf("worker address wrong: %v", worker.Args)
|
||||
}
|
||||
|
||||
// Single node needs no Ray at all.
|
||||
if _, ok := v.RayCommand(singleNodeSpec()); ok {
|
||||
t.Error("a single-node profile was given a Ray command")
|
||||
}
|
||||
// Neither does mp mode.
|
||||
mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
||||
if _, ok := mp.RayCommand(multiNodeSpec(0)); ok {
|
||||
t.Error("mp mode was given a Ray command")
|
||||
if cmds := buildOne(t, mp, multiNodeSpec(0)); cmds.Name != "vllm" {
|
||||
t.Errorf("mp mode was given a Ray process: %+v", cmds)
|
||||
}
|
||||
}
|
||||
|
||||
// --- SGLang ---
|
||||
|
||||
func TestSGLangMultiNodeArgs(t *testing.T) {
|
||||
cmd, err := (&SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}).BuildCommand(multiNodeSpec(1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd := buildOne(t, &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}, multiNodeSpec(1))
|
||||
if cmd.Path != "/opt/sglang/bin/python" {
|
||||
t.Errorf("Path=%q", cmd.Path)
|
||||
}
|
||||
@@ -258,7 +307,7 @@ func TestSGLangMultiNodeArgs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
||||
cmd, _ := (&SGLang{}).BuildCommand(singleNodeSpec())
|
||||
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
|
||||
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
||||
if slices.Contains(cmd.Args, flag) {
|
||||
t.Errorf("%s was passed for a single-node profile", flag)
|
||||
@@ -270,7 +319,7 @@ func TestProfileArgsComeLast(t *testing.T) {
|
||||
// Operators must be able to override anything the adapter generated.
|
||||
s := singleNodeSpec()
|
||||
s.Profile.Args = []string{"--max-model-len", "32768"}
|
||||
cmd, _ := (&VLLM{}).BuildCommand(s)
|
||||
cmd := buildEngine(t, &VLLM{}, s)
|
||||
|
||||
i := slices.Index(cmd.Args, "--max-model-len")
|
||||
j := slices.Index(cmd.Args, "--tensor-parallel-size")
|
||||
|
||||
@@ -35,11 +35,26 @@ func (*SGLang) Validate(p Profile, gpus []GPUInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildCommand produces the launch command for one shard.
|
||||
// ServesHTTP reports whether this rank answers the API.
|
||||
//
|
||||
// SGLang supports multi-node natively, so unlike vLLM there is no second
|
||||
// process to sequence: every rank runs the same command with its own rank number.
|
||||
func (g *SGLang) BuildCommand(s ShardSpec) (Command, error) {
|
||||
// Every SGLang rank runs the same launcher, but only node-rank 0 binds the HTTP
|
||||
// port; the others exist to contribute GPUs to the tensor-parallel group.
|
||||
func (*SGLang) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||
|
||||
// Commands returns the processes for one shard.
|
||||
//
|
||||
// SGLang supports multi-node natively, so there is only ever one process: every
|
||||
// rank runs the same launcher with its own rank number and they rendezvous at
|
||||
// the leader's dist-init address.
|
||||
func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
|
||||
c, err := g.serverCommand(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []Command{c}, nil
|
||||
}
|
||||
|
||||
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
|
||||
python := g.cfg.Python
|
||||
if python == "" {
|
||||
python = "python3"
|
||||
@@ -64,5 +79,5 @@ func (g *SGLang) BuildCommand(s ShardSpec) (Command, error) {
|
||||
}
|
||||
args = append(args, p.Args...)
|
||||
|
||||
return Command{Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||
return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
|
||||
}
|
||||
@@ -46,8 +46,38 @@ func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildCommand produces the launch command for one shard.
|
||||
func (v *VLLM) BuildCommand(s ShardSpec) (Command, error) {
|
||||
// ServesHTTP reports whether this rank answers the API.
|
||||
//
|
||||
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
|
||||
// only the driver binds the port. Either way it is the leader alone.
|
||||
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
||||
|
||||
// Commands returns the processes for one shard, in start order.
|
||||
//
|
||||
// Single node, or multi-node under multiprocessing: one server process. Under
|
||||
// Ray: the leader starts a head and then the server, a worker starts only the
|
||||
// join. A Ray worker running no vLLM process at all is correct — the cluster
|
||||
// gives the leader its GPUs.
|
||||
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
|
||||
ray, useRay := v.rayCommand(s)
|
||||
if useRay && !s.Leader() {
|
||||
// A worker is the Ray join and nothing else. It blocks, so its exit is
|
||||
// the shard's exit, which is what the supervisor needs.
|
||||
return []Command{ray}, nil
|
||||
}
|
||||
|
||||
server, err := v.serverCommand(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if useRay {
|
||||
return []Command{ray, server}, nil
|
||||
}
|
||||
return []Command{server}, nil
|
||||
}
|
||||
|
||||
// serverCommand produces the vLLM server process.
|
||||
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
bin := v.cfg.Bin
|
||||
if bin == "" {
|
||||
bin = "vllm"
|
||||
@@ -88,15 +118,12 @@ func (v *VLLM) BuildCommand(s ShardSpec) (Command, error) {
|
||||
// Profile arguments last, so an operator can override anything above.
|
||||
args = append(args, p.Args...)
|
||||
|
||||
return Command{Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
||||
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
||||
}
|
||||
|
||||
// RayCommand returns the command that joins this node to the Ray cluster, or
|
||||
// rayCommand returns the command that starts or joins the Ray cluster, or
|
||||
// ok=false when the launch mode does not use Ray.
|
||||
//
|
||||
// It is separate from BuildCommand because it has to run first and finish
|
||||
// before vLLM starts: the supervisor sequences the two.
|
||||
func (v *VLLM) RayCommand(s ShardSpec) (Command, bool) {
|
||||
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
||||
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
|
||||
return Command{}, false
|
||||
}
|
||||
@@ -113,5 +140,10 @@ func (v *VLLM) RayCommand(s ShardSpec) (Command, bool) {
|
||||
}
|
||||
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
|
||||
|
||||
return Command{Path: bin, Args: args, Env: materializeEnv(baseEnv(s))}, true
|
||||
return Command{
|
||||
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
||||
// The head must be listening before anything else happens: a worker
|
||||
// cannot join, and the leader's server cannot claim GPUs, until it is.
|
||||
WaitForLeaderPort: true,
|
||||
}, true
|
||||
}
|
||||
Reference in new issue
Block a user