From 45e3f52da3d9fa58c2d8ad5ee394701d542e16f6 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 29 Sep 2026 09:13:59 +0200 Subject: [PATCH] Launch multi-node vLLM its own way by default, not through Ray An empty launch_mode meant ray: a Ray head and joins on every node, a dependency to install, and a path never checked against a real cluster. It now means mp - every node runs vllm serve with its rank and the leader's address, the workers headless - which is how vLLM documents a multi-node launch today and needs nothing but vLLM. ray is still there for a vLLM too old for mp's flags. The example agent configuration, doctor and the reference say so. Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/examples/agent.yaml | 9 +++++---- docs/reference.md | 10 ++++++---- internal/agent/engines/engines_test.go | 13 +++++++++++++ internal/agent/engines/vllm.go | 16 +++++++++------- internal/doctor/doctor.go | 2 +- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/deploy/examples/agent.yaml b/deploy/examples/agent.yaml index 938d388..2b46506 100644 --- a/deploy/examples/agent.yaml +++ b/deploy/examples/agent.yaml @@ -96,10 +96,11 @@ discovery: engines: vllm: bin: /opt/venvs/vllm/bin/vllm - # Which multi-node mechanism works depends on the vLLM version, and the - # flags have changed more than once. Check the version you deploy. - launch_mode: ray # "ray" or "mp" - ray_bin: /opt/venvs/vllm/bin/ray + # How a replica spanning nodes is launched. mp, the default, is what vLLM + # documents today and needs nothing but vLLM; ray is for a vLLM too old + # for mp's multi-node flags, and needs Ray on every node. + launch_mode: mp # "mp" or "ray" + # ray_bin: /opt/venvs/vllm/bin/ray sglang: # SGLang is launched as a module, which is its documented entry point. python: /opt/venvs/sglang/bin/python diff --git a/docs/reference.md b/docs/reference.md index e572d1a..292646e 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1351,11 +1351,13 @@ Bin is the vllm executable. `string` -LaunchMode selects the multi-node mechanism: "ray" or "mp". +LaunchMode selects the multi-node mechanism: "mp", the default, or +"ray". -Which one works depends on the vLLM version, and the flags have changed -more than once (plan §14). Both paths exist so a cluster can pick the one -its version supports instead of waiting for a code change. +mp is what vLLM documents today — every node runs serve with its rank, +the workers headless — and needs nothing but vLLM. ray is the older way, +kept for a vLLM too old for mp's multi-node flags; it needs Ray on every +node and has not been checked against a real cluster. ### `ray_bin` diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index ad145fa..b59a24f 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -463,3 +463,16 @@ func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) { t.Error("a replica on InfiniBand was told to use multi-node NVLink") } } + +func TestMultiNodeVLLMDefaultsToItsOwnLaunch(t *testing.T) { + // mp is what vLLM documents and needs nothing but vLLM; Ray was the + // default and was never checked against a real cluster. + cmds, err := (&VLLM{}).Commands(multiNodeSpec(1)) + if err != nil { + t.Fatal(err) + } + if len(cmds) != 1 || cmds[0].Name != "vllm" || argValue(cmds[0].Args, "--distributed-executor-backend") != "mp" || + !slices.Contains(cmds[0].Args, "--headless") { + t.Errorf("default worker = %+v", cmds) + } +} diff --git a/internal/agent/engines/vllm.go b/internal/agent/engines/vllm.go index 76ebd9b..2338bb6 100644 --- a/internal/agent/engines/vllm.go +++ b/internal/agent/engines/vllm.go @@ -9,11 +9,13 @@ import ( type VLLMConfig struct { // Bin is the vllm executable. Bin string `koanf:"bin"` - // LaunchMode selects the multi-node mechanism: "ray" or "mp". + // LaunchMode selects the multi-node mechanism: "mp", the default, or + // "ray". // - // Which one works depends on the vLLM version, and the flags have changed - // more than once (plan §14). Both paths exist so a cluster can pick the one - // its version supports instead of waiting for a code change. + // mp is what vLLM documents today — every node runs serve with its rank, + // the workers headless — and needs nothing but vLLM. ray is the older way, + // kept for a vLLM too old for mp's multi-node flags; it needs Ray on every + // node and has not been checked against a real cluster. LaunchMode string `koanf:"launch_mode"` // RayBin is used when LaunchMode is "ray". RayBin string `koanf:"ray_bin"` @@ -100,13 +102,13 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { if s.NNodes > 1 { switch v.cfg.LaunchMode { - case "", "ray": + case "ray": // Ray is started separately by the supervisor; vLLM then finds the // cluster through the address in the environment. env["VLLM_HOST_IP"] = s.LeaderAddr env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort) args = append(args, "--distributed-executor-backend", "ray") - case "mp": + case "", "mp": // vLLM's own multi-node launch: every node runs the same serve // with its rank and the leader's address, and the workers run // headless — no API server, only their GPUs. VLLM_HOST_IP is @@ -135,7 +137,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { // rayCommand returns the command that starts or joins the Ray cluster, or // ok=false when the launch mode does not use Ray. func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) { - if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") { + if s.NNodes <= 1 || v.cfg.LaunchMode != "ray" { return Command{}, false } bin := v.cfg.RayBin diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 554e53b..407070f 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -212,7 +212,7 @@ func (f *Facts) readConfig(o Options) { f.IfaceExists = err == nil f.LaunchMode = cfg.Engines.VLLM.LaunchMode if f.LaunchMode == "" { - f.LaunchMode = "ray" + f.LaunchMode = "mp" } // What a two-node shard on each fabric would be given, the part of an