Launch multi-node vLLM its own way by default, not through Ray
An empty launch_mode meant ray: a Ray head and joins on every node, a dependency to install, and a path never checked against a real cluster. It now means mp - every node runs vllm serve with its rank and the leader's address, the workers headless - which is how vLLM documents a multi-node launch today and needs nothing but vLLM. ray is still there for a vLLM too old for mp's flags. The example agent configuration, doctor and the reference say so. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
20479a2d5f
commit
45e3f52da3
5 files changed
+34
-16
No files matched your search
@@ -96,10 +96,11 @@ discovery:
|
|||||||
engines:
|
engines:
|
||||||
vllm:
|
vllm:
|
||||||
bin: /opt/venvs/vllm/bin/vllm
|
bin: /opt/venvs/vllm/bin/vllm
|
||||||
# Which multi-node mechanism works depends on the vLLM version, and the
|
# How a replica spanning nodes is launched. mp, the default, is what vLLM
|
||||||
# flags have changed more than once. Check the version you deploy.
|
# documents today and needs nothing but vLLM; ray is for a vLLM too old
|
||||||
launch_mode: ray # "ray" or "mp"
|
# for mp's multi-node flags, and needs Ray on every node.
|
||||||
ray_bin: /opt/venvs/vllm/bin/ray
|
launch_mode: mp # "mp" or "ray"
|
||||||
|
# ray_bin: /opt/venvs/vllm/bin/ray
|
||||||
sglang:
|
sglang:
|
||||||
# SGLang is launched as a module, which is its documented entry point.
|
# SGLang is launched as a module, which is its documented entry point.
|
||||||
python: /opt/venvs/sglang/bin/python
|
python: /opt/venvs/sglang/bin/python
|
||||||
|
|||||||
+6
-4
@@ -1351,11 +1351,13 @@ Bin is the vllm executable.
|
|||||||
|
|
||||||
`string`
|
`string`
|
||||||
|
|
||||||
LaunchMode selects the multi-node mechanism: "ray" or "mp".
|
LaunchMode selects the multi-node mechanism: "mp", the default, or
|
||||||
|
"ray".
|
||||||
|
|
||||||
Which one works depends on the vLLM version, and the flags have changed
|
mp is what vLLM documents today — every node runs serve with its rank,
|
||||||
more than once (plan §14). Both paths exist so a cluster can pick the one
|
the workers headless — and needs nothing but vLLM. ray is the older way,
|
||||||
its version supports instead of waiting for a code change.
|
kept for a vLLM too old for mp's multi-node flags; it needs Ray on every
|
||||||
|
node and has not been checked against a real cluster.
|
||||||
|
|
||||||
### `ray_bin`
|
### `ray_bin`
|
||||||
|
|
||||||
|
|||||||
@@ -463,3 +463,16 @@ func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
|
|||||||
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
|
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMultiNodeVLLMDefaultsToItsOwnLaunch(t *testing.T) {
|
||||||
|
// mp is what vLLM documents and needs nothing but vLLM; Ray was the
|
||||||
|
// default and was never checked against a real cluster.
|
||||||
|
cmds, err := (&VLLM{}).Commands(multiNodeSpec(1))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmds) != 1 || cmds[0].Name != "vllm" || argValue(cmds[0].Args, "--distributed-executor-backend") != "mp" ||
|
||||||
|
!slices.Contains(cmds[0].Args, "--headless") {
|
||||||
|
t.Errorf("default worker = %+v", cmds)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,11 +9,13 @@ import (
|
|||||||
type VLLMConfig struct {
|
type VLLMConfig struct {
|
||||||
// Bin is the vllm executable.
|
// Bin is the vllm executable.
|
||||||
Bin string `koanf:"bin"`
|
Bin string `koanf:"bin"`
|
||||||
// LaunchMode selects the multi-node mechanism: "ray" or "mp".
|
// LaunchMode selects the multi-node mechanism: "mp", the default, or
|
||||||
|
// "ray".
|
||||||
//
|
//
|
||||||
// Which one works depends on the vLLM version, and the flags have changed
|
// mp is what vLLM documents today — every node runs serve with its rank,
|
||||||
// more than once (plan §14). Both paths exist so a cluster can pick the one
|
// the workers headless — and needs nothing but vLLM. ray is the older way,
|
||||||
// its version supports instead of waiting for a code change.
|
// kept for a vLLM too old for mp's multi-node flags; it needs Ray on every
|
||||||
|
// node and has not been checked against a real cluster.
|
||||||
LaunchMode string `koanf:"launch_mode"`
|
LaunchMode string `koanf:"launch_mode"`
|
||||||
// RayBin is used when LaunchMode is "ray".
|
// RayBin is used when LaunchMode is "ray".
|
||||||
RayBin string `koanf:"ray_bin"`
|
RayBin string `koanf:"ray_bin"`
|
||||||
@@ -100,13 +102,13 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
|||||||
|
|
||||||
if s.NNodes > 1 {
|
if s.NNodes > 1 {
|
||||||
switch v.cfg.LaunchMode {
|
switch v.cfg.LaunchMode {
|
||||||
case "", "ray":
|
case "ray":
|
||||||
// Ray is started separately by the supervisor; vLLM then finds the
|
// Ray is started separately by the supervisor; vLLM then finds the
|
||||||
// cluster through the address in the environment.
|
// cluster through the address in the environment.
|
||||||
env["VLLM_HOST_IP"] = s.LeaderAddr
|
env["VLLM_HOST_IP"] = s.LeaderAddr
|
||||||
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
||||||
args = append(args, "--distributed-executor-backend", "ray")
|
args = append(args, "--distributed-executor-backend", "ray")
|
||||||
case "mp":
|
case "", "mp":
|
||||||
// vLLM's own multi-node launch: every node runs the same serve
|
// vLLM's own multi-node launch: every node runs the same serve
|
||||||
// with its rank and the leader's address, and the workers run
|
// with its rank and the leader's address, and the workers run
|
||||||
// headless — no API server, only their GPUs. VLLM_HOST_IP is
|
// headless — no API server, only their GPUs. VLLM_HOST_IP is
|
||||||
@@ -135,7 +137,7 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
|||||||
// rayCommand returns the command that starts or joins the Ray cluster, or
|
// rayCommand returns the command that starts or joins the Ray cluster, or
|
||||||
// ok=false when the launch mode does not use Ray.
|
// ok=false when the launch mode does not use Ray.
|
||||||
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
||||||
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
|
if s.NNodes <= 1 || v.cfg.LaunchMode != "ray" {
|
||||||
return Command{}, false
|
return Command{}, false
|
||||||
}
|
}
|
||||||
bin := v.cfg.RayBin
|
bin := v.cfg.RayBin
|
||||||
|
|||||||
@@ -212,7 +212,7 @@ func (f *Facts) readConfig(o Options) {
|
|||||||
f.IfaceExists = err == nil
|
f.IfaceExists = err == nil
|
||||||
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
|
f.LaunchMode = cfg.Engines.VLLM.LaunchMode
|
||||||
if f.LaunchMode == "" {
|
if f.LaunchMode == "" {
|
||||||
f.LaunchMode = "ray"
|
f.LaunchMode = "mp"
|
||||||
}
|
}
|
||||||
|
|
||||||
// What a two-node shard on each fabric would be given, the part of an
|
// What a two-node shard on each fabric would be given, the part of an
|
||||||
|
|||||||
Reference in new issue
Block a user