vLLM's mp launch gave workers no rank and the leader's address as their own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank, --master-addr and --master-port, and the workers --headless, as vLLM documents it. NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one interface name for every node; the agent already worked out its own - the one holding its management address - and now uses it. On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts, which does not route. The agent finds the RoCE v2 entry holding an IPv4 address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none. A worker whose only process needs the leader - a Ray join - started without waiting for it, since the wait was honoured for sidecars alone; it found no head, exited and failed the replica. It now waits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
161 lines
5.2 KiB
Go
161 lines
5.2 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
)
|
|
|
|
// VLLMConfig locates the vLLM installation and picks how it is launched.
|
|
type VLLMConfig struct {
|
|
// Bin is the vllm executable.
|
|
Bin string `koanf:"bin"`
|
|
// LaunchMode selects the multi-node mechanism: "ray" or "mp".
|
|
//
|
|
// Which one works depends on the vLLM version, and the flags have changed
|
|
// more than once (plan §14). Both paths exist so a cluster can pick the one
|
|
// its version supports instead of waiting for a code change.
|
|
LaunchMode string `koanf:"launch_mode"`
|
|
// RayBin is used when LaunchMode is "ray".
|
|
RayBin string `koanf:"ray_bin"`
|
|
}
|
|
|
|
// VLLM builds vLLM launch commands.
|
|
type VLLM struct{ cfg VLLMConfig }
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*VLLM) Name() string { return "vllm" }
|
|
|
|
// HealthPath is vLLM's readiness endpoint.
|
|
func (*VLLM) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is vLLM's Prometheus endpoint.
|
|
func (*VLLM) MetricsPath() string { return "/metrics" }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
|
|
// back silently to a slower path, and a silent fallback is worse.
|
|
if q := quantizationOf(p.Args); q == "fp8" {
|
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ServesHTTP reports whether this rank answers the API.
|
|
//
|
|
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
|
|
// only the driver binds the port. Either way it is the leader alone.
|
|
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
|
|
|
// Commands returns the processes for one shard, in start order.
|
|
//
|
|
// Single node, or multi-node under multiprocessing: one server process. Under
|
|
// Ray: the leader starts a head and then the server, a worker starts only the
|
|
// join. A Ray worker running no vLLM process at all is correct — the cluster
|
|
// gives the leader its GPUs.
|
|
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
|
|
ray, useRay := v.rayCommand(s)
|
|
if useRay && !s.Leader() {
|
|
// A worker is the Ray join and nothing else. It blocks, so its exit is
|
|
// the shard's exit, which is what the supervisor needs.
|
|
return []Command{ray}, nil
|
|
}
|
|
|
|
server, err := v.serverCommand(s)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if useRay {
|
|
return []Command{ray, server}, nil
|
|
}
|
|
return []Command{server}, nil
|
|
}
|
|
|
|
// serverCommand produces the vLLM server process.
|
|
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
|
bin := v.cfg.Bin
|
|
if bin == "" {
|
|
bin = "vllm"
|
|
}
|
|
p := s.Profile
|
|
|
|
args := []string{"serve", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
"--tensor-parallel-size", strconv.Itoa(p.TP),
|
|
}
|
|
if p.PP > 1 {
|
|
args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--served-model-name", p.ServedModelName)
|
|
}
|
|
|
|
env := baseEnv(s)
|
|
|
|
if s.NNodes > 1 {
|
|
switch v.cfg.LaunchMode {
|
|
case "", "ray":
|
|
// Ray is started separately by the supervisor; vLLM then finds the
|
|
// cluster through the address in the environment.
|
|
env["VLLM_HOST_IP"] = s.LeaderAddr
|
|
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
|
args = append(args, "--distributed-executor-backend", "ray")
|
|
case "mp":
|
|
// vLLM's own multi-node launch: every node runs the same serve
|
|
// with its rank and the leader's address, and the workers run
|
|
// headless — no API server, only their GPUs. VLLM_HOST_IP is
|
|
// this node's address for vLLM to find itself, so it is left
|
|
// to vLLM; the leader's, which workers used to be given, made
|
|
// every worker announce itself as the leader.
|
|
args = append(args, "--distributed-executor-backend", "mp",
|
|
"--nnodes", strconv.Itoa(s.NNodes),
|
|
"--node-rank", strconv.Itoa(s.Rank),
|
|
"--master-addr", s.LeaderAddr,
|
|
"--master-port", strconv.Itoa(s.LeaderPort))
|
|
if !s.Leader() {
|
|
args = append(args, "--headless")
|
|
}
|
|
default:
|
|
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
|
|
}
|
|
}
|
|
|
|
// Profile arguments last, so an operator can override anything above.
|
|
args = append(args, p.Args...)
|
|
|
|
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
|
}
|
|
|
|
// rayCommand returns the command that starts or joins the Ray cluster, or
|
|
// ok=false when the launch mode does not use Ray.
|
|
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
|
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
|
|
return Command{}, false
|
|
}
|
|
bin := v.cfg.RayBin
|
|
if bin == "" {
|
|
bin = "ray"
|
|
}
|
|
|
|
args := []string{"start", "--block"}
|
|
if s.Leader() {
|
|
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
|
|
} else {
|
|
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
|
|
}
|
|
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
|
|
|
|
return Command{
|
|
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
|
// The head must be listening before anything else happens: a worker
|
|
// cannot join, and the leader's server cannot claim GPUs, until it is.
|
|
WaitForLeaderPort: true,
|
|
}, true
|
|
}
|