Files
llmgw/internal/agent/engines/vllm.go
T
Ilker ManapandClaude Opus 5.5 9ee21dc814 Fix what a multi-node launch would have got wrong on real hardware
vLLM's mp launch gave workers no rank and the leader's address as their
own VLLM_HOST_IP; every node now runs serve with --nnodes, --node-rank,
--master-addr and --master-port, and the workers --headless, as vLLM
documents it.

NCCL_SOCKET_IFNAME and GLOO_SOCKET_IFNAME were the control plane's one
interface name for every node; the agent already worked out its own -
the one holding its management address - and now uses it.

On RoCE, NCCL's default GID index is a v1 link-local entry on most hosts,
which does not route. The agent finds the RoCE v2 entry holding an IPv4
address and passes it as NCCL_IB_GID_INDEX; InfiniBand gets none.

A worker whose only process needs the leader - a Ray join - started
without waiting for it, since the wait was honoured for sidecars alone;
it found no head, exited and failed the replica. It now waits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 00:17:31 +02:00

161 lines
5.2 KiB
Go

package engines
import (
"fmt"
"strconv"
)
// VLLMConfig locates the vLLM installation and picks how it is launched.
type VLLMConfig struct {
// Bin is the vllm executable.
Bin string `koanf:"bin"`
// LaunchMode selects the multi-node mechanism: "ray" or "mp".
//
// Which one works depends on the vLLM version, and the flags have changed
// more than once (plan §14). Both paths exist so a cluster can pick the one
// its version supports instead of waiting for a code change.
LaunchMode string `koanf:"launch_mode"`
// RayBin is used when LaunchMode is "ray".
RayBin string `koanf:"ray_bin"`
}
// VLLM builds vLLM launch commands.
type VLLM struct{ cfg VLLMConfig }
// Name identifies the engine in the catalog.
func (*VLLM) Name() string { return "vllm" }
// HealthPath is vLLM's readiness endpoint.
func (*VLLM) HealthPath() string { return "/health" }
// MetricsPath is vLLM's Prometheus endpoint.
func (*VLLM) MetricsPath() string { return "/metrics" }
// Validate checks the profile against the GPUs it would run on.
func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
if err := validateCommon(p, gpus); err != nil {
return err
}
// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
// back silently to a slower path, and a silent fallback is worse.
if q := quantizationOf(p.Args); q == "fp8" {
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
return err
}
}
return nil
}
// ServesHTTP reports whether this rank answers the API.
//
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
// only the driver binds the port. Either way it is the leader alone.
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
// Commands returns the processes for one shard, in start order.
//
// Single node, or multi-node under multiprocessing: one server process. Under
// Ray: the leader starts a head and then the server, a worker starts only the
// join. A Ray worker running no vLLM process at all is correct — the cluster
// gives the leader its GPUs.
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
ray, useRay := v.rayCommand(s)
if useRay && !s.Leader() {
// A worker is the Ray join and nothing else. It blocks, so its exit is
// the shard's exit, which is what the supervisor needs.
return []Command{ray}, nil
}
server, err := v.serverCommand(s)
if err != nil {
return nil, err
}
if useRay {
return []Command{ray, server}, nil
}
return []Command{server}, nil
}
// serverCommand produces the vLLM server process.
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
p := s.Profile
args := []string{"serve", p.ModelPath,
"--host", "0.0.0.0",
"--port", strconv.Itoa(s.HTTPPort),
"--tensor-parallel-size", strconv.Itoa(p.TP),
}
if p.PP > 1 {
args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
}
if p.ServedModelName != "" {
args = append(args, "--served-model-name", p.ServedModelName)
}
env := baseEnv(s)
if s.NNodes > 1 {
switch v.cfg.LaunchMode {
case "", "ray":
// Ray is started separately by the supervisor; vLLM then finds the
// cluster through the address in the environment.
env["VLLM_HOST_IP"] = s.LeaderAddr
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
args = append(args, "--distributed-executor-backend", "ray")
case "mp":
// vLLM's own multi-node launch: every node runs the same serve
// with its rank and the leader's address, and the workers run
// headless — no API server, only their GPUs. VLLM_HOST_IP is
// this node's address for vLLM to find itself, so it is left
// to vLLM; the leader's, which workers used to be given, made
// every worker announce itself as the leader.
args = append(args, "--distributed-executor-backend", "mp",
"--nnodes", strconv.Itoa(s.NNodes),
"--node-rank", strconv.Itoa(s.Rank),
"--master-addr", s.LeaderAddr,
"--master-port", strconv.Itoa(s.LeaderPort))
if !s.Leader() {
args = append(args, "--headless")
}
default:
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
}
}
// Profile arguments last, so an operator can override anything above.
args = append(args, p.Args...)
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
}
// rayCommand returns the command that starts or joins the Ray cluster, or
// ok=false when the launch mode does not use Ray.
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
return Command{}, false
}
bin := v.cfg.RayBin
if bin == "" {
bin = "ray"
}
args := []string{"start", "--block"}
if s.Leader() {
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
} else {
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
}
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
return Command{
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
// The head must be listening before anything else happens: a worker
// cannot join, and the leader's server cannot claim GPUs, until it is.
WaitForLeaderPort: true,
}, true
}