Files
llmgw/internal/agent/engines/vllm.go
T
ilkerandClaude Opus 5 1d1a6aebb0
CI / test (push) Failing after 53s
Phase 4: make a multi-node replica able to start
Placement and environment generation had landed early, which made multi-node
look further along than it was. A replica spanning nodes could not come up at
all, and all three reasons came down to assumptions that only hold on one node.

Not every rank serves HTTP. The agent health-checked every shard at its own
port, but only the leader answers; workers timed out, reported Failed and took
the replica with them. Engine.ServesHTTP now says which ranks serve, and a
worker is judged by staying alive. That is deliberately weaker evidence: the
replica's real readiness is the leader answering, which only happens once every
worker has joined.

A shard is not always one process. Engine.Commands returns a sequence — every
command but the last is a sidecar kept alive for the shard's lifetime and torn
down with it, and the last one's exit is the shard's. vLLM under Ray is why: the
leader runs a Ray head and then the server against it, while a worker runs only
the join and no vLLM process at all. RayCommand had been written and tested but
never called, so launch_mode: ray would have run vllm serve against a cluster
that did not exist. Sidecars are killed with the shard, because a Ray head left
behind holds the port the next replica needs.

Start order was backwards. Workers went first, on the theory that the head
should wait for them. Both engines put the rendezvous point on rank 0, so a
worker started first has nothing to connect to. vLLM's server blocks until the
Ray cluster has the GPUs it needs, so the leader can start immediately and waits
on its own. A command can now also require that rendezvous point to be listening
before the next starts, bounded by a timeout so a head that never appears fails
the shard instead of hanging it.

Stop was also killing only the engine, leaving sidecars behind.

None of this is verified against real engines — it is tested with stub
processes. The Ray sequencing, and which launch mode a given vLLM version wants,
can only be confirmed on hardware.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 06:20:03 +02:00

150 lines
4.7 KiB
Go

package engines
import (
"fmt"
"strconv"
)
// VLLMConfig locates the vLLM installation and picks how it is launched.
type VLLMConfig struct {
// Bin is the vllm executable.
Bin string `koanf:"bin"`
// LaunchMode selects the multi-node mechanism: "ray" or "mp".
//
// Which one works depends on the vLLM version, and the flags have changed
// more than once (plan §14). Both paths exist so a cluster can pick the one
// its version supports instead of waiting for a code change.
LaunchMode string `koanf:"launch_mode"`
// RayBin is used when LaunchMode is "ray".
RayBin string `koanf:"ray_bin"`
}
// VLLM builds vLLM launch commands.
type VLLM struct{ cfg VLLMConfig }
// Name identifies the engine in the catalog.
func (*VLLM) Name() string { return "vllm" }
// HealthPath is vLLM's readiness endpoint.
func (*VLLM) HealthPath() string { return "/health" }
// MetricsPath is vLLM's Prometheus endpoint.
func (*VLLM) MetricsPath() string { return "/metrics" }
// Validate checks the profile against the GPUs it would run on.
func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
if err := validateCommon(p, gpus); err != nil {
return err
}
// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
// back silently to a slower path, and a silent fallback is worse.
if q := quantizationOf(p.Args); q == "fp8" {
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
return err
}
}
return nil
}
// ServesHTTP reports whether this rank answers the API.
//
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
// only the driver binds the port. Either way it is the leader alone.
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
// Commands returns the processes for one shard, in start order.
//
// Single node, or multi-node under multiprocessing: one server process. Under
// Ray: the leader starts a head and then the server, a worker starts only the
// join. A Ray worker running no vLLM process at all is correct — the cluster
// gives the leader its GPUs.
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
ray, useRay := v.rayCommand(s)
if useRay && !s.Leader() {
// A worker is the Ray join and nothing else. It blocks, so its exit is
// the shard's exit, which is what the supervisor needs.
return []Command{ray}, nil
}
server, err := v.serverCommand(s)
if err != nil {
return nil, err
}
if useRay {
return []Command{ray, server}, nil
}
return []Command{server}, nil
}
// serverCommand produces the vLLM server process.
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
p := s.Profile
args := []string{"serve", p.ModelPath,
"--host", "0.0.0.0",
"--port", strconv.Itoa(s.HTTPPort),
"--tensor-parallel-size", strconv.Itoa(p.TP),
}
if p.PP > 1 {
args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
}
if p.ServedModelName != "" {
args = append(args, "--served-model-name", p.ServedModelName)
}
env := baseEnv(s)
if s.NNodes > 1 {
switch v.cfg.LaunchMode {
case "", "ray":
// Ray is started separately by the supervisor; vLLM then finds the
// cluster through the address in the environment.
env["VLLM_HOST_IP"] = s.LeaderAddr
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
args = append(args, "--distributed-executor-backend", "ray")
case "mp":
env["VLLM_HOST_IP"] = s.LeaderAddr
env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort)
args = append(args, "--distributed-executor-backend", "mp")
default:
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
}
}
// Profile arguments last, so an operator can override anything above.
args = append(args, p.Args...)
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
}
// rayCommand returns the command that starts or joins the Ray cluster, or
// ok=false when the launch mode does not use Ray.
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") {
return Command{}, false
}
bin := v.cfg.RayBin
if bin == "" {
bin = "ray"
}
args := []string{"start", "--block"}
if s.Leader() {
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
} else {
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
}
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
return Command{
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
// The head must be listening before anything else happens: a worker
// cannot join, and the leader's server cannot claim GPUs, until it is.
WaitForLeaderPort: true,
}, true
}