vLLM was installed by hand and named in each agent's configuration. The agent now installs it: a Python 3.12 environment built by uv, which is itself a GitHub release checked against its checksum, from PyPI resolved as it stood a day after that vLLM was published and installed with --require-hashes. The resolved list is kept in the environment. The environment is built relocatable under a scratch name and renamed into place once it runs, like an archive. The default is 0.30.0, whose flags were checked against the ones llmgw builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or newer, and 25 GB free while it installs; a node lacking one says which, and a node with no GPU skips it without being shown as a fault. gcc comes with it because Triton compiles its launcher at the first model load. SGLang installs the same way when the control plane names a version. llmgw doctor reports whether vLLM can be installed on the node. Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
182 lines
5.9 KiB
Go
182 lines
5.9 KiB
Go
package engines
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"sync/atomic"
|
|
)
|
|
|
|
// VLLMConfig locates the vLLM installation and picks how it is launched.
|
|
type VLLMConfig struct {
|
|
// Bin is the vllm executable.
|
|
Bin string `koanf:"bin"`
|
|
// LaunchMode selects the multi-node mechanism: "mp", the default, or
|
|
// "ray".
|
|
//
|
|
// mp is what vLLM documents today — every node runs serve with its rank,
|
|
// the workers headless — and needs nothing but vLLM. ray is the older way,
|
|
// kept for a vLLM too old for mp's multi-node flags; it needs Ray on every
|
|
// node and has not been checked against a real cluster.
|
|
LaunchMode string `koanf:"launch_mode"`
|
|
// RayBin is used when LaunchMode is "ray".
|
|
RayBin string `koanf:"ray_bin"`
|
|
}
|
|
|
|
// VLLM builds vLLM launch commands.
|
|
type VLLM struct {
|
|
cfg VLLMConfig
|
|
installed atomic.Pointer[string]
|
|
}
|
|
|
|
// UseInstalled points the engine at the environment the agent installed and
|
|
// checked. It takes precedence over the configuration, as llama.cpp's does:
|
|
// it is the one seen to run. An operator who wants their own sets the
|
|
// control plane's engines.vllm to "".
|
|
func (v *VLLM) UseInstalled(path string) { v.installed.Store(&path) }
|
|
|
|
// bin is the vllm to run: the installed one, else the configured one, else
|
|
// whatever PATH has.
|
|
func (v *VLLM) bin() string {
|
|
if p := v.installed.Load(); p != nil {
|
|
return *p
|
|
}
|
|
if v.cfg.Bin != "" {
|
|
return v.cfg.Bin
|
|
}
|
|
return "vllm"
|
|
}
|
|
|
|
// Name identifies the engine in the catalog.
|
|
func (*VLLM) Name() string { return "vllm" }
|
|
|
|
// HealthPath is vLLM's readiness endpoint.
|
|
func (*VLLM) HealthPath() string { return "/health" }
|
|
|
|
// MetricsPath is vLLM's Prometheus endpoint.
|
|
func (*VLLM) MetricsPath() string { return "/metrics" }
|
|
|
|
// Validate checks the profile against the GPUs it would run on.
|
|
func (*VLLM) Validate(p Profile, gpus []GPUInfo) error {
|
|
if err := validateCommon(p, gpus); err != nil {
|
|
return err
|
|
}
|
|
// FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls
|
|
// back silently to a slower path, and a silent fallback is worse.
|
|
if q := quantizationOf(p.Args); q == "fp8" {
|
|
if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ServesHTTP reports whether this rank answers the API.
|
|
//
|
|
// Under Ray a worker runs nothing but the Ray join, and under multiprocessing
|
|
// only the driver binds the port. Either way it is the leader alone.
|
|
func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() }
|
|
|
|
// Commands returns the processes for one shard, in start order.
|
|
//
|
|
// Single node, or multi-node under multiprocessing: one server process. Under
|
|
// Ray: the leader starts a head and then the server, a worker starts only the
|
|
// join. A Ray worker running no vLLM process at all is correct — the cluster
|
|
// gives the leader its GPUs.
|
|
func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
|
|
ray, useRay := v.rayCommand(s)
|
|
if useRay && !s.Leader() {
|
|
// A worker is the Ray join and nothing else. It blocks, so its exit is
|
|
// the shard's exit, which is what the supervisor needs.
|
|
return []Command{ray}, nil
|
|
}
|
|
|
|
server, err := v.serverCommand(s)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if useRay {
|
|
return []Command{ray, server}, nil
|
|
}
|
|
return []Command{server}, nil
|
|
}
|
|
|
|
// serverCommand produces the vLLM server process.
|
|
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
|
bin := v.bin()
|
|
p := s.Profile
|
|
|
|
args := []string{"serve", p.ModelPath,
|
|
"--host", "0.0.0.0",
|
|
"--port", strconv.Itoa(s.HTTPPort),
|
|
"--tensor-parallel-size", strconv.Itoa(p.TP),
|
|
}
|
|
if p.PP > 1 {
|
|
args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP))
|
|
}
|
|
if p.ServedModelName != "" {
|
|
args = append(args, "--served-model-name", p.ServedModelName)
|
|
}
|
|
|
|
env := baseEnv(s)
|
|
|
|
if s.NNodes > 1 {
|
|
switch v.cfg.LaunchMode {
|
|
case "ray":
|
|
// Ray is started separately by the supervisor; vLLM then finds the
|
|
// cluster through the address in the environment.
|
|
env["VLLM_HOST_IP"] = s.LeaderAddr
|
|
env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)
|
|
args = append(args, "--distributed-executor-backend", "ray")
|
|
case "", "mp":
|
|
// vLLM's own multi-node launch: every node runs the same serve
|
|
// with its rank and the leader's address, and the workers run
|
|
// headless — no API server, only their GPUs. VLLM_HOST_IP is
|
|
// this node's address for vLLM to find itself, so it is left
|
|
// to vLLM; the leader's, which workers used to be given, made
|
|
// every worker announce itself as the leader.
|
|
args = append(args, "--distributed-executor-backend", "mp",
|
|
"--nnodes", strconv.Itoa(s.NNodes),
|
|
"--node-rank", strconv.Itoa(s.Rank),
|
|
"--master-addr", s.LeaderAddr,
|
|
"--master-port", strconv.Itoa(s.LeaderPort))
|
|
if !s.Leader() {
|
|
args = append(args, "--headless")
|
|
}
|
|
default:
|
|
return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode)
|
|
}
|
|
}
|
|
|
|
// Profile arguments last, so an operator can override anything above.
|
|
args = append(args, p.Args...)
|
|
|
|
return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
|
|
}
|
|
|
|
// rayCommand returns the command that starts or joins the Ray cluster, or
|
|
// ok=false when the launch mode does not use Ray.
|
|
func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
|
|
if s.NNodes <= 1 || v.cfg.LaunchMode != "ray" {
|
|
return Command{}, false
|
|
}
|
|
bin := v.cfg.RayBin
|
|
if bin == "" {
|
|
bin = "ray"
|
|
}
|
|
|
|
args := []string{"start", "--block"}
|
|
if s.Leader() {
|
|
args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
|
|
} else {
|
|
args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
|
|
}
|
|
args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices)))
|
|
|
|
return Command{
|
|
Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
|
|
// The head must be listening before anything else happens: a worker
|
|
// cannot join, and the leader's server cannot claim GPUs, until it is.
|
|
WaitForLeaderPort: true,
|
|
}, true
|
|
}
|