package engines import ( "fmt" "strconv" ) // VLLMConfig locates the vLLM installation and picks how it is launched. type VLLMConfig struct { // Bin is the vllm executable. Bin string `koanf:"bin"` // LaunchMode selects the multi-node mechanism: "ray" or "mp". // // Which one works depends on the vLLM version, and the flags have changed // more than once (plan §14). Both paths exist so a cluster can pick the one // its version supports instead of waiting for a code change. LaunchMode string `koanf:"launch_mode"` // RayBin is used when LaunchMode is "ray". RayBin string `koanf:"ray_bin"` } // VLLM builds vLLM launch commands. type VLLM struct{ cfg VLLMConfig } // Name identifies the engine in the catalog. func (*VLLM) Name() string { return "vllm" } // HealthPath is vLLM's readiness endpoint. func (*VLLM) HealthPath() string { return "/health" } // MetricsPath is vLLM's Prometheus endpoint. func (*VLLM) MetricsPath() string { return "/metrics" } // Validate checks the profile against the GPUs it would run on. func (*VLLM) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } // FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls // back silently to a slower path, and a silent fallback is worse. if q := quantizationOf(p.Args); q == "fp8" { if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil { return err } } return nil } // ServesHTTP reports whether this rank answers the API. // // Under Ray a worker runs nothing but the Ray join, and under multiprocessing // only the driver binds the port. Either way it is the leader alone. func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() } // Commands returns the processes for one shard, in start order. // // Single node, or multi-node under multiprocessing: one server process. Under // Ray: the leader starts a head and then the server, a worker starts only the // join. A Ray worker running no vLLM process at all is correct — the cluster // gives the leader its GPUs. func (v *VLLM) Commands(s ShardSpec) ([]Command, error) { ray, useRay := v.rayCommand(s) if useRay && !s.Leader() { // A worker is the Ray join and nothing else. It blocks, so its exit is // the shard's exit, which is what the supervisor needs. return []Command{ray}, nil } server, err := v.serverCommand(s) if err != nil { return nil, err } if useRay { return []Command{ray, server}, nil } return []Command{server}, nil } // serverCommand produces the vLLM server process. func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { bin := v.cfg.Bin if bin == "" { bin = "vllm" } p := s.Profile args := []string{"serve", p.ModelPath, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), "--tensor-parallel-size", strconv.Itoa(p.TP), } if p.PP > 1 { args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP)) } if p.ServedModelName != "" { args = append(args, "--served-model-name", p.ServedModelName) } env := baseEnv(s) if s.NNodes > 1 { switch v.cfg.LaunchMode { case "", "ray": // Ray is started separately by the supervisor; vLLM then finds the // cluster through the address in the environment. env["VLLM_HOST_IP"] = s.LeaderAddr env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort) args = append(args, "--distributed-executor-backend", "ray") case "mp": env["VLLM_HOST_IP"] = s.LeaderAddr env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort) args = append(args, "--distributed-executor-backend", "mp") default: return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode) } } // Profile arguments last, so an operator can override anything above. args = append(args, p.Args...) return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil } // rayCommand returns the command that starts or joins the Ray cluster, or // ok=false when the launch mode does not use Ray. func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) { if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") { return Command{}, false } bin := v.cfg.RayBin if bin == "" { bin = "ray" } args := []string{"start", "--block"} if s.Leader() { args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort)) } else { args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)) } args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices))) return Command{ Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), // The head must be listening before anything else happens: a worker // cannot join, and the leader's server cannot claim GPUs, until it is. WaitForLeaderPort: true, }, true }