package engines import ( "fmt" "strconv" ) // VLLMConfig locates the vLLM installation and picks how it is launched. type VLLMConfig struct { // Bin is the vllm executable. Bin string `koanf:"bin"` // LaunchMode selects the multi-node mechanism: "ray" or "mp". // // Which one works depends on the vLLM version, and the flags have changed // more than once (plan ยง14). Both paths exist so a cluster can pick the one // its version supports instead of waiting for a code change. LaunchMode string `koanf:"launch_mode"` // RayBin is used when LaunchMode is "ray". RayBin string `koanf:"ray_bin"` } // VLLM builds vLLM launch commands. type VLLM struct{ cfg VLLMConfig } // Name identifies the engine in the catalog. func (*VLLM) Name() string { return "vllm" } // HealthPath is vLLM's readiness endpoint. func (*VLLM) HealthPath() string { return "/health" } // MetricsPath is vLLM's Prometheus endpoint. func (*VLLM) MetricsPath() string { return "/metrics" } // Validate checks the profile against the GPUs it would run on. func (*VLLM) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } // FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls // back silently to a slower path, and a silent fallback is worse. if q := quantizationOf(p.Args); q == "fp8" { if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil { return err } } return nil } // BuildCommand produces the launch command for one shard. func (v *VLLM) BuildCommand(s ShardSpec) (Command, error) { bin := v.cfg.Bin if bin == "" { bin = "vllm" } p := s.Profile args := []string{"serve", p.ModelPath, "--host", "0.0.0.0", "--port", strconv.Itoa(s.HTTPPort), "--tensor-parallel-size", strconv.Itoa(p.TP), } if p.PP > 1 { args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP)) } if p.ServedModelName != "" { args = append(args, "--served-model-name", p.ServedModelName) } env := baseEnv(s) if s.NNodes > 1 { switch v.cfg.LaunchMode { case "", "ray": // Ray is started separately by the supervisor; vLLM then finds the // cluster through the address in the environment. env["VLLM_HOST_IP"] = s.LeaderAddr env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort) args = append(args, "--distributed-executor-backend", "ray") case "mp": env["VLLM_HOST_IP"] = s.LeaderAddr env["VLLM_PORT"] = strconv.Itoa(s.LeaderPort) args = append(args, "--distributed-executor-backend", "mp") default: return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode) } } // Profile arguments last, so an operator can override anything above. args = append(args, p.Args...) return Command{Path: bin, Args: args, Env: materializeEnv(env)}, nil } // RayCommand returns the command that joins this node to the Ray cluster, or // ok=false when the launch mode does not use Ray. // // It is separate from BuildCommand because it has to run first and finish // before vLLM starts: the supervisor sequences the two. func (v *VLLM) RayCommand(s ShardSpec) (Command, bool) { if s.NNodes <= 1 || (v.cfg.LaunchMode != "" && v.cfg.LaunchMode != "ray") { return Command{}, false } bin := v.cfg.RayBin if bin == "" { bin = "ray" } args := []string{"start", "--block"} if s.Leader() { args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort)) } else { args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)) } args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices))) return Command{Path: bin, Args: args, Env: materializeEnv(baseEnv(s))}, true }