package engines import ( "fmt" "maps" "strconv" "sync/atomic" "gitea.mynodes.xyz/ilker/llmgw/internal/kvevents" ) // VLLMConfig locates the vLLM installation and picks how it is launched. type VLLMConfig struct { // Bin is the vllm executable. Bin string `koanf:"bin"` // LaunchMode selects the multi-node mechanism: "mp", the default, or // "ray". // // mp is what vLLM documents today — every node runs serve with its rank, // the workers headless — and needs nothing but vLLM. ray is the older way, // kept for a vLLM too old for mp's multi-node flags; it needs Ray on every // node and has not been checked against a real cluster. LaunchMode string `koanf:"launch_mode"` // RayBin is used when LaunchMode is "ray". RayBin string `koanf:"ray_bin"` } // VLLM builds vLLM launch commands. type VLLM struct { cfg VLLMConfig installed atomic.Pointer[string] } // UseInstalled points the engine at the environment the agent installed and // checked. It takes precedence over the configuration, as llama.cpp's does: // it is the one seen to run. An operator who wants their own sets the // control plane's engines.vllm to "". func (v *VLLM) UseInstalled(path string) { v.installed.Store(&path) } // bin is the vllm to run: the installed one, else the configured one, else // whatever PATH has. func (v *VLLM) bin() string { if p := v.installed.Load(); p != nil { return *p } if v.cfg.Bin != "" { return v.cfg.Bin } return "vllm" } // Name identifies the engine in the catalog. func (*VLLM) Name() string { return "vllm" } // HealthPath is vLLM's readiness endpoint. func (*VLLM) HealthPath() string { return "/health" } // MetricsPath is vLLM's Prometheus endpoint. func (*VLLM) MetricsPath() string { return "/metrics" } // Validate checks the profile against the GPUs it would run on. func (*VLLM) Validate(p Profile, gpus []GPUInfo) error { if err := validateCommon(p, gpus); err != nil { return err } // FP8 needs Hopper or Ada; on anything older vLLM either refuses or falls // back silently to a slower path, and a silent fallback is worse. if q := quantizationOf(p.Args); q == "fp8" { if err := requireComputeCapability(p, gpus, 8.9, "fp8 quantization"); err != nil { return err } } return nil } // ServesHTTP reports whether this rank answers the API. // // Under Ray a worker runs nothing but the Ray join, and under multiprocessing // only the driver binds the port. Either way it is the leader alone. func (*VLLM) ServesHTTP(s ShardSpec) bool { return s.Leader() } // Commands returns the processes for one shard, in start order. // // Single node, or multi-node under multiprocessing: one server process. Under // Ray: the leader starts a head and then the server, a worker starts only the // join. A Ray worker running no vLLM process at all is correct — the cluster // gives the leader its GPUs. func (v *VLLM) Commands(s ShardSpec) ([]Command, error) { ray, useRay := v.rayCommand(s) if useRay && !s.Leader() { // A worker is the Ray join and nothing else. It blocks, so its exit is // the shard's exit, which is what the supervisor needs. return []Command{ray}, nil } server, err := v.serverCommand(s) if err != nil { return nil, err } if useRay { return []Command{ray, server}, nil } return []Command{server}, nil } // serverCommand produces the vLLM server process. func (v *VLLM) serverCommand(s ShardSpec) (Command, error) { bin := v.bin() p := s.Profile args := []string{"serve", p.ModelPath, "--host", s.Host(), "--port", strconv.Itoa(s.HTTPPort), "--tensor-parallel-size", strconv.Itoa(p.TP), } if p.PP > 1 { args = append(args, "--pipeline-parallel-size", strconv.Itoa(p.PP)) } if p.ServedModelName != "" { args = append(args, "--served-model-name", p.ServedModelName) } env := baseEnv(s) if s.NNodes > 1 { switch v.cfg.LaunchMode { case "ray": // Ray is started separately by the supervisor; vLLM then finds the // cluster through the address in the environment. env["VLLM_HOST_IP"] = s.LeaderAddr env["RAY_ADDRESS"] = fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort) args = append(args, "--distributed-executor-backend", "ray") case "", "mp": // vLLM's own multi-node launch: every node runs the same serve // with its rank and the leader's address, and the workers run // headless — no API server, only their GPUs. VLLM_HOST_IP is // this node's address for vLLM to find itself, so it is left // to vLLM; the leader's, which workers used to be given, made // every worker announce itself as the leader. args = append(args, "--distributed-executor-backend", "mp", "--nnodes", strconv.Itoa(s.NNodes), "--node-rank", strconv.Itoa(s.Rank), "--master-addr", s.LeaderAddr, "--master-port", strconv.Itoa(s.LeaderPort)) if !s.Leader() { args = append(args, "--headless") } default: return Command{}, fmt.Errorf("vllm: unknown launch_mode %q (want \"ray\" or \"mp\")", v.cfg.LaunchMode) } } // Adapters loaded at start are registered under their names, and a // request names one as its model. Adapters loaded on demand start // unregistered: the router loads each when a request first wants it // here, by the link the agent made (RuntimeLoRADir). --max-loras is // vLLM's default of one at start — raise it in the profile's arguments // to mix more adapters in a batch — and the cap on demand, which // WithRuntimeLoRA put in the arguments. if len(p.Adapters) > 0 { args = append(args, "--enable-lora") if !p.RuntimeLoRA() { args = append(args, loraModules(p.Adapters)...) } } // The KV cache events, for the router's precise index: published to the // agent on loopback, hashed the way the router hashes. Only the leader // has a scheduler to publish from. if p.KVEvents && s.KVEventsPort > 0 && s.Leader() { args = append(args, kvevents.VLLMArgs(fmt.Sprintf("tcp://127.0.0.1:%d", s.KVEventsPort))...) maps.Copy(env, kvevents.VLLMEnv()) } // Profile arguments last, so an operator can override anything above. args = append(args, p.Args...) if s.APIKey != "" { env["VLLM_API_KEY"] = s.APIKey } return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil } // rayCommand returns the command that starts or joins the Ray cluster, or // ok=false when the launch mode does not use Ray. func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) { if s.NNodes <= 1 || v.cfg.LaunchMode != "ray" { return Command{}, false } bin := v.cfg.RayBin if bin == "" { bin = "ray" } args := []string{"start", "--block"} if s.Leader() { // Ray's head has no authentication: whoever reaches its port can run // code on the node. Its dashboard, and the job API behind it, is the // easiest way in, and nothing here uses it. args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort), "--include-dashboard=false") } else { args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort)) } args = append(args, "--num-gpus", strconv.Itoa(len(s.GPUIndices))) return Command{ Name: "ray", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), // The head must be listening before anything else happens: a worker // cannot join, and the leader's server cannot claim GPUs, until it is. WaitForLeaderPort: true, }, true }