Install vLLM on every GPU node that can run it, as llama.cpp is

vLLM was installed by hand and named in each agent's configuration. The
agent now installs it: a Python 3.12 environment built by uv, which is
itself a GitHub release checked against its checksum, from PyPI resolved as
it stood a day after that vLLM was published and installed with
--require-hashes. The resolved list is kept in the environment. The
environment is built relocatable under a scratch name and renamed into place
once it runs, like an archive.

The default is 0.30.0, whose flags were checked against the ones llmgw
builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or
newer, and 25 GB free while it installs; a node lacking one says which, and a
node with no GPU skips it without being shown as a fault. gcc comes with it
because Triton compiles its launcher at the first model load. SGLang
installs the same way when the control plane names a version. llmgw doctor
reports whether vLLM can be installed on the node.

Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for
CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 13:15:54 +02:00
1 parent e751c89937
commit 4c5a2fd6da
21 files changed
+853 -100

No files matched your search

+2 -10
View File
@@ -202,11 +202,7 @@ func (g *SGLang) Configured() bool { return g.cfg.Python != "" }
// Probe reports whether vLLM is installed.
func (v *VLLM) Probe(context.Context) (string, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
return installedBinary(bin)
return installedBinary(v.bin())
}
// Probe reports the SGLang version.
@@ -215,10 +211,6 @@ func (v *VLLM) Probe(context.Context) (string, error) {
// asking the shell for "sglang" finds nothing on a machine that can run it
// perfectly well.
func (g *SGLang) Probe(ctx context.Context) (string, error) {
python := g.cfg.Python
if python == "" {
python = "python3"
}
return versionOfModule(ctx, python, "-c",
return versionOfModule(ctx, g.python(), "-c",
"import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))")
}
+24 -6
View File
@@ -1,6 +1,9 @@
package engines
import "strconv"
import (
"strconv"
"sync/atomic"
)
// SGLangConfig locates the SGLang installation.
type SGLangConfig struct {
@@ -11,7 +14,25 @@ type SGLangConfig struct {
}
// SGLang builds SGLang launch commands.
type SGLang struct{ cfg SGLangConfig }
type SGLang struct {
cfg SGLangConfig
installed atomic.Pointer[string]
}
// UseInstalled points the engine at the interpreter of the environment the
// agent installed and checked, ahead of the configuration.
func (g *SGLang) UseInstalled(path string) { g.installed.Store(&path) }
// python is the interpreter to launch SGLang with.
func (g *SGLang) python() string {
if p := g.installed.Load(); p != nil {
return *p
}
if g.cfg.Python != "" {
return g.cfg.Python
}
return "python3"
}
// Name identifies the engine in the catalog.
func (*SGLang) Name() string { return "sglang" }
@@ -55,10 +76,7 @@ func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
}
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
python := g.cfg.Python
if python == "" {
python = "python3"
}
python := g.python()
p := s.Profile
args := []string{"-m", "sglang.launch_server",
+24 -5
View File
@@ -3,6 +3,7 @@ package engines
import (
"fmt"
"strconv"
"sync/atomic"
)
// VLLMConfig locates the vLLM installation and picks how it is launched.
@@ -22,7 +23,28 @@ type VLLMConfig struct {
}
// VLLM builds vLLM launch commands.
type VLLM struct{ cfg VLLMConfig }
type VLLM struct {
cfg VLLMConfig
installed atomic.Pointer[string]
}
// UseInstalled points the engine at the environment the agent installed and
// checked. It takes precedence over the configuration, as llama.cpp's does:
// it is the one seen to run. An operator who wants their own sets the
// control plane's engines.vllm to "".
func (v *VLLM) UseInstalled(path string) { v.installed.Store(&path) }
// bin is the vllm to run: the installed one, else the configured one, else
// whatever PATH has.
func (v *VLLM) bin() string {
if p := v.installed.Load(); p != nil {
return *p
}
if v.cfg.Bin != "" {
return v.cfg.Bin
}
return "vllm"
}
// Name identifies the engine in the catalog.
func (*VLLM) Name() string { return "vllm" }
@@ -80,10 +102,7 @@ func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
// serverCommand produces the vLLM server process.
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
bin := v.bin()
p := s.Profile
args := []string{"serve", p.ModelPath,