Install vLLM on every GPU node that can run it, as llama.cpp is

vLLM was installed by hand and named in each agent's configuration. The
agent now installs it: a Python 3.12 environment built by uv, which is
itself a GitHub release checked against its checksum, from PyPI resolved as
it stood a day after that vLLM was published and installed with
--require-hashes. The resolved list is kept in the environment. The
environment is built relocatable under a scratch name and renamed into place
once it runs, like an archive.

The default is 0.30.0, whose flags were checked against the ones llmgw
builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or
newer, and 25 GB free while it installs; a node lacking one says which, and a
node with no GPU skips it without being shown as a fault. gcc comes with it
because Triton compiles its launcher at the first model load. SGLang
installs the same way when the control plane names a version. llmgw doctor
reports whether vLLM can be installed on the node.

Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for
CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 13:15:54 +02:00
1 parent e751c89937
commit 4c5a2fd6da
21 files changed
+853 -100

No files matched your search

+1
View File
@@ -798,6 +798,7 @@ func (a *Agent) scan(ctx context.Context) error {
opts.ModelDirs = a.cfg.ModelDirs
inv, err := discovery.Discover(ctx, opts, a.log)
host := discovery.DiscoverHost()
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
a.mu.Lock()
a.inv = inv
a.prov.Host = host
+20
View File
@@ -4,6 +4,7 @@ import (
"bufio"
"os"
"runtime"
"strconv"
"strings"
)
@@ -26,6 +27,9 @@ type Host struct {
CPUFlags []string `json:"cpu_flags,omitempty"`
// NVIDIADriver is the loaded driver's version, empty when there is none.
NVIDIADriver string `json:"nvidia_driver,omitempty"`
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
// is not known. Not read here but from the GPU inventory, by the agent.
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
}
// Family is the packaging family: "deb", "rpm", or "" when neither is known.
@@ -152,3 +156,19 @@ func nvidiaDriverFrom(path string) string {
}
return ""
}
// MinComputeCapability is the oldest GPU's compute capability, 0 when none
// is known: an engine built for newer GPUs is refused by the oldest.
func MinComputeCapability(gpus []GPU) float64 {
low := 0.0
for _, g := range gpus {
cc, err := strconv.ParseFloat(strings.TrimSpace(g.ComputeCapability), 64)
if err != nil || cc <= 0 {
continue
}
if low == 0 || cc < low {
low = cc
}
}
return low
}
+2 -10
View File
@@ -202,11 +202,7 @@ func (g *SGLang) Configured() bool { return g.cfg.Python != "" }
// Probe reports whether vLLM is installed.
func (v *VLLM) Probe(context.Context) (string, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
return installedBinary(bin)
return installedBinary(v.bin())
}
// Probe reports the SGLang version.
@@ -215,10 +211,6 @@ func (v *VLLM) Probe(context.Context) (string, error) {
// asking the shell for "sglang" finds nothing on a machine that can run it
// perfectly well.
func (g *SGLang) Probe(ctx context.Context) (string, error) {
python := g.cfg.Python
if python == "" {
python = "python3"
}
return versionOfModule(ctx, python, "-c",
return versionOfModule(ctx, g.python(), "-c",
"import sglang, sys; sys.stdout.write(getattr(sglang, '__version__', ''))")
}
+24 -6
View File
@@ -1,6 +1,9 @@
package engines
import "strconv"
import (
"strconv"
"sync/atomic"
)
// SGLangConfig locates the SGLang installation.
type SGLangConfig struct {
@@ -11,7 +14,25 @@ type SGLangConfig struct {
}
// SGLang builds SGLang launch commands.
type SGLang struct{ cfg SGLangConfig }
type SGLang struct {
cfg SGLangConfig
installed atomic.Pointer[string]
}
// UseInstalled points the engine at the interpreter of the environment the
// agent installed and checked, ahead of the configuration.
func (g *SGLang) UseInstalled(path string) { g.installed.Store(&path) }
// python is the interpreter to launch SGLang with.
func (g *SGLang) python() string {
if p := g.installed.Load(); p != nil {
return *p
}
if g.cfg.Python != "" {
return g.cfg.Python
}
return "python3"
}
// Name identifies the engine in the catalog.
func (*SGLang) Name() string { return "sglang" }
@@ -55,10 +76,7 @@ func (g *SGLang) Commands(s ShardSpec) ([]Command, error) {
}
func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
python := g.cfg.Python
if python == "" {
python = "python3"
}
python := g.python()
p := s.Profile
args := []string{"-m", "sglang.launch_server",
+24 -5
View File
@@ -3,6 +3,7 @@ package engines
import (
"fmt"
"strconv"
"sync/atomic"
)
// VLLMConfig locates the vLLM installation and picks how it is launched.
@@ -22,7 +23,28 @@ type VLLMConfig struct {
}
// VLLM builds vLLM launch commands.
type VLLM struct{ cfg VLLMConfig }
type VLLM struct {
cfg VLLMConfig
installed atomic.Pointer[string]
}
// UseInstalled points the engine at the environment the agent installed and
// checked. It takes precedence over the configuration, as llama.cpp's does:
// it is the one seen to run. An operator who wants their own sets the
// control plane's engines.vllm to "".
func (v *VLLM) UseInstalled(path string) { v.installed.Store(&path) }
// bin is the vllm to run: the installed one, else the configured one, else
// whatever PATH has.
func (v *VLLM) bin() string {
if p := v.installed.Load(); p != nil {
return *p
}
if v.cfg.Bin != "" {
return v.cfg.Bin
}
return "vllm"
}
// Name identifies the engine in the catalog.
func (*VLLM) Name() string { return "vllm" }
@@ -80,10 +102,7 @@ func (v *VLLM) Commands(s ShardSpec) ([]Command, error) {
// serverCommand produces the vLLM server process.
func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
bin := v.cfg.Bin
if bin == "" {
bin = "vllm"
}
bin := v.bin()
p := s.Profile
args := []string{"serve", p.ModelPath,
+64 -16
View File
@@ -34,6 +34,9 @@ const (
Ready = "ready"
Failed = "failed"
Unsupported = "unsupported"
// Skipped is an engine this node is not meant to run, such as vLLM on a
// machine with no GPU. Nothing is wrong, and nothing is shown as wrong.
Skipped = "skipped"
)
// Status is where one engine's installation stands.
@@ -62,9 +65,16 @@ type Installer struct {
// Run executes a command; a seam for tests. Nil runs it for real.
Run func(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
// APIBase and DownloadBase default to GitHub; tests point them at a fake.
// FreeSpace reports the gigabytes free under a directory; a seam for
// tests. Nil asks the filesystem.
FreeSpace func(dir string) (float64, error)
// RunStdin is Run with input; a seam for tests. Nil runs it for real.
RunStdin func(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error)
// APIBase and DownloadBase default to GitHub, PyPIBase to PyPI; tests
// point them at a fake.
APIBase string
DownloadBase string
PyPIBase string
}
// DefaultRoot is where engines are installed when nothing says otherwise.
@@ -75,12 +85,16 @@ const DefaultRoot = "/opt/llmgw/engines"
const marker = ".llmgw-installed"
type markerFile struct {
Engine string `json:"engine"`
Version string `json:"version"`
Variant string `json:"variant"`
Binary string `json:"binary"`
SHA256 map[string]string `json:"sha256"`
Installed time.Time `json:"installed"`
Engine string `json:"engine"`
Version string `json:"version"`
Variant string `json:"variant"`
Binary string `json:"binary"`
SHA256 map[string]string `json:"sha256,omitempty"`
// For a Python engine: what was asked for, PyPI as of when, by which uv.
Requirements []string `json:"requirements,omitempty"`
ExcludeNewer string `json:"exclude_newer,omitempty"`
UV string `json:"uv,omitempty"`
Installed time.Time `json:"installed"`
}
// Converge brings every wanted engine to ready, or says why it cannot.
@@ -110,6 +124,10 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
build, err := recipe.Choose(w.Version, in.Host)
if err != nil {
st.State, st.Detail = Unsupported, err.Error()
var nh NotHere
if errors.As(err, &nh) {
st.State = Skipped
}
return st
}
st.Variant = build.Variant
@@ -168,6 +186,9 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
// Into a scratch directory renamed into place at the end, so a failure halfway
// leaves nothing that looks installed.
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
if b.Python != "" {
return in.installPython(ctx, r, w, b, dir, st)
}
if err := in.packages(ctx, b.Packages, st); err != nil {
return err
}
@@ -235,18 +256,32 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
// running it is the only proof that the rest — the right instruction set, the
// right driver — is in place.
func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, version string, err error) {
bin, err = findFile(dir, b.Binary)
if err != nil {
return "", "", err
}
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
if missing := lddMissing(string(out)); len(missing) > 0 {
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
timeout := 60 * time.Second
if b.Python != "" {
// A script in the environment, not a binary to ldd; and importing
// torch and the engine behind it takes a while on a cold disk.
bin, timeout = filepath.Join(dir, "bin", b.Binary), 3*time.Minute
if _, err := os.Stat(bin); err != nil {
return "", "", fmt.Errorf("%s is not in the environment: %w", b.Binary, err)
}
} else {
bin, err = findFile(dir, b.Binary)
if err != nil {
return "", "", err
}
if out, err := in.run(ctx, nil, "ldd", bin); err == nil {
if missing := lddMissing(string(out)); len(missing) > 0 {
return "", "", fmt.Errorf("%s cannot run on this system: %s", b.Binary, strings.Join(missing, "; "))
}
}
}
tctx, cancel := context.WithTimeout(ctx, 60*time.Second)
args := b.VersionArgs
if len(args) == 0 {
args = []string{"--version"}
}
tctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
out, err := in.run(tctx, nil, bin, "--version")
out, err := in.run(tctx, nil, bin, args...)
if err != nil {
return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
}
@@ -607,6 +642,13 @@ func (in *Installer) report(st Status) {
}
}
func runWithStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // fixed commands and paths we installed
cmd.Env = append(os.Environ(), env...)
cmd.Stdin = strings.NewReader(stdin)
return cmd.CombinedOutput()
}
func (in *Installer) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
if in.Run != nil {
return in.Run(ctx, env, name, args...)
@@ -662,5 +704,11 @@ func firstVersionLine(b []byte) string {
return strings.TrimSpace(l)
}
}
// vllm --version says the number and nothing else.
for _, l := range strings.Split(string(b), "\n") {
if l = strings.TrimSpace(l); l != "" && l[0] >= '0' && l[0] <= '9' {
return l
}
}
return ""
}
+2 -2
View File
@@ -265,8 +265,8 @@ func TestTheBuildFollowsTheHardware(t *testing.T) {
func TestAnEngineTheAgentCannotInstallSaysSo(t *testing.T) {
in := &Installer{Root: t.TempDir(), Host: ubuntu}
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.29.0"}})
if got[0].State != Unsupported {
got := in.Converge(t.Context(), []Want{{Engine: "tgi", Version: "3.0.0"}})
if got[0].State != Unsupported || !strings.Contains(got[0].Detail, "install it by hand") {
t.Errorf("got %+v", got[0])
}
}
+205
View File
@@ -0,0 +1,205 @@
package provision
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"syscall"
"time"
)
// A Python engine is a virtual environment built by uv: an interpreter uv
// fetches, and the engine with every dependency from PyPI.
//
// Two things make it as trustworthy as an archive checked against its
// publisher's checksum. The dependencies are resolved as PyPI stood a day
// after the engine was published (--exclude-newer), so a node built next
// month gets what one built today got, not whatever a dependency released
// since — the usual way a working vLLM stops working. And the resolution is
// written out with every file's sha256 from PyPI and installed with
// --require-hashes, so a file that does not match what PyPI published is
// refused. The resolved list is kept in the environment, as the record of
// exactly what is installed.
// pythonInstallTimeout bounds one installation: several gigabytes, on a
// network that may be slow, but not forever.
const pythonInstallTimeout = time.Hour
// lockName is the resolved requirements, kept in the environment.
const lockName = "llmgw-requirements.txt"
func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
ctx, cancel := context.WithTimeout(ctx, pythonInstallTimeout)
defer cancel()
if err := in.packages(ctx, b.Packages, st); err != nil {
return err
}
if err := os.MkdirAll(in.root(), 0o755); err != nil {
return err
}
if b.FreeGB > 0 {
if free, err := in.free(in.root()); err == nil && free < float64(b.FreeGB) {
return fmt.Errorf("%s needs about %d GB free under %s and there are %.1f GB",
w.Engine, b.FreeGB, in.root(), free)
}
}
in.progress(st, "installing uv %s", UVVersion)
uv := in.one(ctx, Want{Engine: "uv", Version: UVVersion})
if uv.State != Ready {
return fmt.Errorf("installing uv, which installs %s: %s", w.Engine, uv.Detail)
}
in.progress(st, "asking PyPI when %s %s was published", r.PyPI, w.Version)
published, err := in.published(ctx, r.PyPI, w.Version)
if err != nil {
return err
}
asOf := published.Add(24 * time.Hour).UTC().Format(time.RFC3339)
cache := filepath.Join(in.root(), ".uv-cache")
env := []string{
"UV_CACHE_DIR=" + cache,
"UV_PYTHON_INSTALL_DIR=" + filepath.Join(in.root(), "python"),
// Only the interpreter uv fetched: the system's may be the wrong
// version, or change under the environment with the next upgrade.
"UV_PYTHON_PREFERENCE=only-managed",
// Nobody's uv.toml or pip.conf decides what is installed here.
"UV_NO_CONFIG=1",
}
tmp := dir + ".partial"
_ = os.RemoveAll(tmp)
defer os.RemoveAll(tmp) //nolint:errcheck // gone either way once renamed
in.progress(st, "creating a Python %s environment for %s", b.Python, w.Engine)
// Relocatable, because it is built under one name and renamed into place
// once it works: its scripts find their interpreter relative to
// themselves rather than by the path they were built at.
if out, err := in.run(ctx, env, uv.Binary, "venv", "--relocatable", "--python", b.Python, tmp); err != nil {
return fmt.Errorf("uv venv: %v: %s", err, tail(out))
}
python := filepath.Join(tmp, "bin", "python")
in.progress(st, "resolving %s against PyPI as of %s", strings.Join(b.Requirements, " "), asOf[:10])
lock := filepath.Join(tmp, lockName)
args := []string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
"--generate-hashes", "--quiet", "--output-file", lock, "-"}
if out, err := in.runStdin(ctx, env, strings.Join(b.Requirements, "\n")+"\n", uv.Binary, args...); err != nil {
return fmt.Errorf("resolving %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
}
in.progress(st, "downloading and installing %s and its dependencies: several gigabytes, which takes minutes",
strings.Join(b.Requirements, " "))
if out, err := in.run(ctx, env, uv.Binary, "pip", "install", "--python", python,
"--require-hashes", "--no-deps", "-r", lock); err != nil {
return fmt.Errorf("installing %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
}
in.progress(st, "checking that %s runs", w.Engine)
if _, _, err := in.verify(ctx, tmp, b); err != nil {
return err
}
m, _ := json.MarshalIndent(markerFile{
Engine: w.Engine, Version: w.Version, Variant: b.Variant,
Binary: filepath.Join("bin", b.Binary), Requirements: b.Requirements,
ExcludeNewer: asOf, UV: UVVersion, Installed: time.Now().UTC(),
}, "", " ")
if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
return err
}
_ = os.RemoveAll(dir)
if err := os.Rename(tmp, dir); err != nil {
return err
}
// The cache is as large as the environment. Kept after a failure, so the
// next attempt does not download it all again; dropped after success.
_ = os.RemoveAll(cache)
in.log().Info("installed an engine", "engine", w.Engine, "version", w.Version,
"variant", b.Variant, "dir", dir, "as_of", asOf)
return nil
}
// published is when PyPI received a release's last file.
func (in *Installer) published(ctx context.Context, project, version string) (time.Time, error) {
url := strings.TrimSuffix(in.pypiBase(), "/") + "/pypi/" + project + "/" + version + "/json"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return time.Time{}, err
}
resp, err := in.client().Do(req)
if err != nil {
return time.Time{}, fmt.Errorf("asking PyPI for %s %s: %w", project, version, err)
}
defer resp.Body.Close() //nolint:errcheck // read-only
if resp.StatusCode == http.StatusNotFound {
return time.Time{}, fmt.Errorf("PyPI has no %s %s", project, version)
}
if resp.StatusCode != http.StatusOK {
return time.Time{}, fmt.Errorf("asking PyPI for %s %s: %s", project, version, resp.Status)
}
var rel struct {
URLs []struct {
Uploaded time.Time `json:"upload_time_iso_8601"`
} `json:"urls"`
}
if err := json.NewDecoder(resp.Body).Decode(&rel); err != nil {
return time.Time{}, fmt.Errorf("reading PyPI's answer for %s %s: %w", project, version, err)
}
var last time.Time
for _, u := range rel.URLs {
if u.Uploaded.After(last) {
last = u.Uploaded
}
}
if last.IsZero() {
return time.Time{}, errors.New("PyPI lists no files for " + project + " " + version)
}
return last, nil
}
func (in *Installer) free(dir string) (float64, error) {
if in.FreeSpace != nil {
return in.FreeSpace(dir)
}
return FreeGB(dir)
}
func (in *Installer) pypiBase() string {
if in.PyPIBase != "" {
return in.PyPIBase
}
return "https://pypi.org"
}
// runStdin runs a command with input on its standard input.
func (in *Installer) runStdin(ctx context.Context, env []string, stdin, name string, args ...string) ([]byte, error) {
if in.RunStdin != nil {
return in.RunStdin(ctx, env, stdin, name, args...)
}
return runWithStdin(ctx, env, stdin, name, args...)
}
// FreeGB is the space available to an unprivileged writer under dir.
func FreeGB(dir string) (float64, error) {
var s syscall.Statfs_t
if err := syscall.Statfs(dir, &s); err != nil {
return 0, err
}
return float64(s.Bavail) * float64(s.Bsize) / (1 << 30), nil
}
// tail is the last few lines of a command's output: uv explains a failed
// resolution over several, and the last alone is rarely the reason.
func tail(b []byte) string {
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
if len(lines) > 4 {
lines = lines[len(lines)-4:]
}
return strings.Join(lines, " / ")
}
+184
View File
@@ -0,0 +1,184 @@
package provision
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
var gpuNode = discovery.Host{OSID: "ubuntu", OSVersion: "24.04", Arch: "amd64",
NVIDIADriver: "580.65.06", MinComputeCapability: 8.0}
// fakePython is uv and the environments it builds.
type fakePython struct {
fakeSystem
pmu sync.Mutex
commands []string
failPip bool
}
func (f *fakePython) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
line := name + " " + strings.Join(args, " ")
f.pmu.Lock()
f.commands = append(f.commands, line)
f.pmu.Unlock()
switch {
case strings.HasSuffix(name, "/uv") && len(args) == 1 && args[0] == "--version":
return []byte("uv 0.12.20\n"), nil
case strings.HasSuffix(name, "/uv") && args[0] == "venv":
return nil, os.MkdirAll(filepath.Join(args[len(args)-1], "bin"), 0o755)
case strings.HasSuffix(name, "/uv") && args[0] == "pip" && args[1] == "install":
if f.failPip {
return []byte("error: Hash mismatch for torch\n"), errExit
}
venv := filepath.Dir(filepath.Dir(args[slices.Index(args, "--python")+1]))
return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755)
case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version":
return []byte("0.30.0\n"), nil
}
return f.fakeSystem.run(ctx, env, name, args...)
}
func (f *fakePython) runStdin(_ context.Context, _ []string, stdin, name string, args ...string) ([]byte, error) {
f.pmu.Lock()
f.commands = append(f.commands, name+" "+strings.Join(args, " ")+" <<< "+strings.TrimSpace(stdin))
f.pmu.Unlock()
out := args[slices.Index(args, "--output-file")+1]
return nil, os.WriteFile(out, []byte("vllm==0.30.0 --hash=sha256:ab\n"), 0o644)
}
func pythonInstaller(t *testing.T, host discovery.Host, sys *fakePython) (*Installer, *release) {
uv := newRelease(map[string][]byte{
"uv-x86_64-unknown-linux-gnu.tar.gz": tarGz(t,
map[string]string{"uv-x86_64-unknown-linux-gnu/uv": "#!/bin/sh\n"}, nil),
})
gh := uv.serve(t)
mux := http.NewServeMux()
mux.HandleFunc("/pypi/vllm/0.30.0/json", func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte(`{"urls":[{"upload_time_iso_8601":"2026-09-22T05:22:21.647189Z"},
{"upload_time_iso_8601":"2026-09-22T05:23:10.039243Z"}]}`))
})
pypi := httptest.NewServer(mux)
t.Cleanup(pypi.Close)
return &Installer{
Root: t.TempDir(), Host: host, Run: sys.run, RunStdin: sys.runStdin,
APIBase: gh.URL, DownloadBase: gh.URL, PyPIBase: pypi.URL, HTTP: gh.Client(),
FreeSpace: func(string) (float64, error) { return 100, nil },
}, uv
}
func TestVLLMIsInstalledOnAGPUNode(t *testing.T) {
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
in, _ := pythonInstaller(t, gpuNode, sys)
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
if got[0].State != Ready {
t.Fatalf("not ready: %+v", got[0])
}
dir := filepath.Join(in.Root, "vllm", "0.30.0-cu13")
if got[0].Binary != filepath.Join(dir, "bin", "vllm") {
t.Errorf("binary %s, want the environment's vllm", got[0].Binary)
}
if !sys.installed["gcc"] {
t.Error("gcc, which Triton compiles its launcher with, was not installed")
}
all := strings.Join(sys.commands, "\n")
for _, want := range []string{
// PyPI as it stood a day after vLLM's last file, so the dependencies
// are the ones it was released against.
"--exclude-newer 2026-09-23T05:23:10Z",
"--generate-hashes",
"--require-hashes",
"venv --relocatable --python 3.12",
} {
if !strings.Contains(all, want) {
t.Errorf("no %q in:\n%s", want, all)
}
}
if _, err := os.Stat(filepath.Join(dir, lockName)); err != nil {
t.Errorf("the resolved requirements were not kept: %v", err)
}
if _, err := os.Stat(filepath.Join(in.Root, ".uv-cache")); err == nil {
t.Error("the download cache was left behind after a successful install")
}
// Checked again on the next pass, not installed again.
before := len(sys.commands)
if again := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}}); again[0].State != Ready {
t.Fatalf("second pass: %+v", again[0])
}
for _, c := range sys.commands[before:] {
if strings.Contains(c, "pip install") {
t.Errorf("a ready environment was installed again: %s", c)
}
}
}
func TestVLLMIsNotForEveryNode(t *testing.T) {
cases := []struct {
name string
host discovery.Host
state string
says string
}{
{"no GPU", ubuntu, Skipped, "has none"},
{"old driver", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15"}, Unsupported, "driver 580"},
{"V100", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0}, Unsupported, "7.5"},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
in, uv := pythonInstaller(t, c.host, sys)
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
if got[0].State != c.state || !strings.Contains(got[0].Detail, c.says) {
t.Errorf("got %s %q, want %s saying %q", got[0].State, got[0].Detail, c.state, c.says)
}
if uv.gets != 0 || len(sys.commands) != 0 {
t.Errorf("a node that cannot run it still started installing: %v", sys.commands)
}
})
}
}
func TestAPythonEngineIsRefusedWithoutTheDiskForIt(t *testing.T) {
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}}
in, uv := pythonInstaller(t, gpuNode, sys)
in.FreeSpace = func(string) (float64, error) { return 12.5, nil }
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
if got[0].State != Failed || !strings.Contains(got[0].Detail, "12.5 GB") {
t.Errorf("got %s %q", got[0].State, got[0].Detail)
}
if uv.gets != 0 {
t.Error("it began downloading anyway")
}
}
func TestAFailedPythonInstallLeavesNothingThatLooksInstalled(t *testing.T) {
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}, failPip: true}
in, _ := pythonInstaller(t, gpuNode, sys)
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
if got[0].State != Failed || !strings.Contains(got[0].Detail, "Hash mismatch") {
t.Fatalf("got %s %q", got[0].State, got[0].Detail)
}
entries, _ := os.ReadDir(filepath.Join(in.Root, "vllm"))
for _, e := range entries {
t.Errorf("left behind: %s", e.Name())
}
}
func TestAVersionPyPIDoesNotHaveIsSaidSo(t *testing.T) {
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{"gcc": true}}}
in, _ := pythonInstaller(t, gpuNode, sys)
got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "9.9.9"}})
if got[0].State != Failed || !strings.Contains(got[0].Detail, "PyPI has no vllm 9.9.9") {
t.Errorf("got %s %q", got[0].State, got[0].Detail)
}
}
+131 -4
View File
@@ -33,23 +33,150 @@ type Build struct {
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
// Python, when set, makes the build a Python environment rather than an
// archive: that interpreter version, with Requirements installed from
// PyPI by uv. Binary is then looked for in the environment's bin.
Python string
Requirements []string
// VersionArgs is what Binary is run with to prove the build works, when
// --version is not it.
VersionArgs []string
// FreeGB is the disk space the installation needs before it starts: a
// Python engine is several gigabytes, and one that fills the disk halfway
// leaves the node worse off than one that never began.
FreeGB int
}
// Recipe knows how to install one engine.
type Recipe struct {
// Repo is the GitHub repository whose releases carry the builds.
// Repo is the GitHub repository whose releases carry the builds; empty
// for an engine installed from PyPI.
Repo string
// PyPI is the project an engine installed from PyPI is published as.
PyPI string
// Choose picks the build for a machine, or says why none fits.
Choose func(version string, h discovery.Host) (Build, error)
}
// Recipes are the engines this agent can install.
//
// vLLM and SGLang are not here yet: they are Python environments of several
// gigabytes rather than an archive, and are installed by hand and named in the
// agent's configuration until they are.
// vLLM and SGLang are Python environments of several gigabytes, installed
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
// checked against its published checksum.
var Recipes = map[string]Recipe{
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
}
// UVVersion is the uv that installs the Python engines.
const UVVersion = "0.12.20"
// pythonForEngines is the interpreter the Python engines run on. uv fetches
// it, so the node's own Python — or its absence — does not matter.
const pythonForEngines = "3.12"
// NotHere is the error a recipe gives for a node the engine is not meant
// for, as opposed to one it should run on and cannot: vLLM on a machine
// with no GPU is not a fault, and is not reported as one.
type NotHere struct{ Reason string }
func (e NotHere) Error() string { return e.Reason }
func chooseUV(version string, h discovery.Host) (Build, error) {
var arch string
switch h.Arch {
case "amd64":
arch = "x86_64"
case "arm64":
arch = "aarch64"
default:
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
}
return Build{
Variant: arch,
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
Binary: "uv",
}, nil
}
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
const minDriverForCUDA13 = 580
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
// Volta, so a V100 cannot run these builds at all.
const minComputeForCUDA13 = 7.5
// cudaGate says whether a node can run an engine built on CUDA 13.
func cudaGate(engine string, h discovery.Host) error {
switch h.Arch {
case "amd64", "arm64":
default:
return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
}
if h.NVIDIADriver == "" {
return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
}
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 {
return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s",
engine, minDriverForCUDA13, h.NVIDIADriver)
}
if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 {
return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f",
engine, minComputeForCUDA13, cc)
}
return nil
}
// Python engines need a C compiler at run time: Triton builds a small
// launcher for its kernels the first time a model loads, and without gcc
// that load fails long after the installation looked finished.
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
func chooseVLLM(version string, h discovery.Host) (Build, error) {
if err := cudaGate("vLLM", h); err != nil {
return Build{}, err
}
return Build{
Variant: "cu13",
Python: pythonForEngines,
Requirements: []string{"vllm==" + version},
Binary: "vllm",
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
func chooseSGLang(version string, h discovery.Host) (Build, error) {
if err := cudaGate("SGLang", h); err != nil {
return Build{}, err
}
// SGLang is launched as a module, so what the agent is given is the
// environment's interpreter, and the proof is importing it.
return Build{
Variant: "cu13",
Python: pythonForEngines,
Requirements: []string{"sglang==" + version},
Binary: "python",
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
// leadingInt is the number a version string starts with: 580 of "580.65.06".
func leadingInt(s string) int {
n := 0
for _, c := range strings.TrimSpace(s) {
if c < '0' || c > '9' {
break
}
n = n*10 + int(c-'0')
}
return n
}
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also