Files
llmgw/internal/agent/provision/recipe.go
T
Ilker ManapandClaude Opus 5.5 4c5a2fd6da Install vLLM on every GPU node that can run it, as llama.cpp is
vLLM was installed by hand and named in each agent's configuration. The
agent now installs it: a Python 3.12 environment built by uv, which is
itself a GitHub release checked against its checksum, from PyPI resolved as
it stood a day after that vLLM was published and installed with
--require-hashes. The resolved list is kept in the environment. The
environment is built relocatable under a scratch name and renamed into place
once it runs, like an archive.

The default is 0.30.0, whose flags were checked against the ones llmgw
builds. It is built for CUDA 13, so it needs driver 580 and a Turing GPU or
newer, and 25 GB free while it installs; a node lacking one says which, and a
node with no GPU skips it without being shown as a fault. gcc comes with it
because Triton compiles its launcher at the first model load. SGLang
installs the same way when the control plane names a version. llmgw doctor
reports whether vLLM can be installed on the node.

Checked by installing 0.30.0 this way on a test node: 7.7 GB, torch 2.13 for
CUDA 13.0, importing cleanly; the node has no GPU, so vLLM was not started.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-29 13:15:54 +02:00

237 lines
8.4 KiB
Go

// Package provision installs the engines a node is asked to have.
//
// The control plane says which engines at which versions; everything else is
// worked out here, on the node, because only the node knows it: which build
// fits its processor and driver, what its operating system is missing, and
// whether the result actually runs. Nothing already on the machine is taken
// on trust. An engine counts as installed when this package installed it,
// checked every file against the checksum its publisher gave, and watched it
// start.
package provision
import (
"fmt"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Build is one published build of an engine, chosen for one machine.
type Build struct {
// Variant names the build: "cpu", "cuda-12.8".
Variant string
// Assets are the release files to fetch, unpacked in order into one
// directory.
Assets []string
// Packages are operating system packages the build links against and a
// minimal install may not carry, by packaging family ("deb", "rpm"): the
// same library is libgomp1 on one and libgomp on the other.
Packages map[string][]string
// Binary is the executable's file name, found anywhere in the unpacked
// tree.
Binary string
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
// Python, when set, makes the build a Python environment rather than an
// archive: that interpreter version, with Requirements installed from
// PyPI by uv. Binary is then looked for in the environment's bin.
Python string
Requirements []string
// VersionArgs is what Binary is run with to prove the build works, when
// --version is not it.
VersionArgs []string
// FreeGB is the disk space the installation needs before it starts: a
// Python engine is several gigabytes, and one that fills the disk halfway
// leaves the node worse off than one that never began.
FreeGB int
}
// Recipe knows how to install one engine.
type Recipe struct {
// Repo is the GitHub repository whose releases carry the builds; empty
// for an engine installed from PyPI.
Repo string
// PyPI is the project an engine installed from PyPI is published as.
PyPI string
// Choose picks the build for a machine, or says why none fits.
Choose func(version string, h discovery.Host) (Build, error)
}
// Recipes are the engines this agent can install.
//
// vLLM and SGLang are Python environments of several gigabytes, installed
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
// checked against its published checksum.
var Recipes = map[string]Recipe{
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
}
// UVVersion is the uv that installs the Python engines.
const UVVersion = "0.12.20"
// pythonForEngines is the interpreter the Python engines run on. uv fetches
// it, so the node's own Python — or its absence — does not matter.
const pythonForEngines = "3.12"
// NotHere is the error a recipe gives for a node the engine is not meant
// for, as opposed to one it should run on and cannot: vLLM on a machine
// with no GPU is not a fault, and is not reported as one.
type NotHere struct{ Reason string }
func (e NotHere) Error() string { return e.Reason }
func chooseUV(version string, h discovery.Host) (Build, error) {
var arch string
switch h.Arch {
case "amd64":
arch = "x86_64"
case "arm64":
arch = "aarch64"
default:
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
}
return Build{
Variant: arch,
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
Binary: "uv",
}, nil
}
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
const minDriverForCUDA13 = 580
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
// Volta, so a V100 cannot run these builds at all.
const minComputeForCUDA13 = 7.5
// cudaGate says whether a node can run an engine built on CUDA 13.
func cudaGate(engine string, h discovery.Host) error {
switch h.Arch {
case "amd64", "arm64":
default:
return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
}
if h.NVIDIADriver == "" {
return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
}
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 {
return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s",
engine, minDriverForCUDA13, h.NVIDIADriver)
}
if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 {
return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f",
engine, minComputeForCUDA13, cc)
}
return nil
}
// Python engines need a C compiler at run time: Triton builds a small
// launcher for its kernels the first time a model loads, and without gcc
// that load fails long after the installation looked finished.
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
func chooseVLLM(version string, h discovery.Host) (Build, error) {
if err := cudaGate("vLLM", h); err != nil {
return Build{}, err
}
return Build{
Variant: "cu13",
Python: pythonForEngines,
Requirements: []string{"vllm==" + version},
Binary: "vllm",
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
func chooseSGLang(version string, h discovery.Host) (Build, error) {
if err := cudaGate("SGLang", h); err != nil {
return Build{}, err
}
// SGLang is launched as a module, so what the agent is given is the
// environment's interpreter, and the proof is importing it.
return Build{
Variant: "cu13",
Python: pythonForEngines,
Requirements: []string{"sglang==" + version},
Binary: "python",
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
// leadingInt is the number a version string starts with: 580 of "580.65.06".
func leadingInt(s string) int {
n := 0
for _, c := range strings.TrimSpace(s) {
if c < '0' || c > '9' {
break
}
n = n*10 + int(c-'0')
}
return n
}
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
// carries a matching cudart archive, so the node needs a driver and nothing
// else — no toolkit.
const cudaForLlamaCpp = "12.8"
func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
// The published Linux builds are built on Ubuntu, but nothing about them
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
// name decides nothing here. The build is tried everywhere, and ldd says
// whether this system can load it — in the words of the missing symbol.
var arch string
switch h.Arch {
case "amd64":
arch = "x64"
case "arm64":
arch = "arm64"
default:
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
}
b := Build{
Variant: "cpu",
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
// Everything else the build links is in the archive or in any current
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
// the server does not start without it.
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
Binary: "llama-server",
}
if arch == "x64" {
// The archive carries a backend per CPU generation and loads the best
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
b.CPUFlags = []string{"avx2"}
}
// The CUDA build is published for x64 at this version; on arm64 it is a
// newer CUDA that the driver may not support, so an arm64 GPU node runs
// the CPU build until someone decides otherwise.
if h.NVIDIADriver != "" && arch == "x64" {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
b.Variant = "cuda-" + cudaForLlamaCpp
b.Assets = []string{
fmt.Sprintf("llama-%s-%s", version, name),
fmt.Sprintf("cudart-llama-%s-%s", version, name),
}
}
return b, nil
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return "an unrecognised system"
}
return s
}