Files
llmgw/internal/agent/provision/recipe.go
T
Ilker ManapandClaude Opus 5.5 86e9a1cb83
Release / build (push) Successful in 1m13s
Release / notes (push) Successful in 4s
CI / test (push) Successful in 3m47s
Give V100s the last vLLM built for them, not a build without their kernels
Read from vLLM's release pipeline: every x86 build from 0.20.0 has
kernels for compute 7.5 and up, the CUDA 12.9 one included, and 0.19.1
is the last built for 7.0. PyPI's build is CUDA 13 from 0.20, not 0.26.
A V100 asked for a newer vLLM is refused, naming 0.19.1 for
engine_overrides; the CUDA 12.9 fallback is for older drivers on
Turing and newer. The cu129 wheel is found among the release's assets,
since its manylinux tag differs between versions.

Refs #2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 14:04:45 +02:00

462 lines
17 KiB
Go

// Package provision installs the engines a node is asked to have.
//
// The control plane says which engines at which versions; everything else is
// worked out here, on the node, because only the node knows it: which build
// fits its processor and driver, what its operating system is missing, and
// whether the result actually runs. Nothing already on the machine is taken
// on trust. An engine counts as installed when this package installed it,
// checked every file against the checksum its publisher gave, and watched it
// start.
package provision
import (
"errors"
"fmt"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Build is one published build of an engine, chosen for one machine.
type Build struct {
// Variant names the build: "cpu", "cuda-12.8".
Variant string
// Assets are the release files to fetch, unpacked in order into one
// directory.
Assets []string
// Packages are operating system packages the build links against and a
// minimal install may not carry, by packaging family ("deb", "rpm"): the
// same library is libgomp1 on one and libgomp on the other.
Packages map[string][]string
// Binary is the executable's file name, found anywhere in the unpacked
// tree.
Binary string
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
// Python, when set, makes the build a Python environment rather than an
// archive: that interpreter version, with Requirements installed from
// PyPI by uv. Binary is then looked for in the environment's bin.
Python string
Requirements []string
// VersionArgs is what Binary is run with to prove the build works, when
// --version is not it.
VersionArgs []string
// GPUBackend, when set, is the shared library the build computes on GPUs
// with, loaded at run time rather than linked: "libggml-cuda.so". The
// build must list a device with it, or it is not installed — a llama.cpp
// whose CUDA backend cannot load runs on the CPU without complaint.
GPUBackend string
// Wheel, when set, is the engine itself as a wheel from a GitHub
// release rather than from PyPI, checked against the digest GitHub
// publishes for it. ExtraIndex is a package index besides PyPI that the
// rest resolves from too: the CUDA 12.9 torch the wheel was built on.
Wheel *Wheel
ExtraIndex string
// GPUCheck, on a node with an NVIDIA GPU, has the installed environment
// compute something on one before the build counts as installed. A build
// with no kernels for the card — a GPU generation its torch dropped —
// imports and prints its version, and fails on its first model.
GPUCheck bool
// FreeGB is the disk space the installation needs before it starts: a
// Python engine is several gigabytes, and one that fills the disk halfway
// leaves the node worse off than one that never began.
FreeGB int
}
// Wheel is a package published as a GitHub release asset: the asset whose
// name starts with Prefix and ends with Suffix.
type Wheel struct {
Project, Repo, Tag, Prefix, Suffix string
}
// Recipe knows how to install one engine.
type Recipe struct {
// Repo is the GitHub repository whose releases carry the builds; empty
// for an engine installed from PyPI.
Repo string
// PyPI is the project an engine installed from PyPI is published as.
PyPI string
// Choose picks the build for a machine, or says why none fits. variant
// is a build chosen for the node ("cu129"); empty leaves it to the
// machine.
Choose func(version, variant string, h discovery.Host) (Build, error)
}
// Recipes are the engines this agent can install.
//
// vLLM and SGLang are Python environments of several gigabytes, installed
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
// checked against its published checksum.
var Recipes = map[string]Recipe{
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
}
// Installable are the engines this agent can install, as it tells the
// control plane: every recipe but uv, which is a tool for the others.
func Installable() []string {
var out []string
for name := range Recipes {
if name != "uv" {
out = append(out, name)
}
}
slices.Sort(out)
return out
}
// UVVersion is the uv that installs the Python engines.
const UVVersion = "0.12.20"
// pythonForEngines is the interpreter the Python engines run on. uv fetches
// it, so the node's own Python — or its absence — does not matter.
const pythonForEngines = "3.12"
// NotHere is the error a recipe gives for a node the engine is not meant
// for, as opposed to one it should run on and cannot: vLLM on a machine
// with no GPU is not a fault, and is not reported as one.
type NotHere struct{ Reason string }
func (e NotHere) Error() string { return e.Reason }
func chooseUV(version, _ string, h discovery.Host) (Build, error) {
var arch string
switch h.Arch {
case "amd64":
arch = "x86_64"
case "arm64":
arch = "aarch64"
default:
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
}
return Build{
Variant: arch,
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
Binary: "uv",
}, nil
}
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. vLLM's
// PyPI build is CUDA 13 from 0.20 (before it, CUDA 12, with CUDA 13 as the
// "+cu130" wheel on GitHub), and SGLang's from 0.5.
const minDriverForCUDA13 = 580
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
// Volta, so a V100 cannot run these builds at all.
const minComputeForCUDA13 = 7.5
// The CUDA 12 builds run on any driver from 525, by CUDA's minor version
// compatibility; the oldest GPU one has kernels for is minComputeFor's
// answer, which is Volta only up to voltaUntil.
const (
minDriverForCUDA12 = 525
minComputeForCUDA12 = 7.0
)
// cuda13Since is the first version of each Python engine whose PyPI build is
// CUDA 13, read from vLLM's releases: 0.19.1 publishes "+cu130" beside a
// CUDA 12 default, 0.20.0 "+cu129" beside a CUDA 13 one.
var cuda13Since = map[string]string{"vLLM": "0.20", "SGLang": "0.5"}
// voltaUntil is the last vLLM built for Volta (7.0). Its release pipeline
// built with the Dockerfile's default arch list, "7.0 7.5 8.0 8.9 9.0 10.0
// 12.0"; from 0.20.0 every x86 build starts at 7.5, CUDA 12.9's included.
const voltaUntil = "0.19.1"
// minComputeFor is the oldest GPU a vLLM version's builds have kernels for
// on an architecture: what the project compiled, not what CUDA supports.
func minComputeFor(version, arch string) float64 {
if arch == "arm64" {
// Built for the Grace machines: 8.7 and up before 0.20, 8.0 and up
// since.
if versionBefore(version, "0.20") {
return 8.7
}
return 8.0
}
if versionBefore(version, "0.20") {
return minComputeForCUDA12
}
return minComputeForCUDA13
}
// cudaGate says whether a node can run this version of an engine, and which
// CUDA it is built for.
//
// By version, because the line moved: a V100 cannot run vLLM 0.20 or later,
// and runs 0.19.1 — so a cluster with Volta and newer cards gives the Volta
// nodes the older version (engine_overrides in the server's configuration)
// rather than going without vLLM, or without the Volta nodes.
func cudaGate(engine, version string, h discovery.Host) (variant string, err error) {
switch h.Arch {
case "amd64", "arm64":
default:
return "", fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
}
if h.NVIDIADriver == "" {
return "", NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
}
variant, cuda, minDriver, minCC := "cu13", "13", minDriverForCUDA13, minComputeForCUDA13
if versionBefore(version, cuda13Since[engine]) {
variant, cuda, minDriver, minCC = "cu12", "12", minDriverForCUDA12, minComputeForCUDA12
}
if engine == "vLLM" {
minCC = minComputeFor(version, h.Arch)
}
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriver {
return "", fmt.Errorf("%s %s is built for CUDA %s, which needs NVIDIA driver %d or newer; this node has %s",
engine, version, cuda, minDriver, h.NVIDIADriver)
}
if cc := h.MinComputeCapability; cc > 0 && cc < minCC {
return "", fmt.Errorf("%s %s has kernels for GPUs of compute capability %.1f and newer; this node has %.1f%s",
engine, version, minCC, cc, olderHint(engine, cc))
}
return variant, nil
}
// olderHint says which version runs on a GPU too old for the one asked for.
func olderHint(engine string, cc float64) string {
if engine == "vLLM" && cc >= minComputeForCUDA12 && cc < minComputeForCUDA13 {
return "; the last vLLM built for it is " + voltaUntil +
": give these nodes that version with engine_overrides in the server's configuration"
}
return ""
}
// versionBefore reports whether version a is older than b, comparing the
// dotted numbers in turn ("0.25.1" before "0.26"). A part that is not a
// number counts as zero.
func versionBefore(a, b string) bool {
if b == "" {
return false
}
pa, pb := strings.Split(strings.TrimPrefix(a, "v"), "."), strings.Split(b, ".")
for i := 0; i < max(len(pa), len(pb)); i++ {
x, y := 0, 0
if i < len(pa) {
x = leadingInt(pa[i])
}
if i < len(pb) {
y = leadingInt(pb[i])
}
if x != y {
return x < y
}
}
return false
}
// Python engines need a C compiler at run time: Triton builds a small
// launcher for its kernels the first time a model loads, and without gcc
// that load fails long after the installation looked finished.
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
// VLLMVariants are the builds of vLLM a node can be given by choice: CUDA
// 13 from PyPI, and CUDA 12.9 from the project's GitHub release — the same
// version, built for the drivers and GPUs CUDA 13 left behind.
var VLLMVariants = []string{"cu13", "cu129"}
// pytorchCU129 is where the CUDA 12.9 build's torch is published. PyPI's
// torch is built for CUDA 13; the cu129 vLLM wheel needs this one.
const pytorchCU129 = "https://download.pytorch.org/whl/cu129"
// chooseVLLM picks the vLLM build for a machine.
//
// Left to the machine (variant ""), it is CUDA 13 where that runs, and the
// same version's CUDA 12.9 build where only that does: one version across
// the cluster, which matters because the flags llmgw builds are checked
// against one version, and an older vLLM on the older cards would be a
// second set of flags nobody checked. A variant chosen for the node is that
// build, if the node can run it.
func chooseVLLM(version, variant string, h discovery.Host) (Build, error) {
// What no build of it gets past: a machine it is not published for, or
// one with no NVIDIA GPU. Only a CUDA requirement falls back to the
// CUDA 12.9 build.
switch h.Arch {
case "amd64", "arm64":
default:
return Build{}, fmt.Errorf("vLLM publishes no Linux build for %s", orUnknown(h.Arch))
}
if h.NVIDIADriver == "" {
return Build{}, NotHere{"vLLM runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
}
pypi := func(v string) Build {
return Build{
Variant: v,
Python: pythonForEngines,
Requirements: []string{"vllm==" + version},
Binary: "vllm",
Packages: compilerPackages,
FreeGB: 25,
GPUCheck: true,
}
}
cu129 := func() (Build, error) {
if err := cudaFits("vLLM", version, "12.9", minDriverForCUDA12, minComputeFor(version, h.Arch), h); err != nil {
return Build{}, err
}
arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[h.Arch]
b := pypi("cu129")
b.Requirements = nil
// Found in the release's assets rather than named: the manylinux tag
// in the file's name changes between versions (2_28, 2_31, 2_34).
b.Wheel = &Wheel{
Project: "vllm", Repo: "vllm-project/vllm", Tag: "v" + version,
Prefix: "vllm-" + version + "+cu129-", Suffix: "_" + arch + ".whl",
}
b.ExtraIndex = pytorchCU129
return b, nil
}
// From 0.20 the PyPI build is CUDA 13 and GitHub carries "+cu129".
modern := !versionBefore(version, cuda13Since["vLLM"])
switch variant {
case "":
v, err := cudaGate("vLLM", version, h)
if err == nil {
return pypi(v), nil
}
var nh NotHere
if errors.As(err, &nh) || !modern {
return Build{}, err
}
b, err129 := cu129()
if err129 != nil {
return Build{}, fmt.Errorf("%v; nor can its CUDA 12.9 build: %v", err, err129)
}
return b, nil
case "cu13":
if !modern {
return Build{}, fmt.Errorf("vLLM %s is built for CUDA 12; there is no CUDA 13 build of it", version)
}
if _, err := cudaGate("vLLM", version, h); err != nil {
return Build{}, err
}
return pypi("cu13"), nil
case "cu129":
if !modern {
// Before 0.20 PyPI's own build is CUDA 12, and is the one given.
v, err := cudaGate("vLLM", version, h)
if err != nil {
return Build{}, err
}
return pypi(v), nil
}
if _, err := cudaGate("vLLM", version, h); err != nil {
var nh NotHere
if errors.As(err, &nh) {
return Build{}, err
}
}
return cu129()
}
return Build{}, fmt.Errorf("vLLM has no build %q; the builds are %s", variant, strings.Join(VLLMVariants, ", "))
}
// cudaFits checks a node against a CUDA's driver and GPU requirements.
func cudaFits(engine, version, cuda string, minDriver int, minCC float64, h discovery.Host) error {
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriver {
return fmt.Errorf("%s %s built for CUDA %s needs NVIDIA driver %d or newer; this node has %s",
engine, version, cuda, minDriver, h.NVIDIADriver)
}
if cc := h.MinComputeCapability; cc > 0 && cc < minCC {
return fmt.Errorf("%s %s built for CUDA %s needs GPUs of compute capability %.1f or newer; this node has %.1f%s",
engine, version, cuda, minCC, cc, olderHint(engine, cc))
}
return nil
}
func chooseSGLang(version, chosen string, h discovery.Host) (Build, error) {
if chosen != "" {
return Build{}, fmt.Errorf("SGLang is installed in one build only, not %q", chosen)
}
variant, err := cudaGate("SGLang", version, h)
if err != nil {
return Build{}, err
}
// SGLang is launched as a module, so what the agent is given is the
// environment's interpreter, and the proof is importing it.
return Build{
Variant: variant,
Python: pythonForEngines,
Requirements: []string{"sglang==" + version},
Binary: "python",
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
// leadingInt is the number a version string starts with: 580 of "580.65.06".
func leadingInt(s string) int {
n := 0
for _, c := range strings.TrimSpace(s) {
if c < '0' || c > '9' {
break
}
n = n*10 + int(c-'0')
}
return n
}
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
// carries a matching cudart archive, so the node needs a driver and nothing
// else — no toolkit.
const cudaForLlamaCpp = "12.8"
func chooseLlamaCpp(version, _ string, h discovery.Host) (Build, error) {
// The published Linux builds are built on Ubuntu, but nothing about them
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
// name decides nothing here. The build is tried everywhere, and ldd says
// whether this system can load it — in the words of the missing symbol.
var arch string
switch h.Arch {
case "amd64":
arch = "x64"
case "arm64":
arch = "arm64"
default:
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
}
b := Build{
Variant: "cpu",
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
// Everything else the build links is in the archive or in any current
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
// the server does not start without it.
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
Binary: "llama-server",
}
if arch == "x64" {
// The archive carries a backend per CPU generation and loads the best
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
b.CPUFlags = []string{"avx2"}
}
// The CUDA build is published for x64 at this version; on arm64 it is a
// newer CUDA that the driver may not support, so an arm64 GPU node runs
// the CPU build until someone decides otherwise.
if h.NVIDIADriver != "" && arch == "x64" {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
b.Variant = "cuda-" + cudaForLlamaCpp
b.GPUBackend = "libggml-cuda.so"
b.Assets = []string{
fmt.Sprintf("llama-%s-%s", version, name),
fmt.Sprintf("cudart-llama-%s-%s", version, name),
}
}
return b, nil
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return "an unrecognised system"
}
return s
}