Files
llmgw/internal/agent/provision/recipe.go
T
Ilker ManapandClaude Opus 5.5 e313f32696 Decide engines per node, and let a node join without one it cannot run
engine_overrides give nodes picked by GPU class, id or labels their own
engine versions; node add --disable-engine keeps one off a node; what
vLLM cannot do on a node is a warning rather than a refused node; the
CUDA requirement follows the version; a hand-named binary is no longer
replaced by an installed one.

Refs #2 (items 4 and 7).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-01 12:07:25 +02:00

306 lines
11 KiB
Go

// Package provision installs the engines a node is asked to have.
//
// The control plane says which engines at which versions; everything else is
// worked out here, on the node, because only the node knows it: which build
// fits its processor and driver, what its operating system is missing, and
// whether the result actually runs. Nothing already on the machine is taken
// on trust. An engine counts as installed when this package installed it,
// checked every file against the checksum its publisher gave, and watched it
// start.
package provision
import (
"fmt"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Build is one published build of an engine, chosen for one machine.
type Build struct {
// Variant names the build: "cpu", "cuda-12.8".
Variant string
// Assets are the release files to fetch, unpacked in order into one
// directory.
Assets []string
// Packages are operating system packages the build links against and a
// minimal install may not carry, by packaging family ("deb", "rpm"): the
// same library is libgomp1 on one and libgomp on the other.
Packages map[string][]string
// Binary is the executable's file name, found anywhere in the unpacked
// tree.
Binary string
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
// Python, when set, makes the build a Python environment rather than an
// archive: that interpreter version, with Requirements installed from
// PyPI by uv. Binary is then looked for in the environment's bin.
Python string
Requirements []string
// VersionArgs is what Binary is run with to prove the build works, when
// --version is not it.
VersionArgs []string
// GPUBackend, when set, is the shared library the build computes on GPUs
// with, loaded at run time rather than linked: "libggml-cuda.so". The
// build must list a device with it, or it is not installed — a llama.cpp
// whose CUDA backend cannot load runs on the CPU without complaint.
GPUBackend string
// FreeGB is the disk space the installation needs before it starts: a
// Python engine is several gigabytes, and one that fills the disk halfway
// leaves the node worse off than one that never began.
FreeGB int
}
// Recipe knows how to install one engine.
type Recipe struct {
// Repo is the GitHub repository whose releases carry the builds; empty
// for an engine installed from PyPI.
Repo string
// PyPI is the project an engine installed from PyPI is published as.
PyPI string
// Choose picks the build for a machine, or says why none fits.
Choose func(version string, h discovery.Host) (Build, error)
}
// Recipes are the engines this agent can install.
//
// vLLM and SGLang are Python environments of several gigabytes, installed
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
// checked against its published checksum.
var Recipes = map[string]Recipe{
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
}
// Installable are the engines this agent can install, as it tells the
// control plane: every recipe but uv, which is a tool for the others.
func Installable() []string {
var out []string
for name := range Recipes {
if name != "uv" {
out = append(out, name)
}
}
slices.Sort(out)
return out
}
// UVVersion is the uv that installs the Python engines.
const UVVersion = "0.12.20"
// pythonForEngines is the interpreter the Python engines run on. uv fetches
// it, so the node's own Python — or its absence — does not matter.
const pythonForEngines = "3.12"
// NotHere is the error a recipe gives for a node the engine is not meant
// for, as opposed to one it should run on and cannot: vLLM on a machine
// with no GPU is not a fault, and is not reported as one.
type NotHere struct{ Reason string }
func (e NotHere) Error() string { return e.Reason }
func chooseUV(version string, h discovery.Host) (Build, error) {
var arch string
switch h.Arch {
case "amd64":
arch = "x86_64"
case "arm64":
arch = "aarch64"
default:
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
}
return Build{
Variant: arch,
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
Binary: "uv",
}, nil
}
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
const minDriverForCUDA13 = 580
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
// Volta, so a V100 cannot run these builds at all.
const minComputeForCUDA13 = 7.5
// The CUDA 12 builds before them run on any driver from 525, by CUDA's minor
// version compatibility, and on Volta.
const (
minDriverForCUDA12 = 525
minComputeForCUDA12 = 7.0
)
// cuda13Since is the first version of each Python engine built for CUDA 13.
var cuda13Since = map[string]string{"vLLM": "0.26", "SGLang": "0.5"}
// cudaGate says whether a node can run this version of an engine, and which
// CUDA it is built for.
//
// By version, because the line moved: a V100 cannot run vLLM 0.26 or later,
// and runs 0.25 — so a cluster with Volta and newer cards gives the Volta
// nodes the older build (engine_overrides in the server's configuration)
// rather than going without vLLM, or without the Volta nodes.
func cudaGate(engine, version string, h discovery.Host) (variant string, err error) {
switch h.Arch {
case "amd64", "arm64":
default:
return "", fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
}
if h.NVIDIADriver == "" {
return "", NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
}
variant, cuda, minDriver, minCC := "cu13", "13", minDriverForCUDA13, minComputeForCUDA13
if versionBefore(version, cuda13Since[engine]) {
variant, cuda, minDriver, minCC = "cu12", "12", minDriverForCUDA12, minComputeForCUDA12
}
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriver {
return "", fmt.Errorf("%s %s is built for CUDA %s, which needs NVIDIA driver %d or newer; this node has %s",
engine, version, cuda, minDriver, h.NVIDIADriver)
}
if cc := h.MinComputeCapability; cc > 0 && cc < minCC {
hint := ""
if cuda == "13" {
hint = fmt.Sprintf("; %s before %s is built for CUDA 12 and runs on it", engine, cuda13Since[engine])
}
return "", fmt.Errorf("%s %s is built for CUDA %s, which supports GPUs of compute capability %.1f and newer; this node has %.1f%s",
engine, version, cuda, minCC, cc, hint)
}
return variant, nil
}
// versionBefore reports whether version a is older than b, comparing the
// dotted numbers in turn ("0.25.1" before "0.26"). A part that is not a
// number counts as zero.
func versionBefore(a, b string) bool {
if b == "" {
return false
}
pa, pb := strings.Split(strings.TrimPrefix(a, "v"), "."), strings.Split(b, ".")
for i := 0; i < max(len(pa), len(pb)); i++ {
x, y := 0, 0
if i < len(pa) {
x = leadingInt(pa[i])
}
if i < len(pb) {
y = leadingInt(pb[i])
}
if x != y {
return x < y
}
}
return false
}
// Python engines need a C compiler at run time: Triton builds a small
// launcher for its kernels the first time a model loads, and without gcc
// that load fails long after the installation looked finished.
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
func chooseVLLM(version string, h discovery.Host) (Build, error) {
variant, err := cudaGate("vLLM", version, h)
if err != nil {
return Build{}, err
}
return Build{
Variant: variant,
Python: pythonForEngines,
Requirements: []string{"vllm==" + version},
Binary: "vllm",
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
func chooseSGLang(version string, h discovery.Host) (Build, error) {
variant, err := cudaGate("SGLang", version, h)
if err != nil {
return Build{}, err
}
// SGLang is launched as a module, so what the agent is given is the
// environment's interpreter, and the proof is importing it.
return Build{
Variant: variant,
Python: pythonForEngines,
Requirements: []string{"sglang==" + version},
Binary: "python",
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
Packages: compilerPackages,
FreeGB: 25,
}, nil
}
// leadingInt is the number a version string starts with: 580 of "580.65.06".
func leadingInt(s string) int {
n := 0
for _, c := range strings.TrimSpace(s) {
if c < '0' || c > '9' {
break
}
n = n*10 + int(c-'0')
}
return n
}
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
// carries a matching cudart archive, so the node needs a driver and nothing
// else — no toolkit.
const cudaForLlamaCpp = "12.8"
func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
// The published Linux builds are built on Ubuntu, but nothing about them
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
// name decides nothing here. The build is tried everywhere, and ldd says
// whether this system can load it — in the words of the missing symbol.
var arch string
switch h.Arch {
case "amd64":
arch = "x64"
case "arm64":
arch = "arm64"
default:
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
}
b := Build{
Variant: "cpu",
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
// Everything else the build links is in the archive or in any current
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
// the server does not start without it.
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
Binary: "llama-server",
}
if arch == "x64" {
// The archive carries a backend per CPU generation and loads the best
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
b.CPUFlags = []string{"avx2"}
}
// The CUDA build is published for x64 at this version; on arm64 it is a
// newer CUDA that the driver may not support, so an arm64 GPU node runs
// the CPU build until someone decides otherwise.
if h.NVIDIADriver != "" && arch == "x64" {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
b.Variant = "cuda-" + cudaForLlamaCpp
b.GPUBackend = "libggml-cuda.so"
b.Assets = []string{
fmt.Sprintf("llama-%s-%s", version, name),
fmt.Sprintf("cudart-llama-%s-%s", version, name),
}
}
return b, nil
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return "an unrecognised system"
}
return s
}