A control plane of this release asks every node for vLLM. An agent of the last one does not know how to install it, and answered "unsupported", so a cluster half way through its upgrade showed every node as failing. Agents now say which engines they can install (Register.installable_engines), and are asked for no others; an agent that says nothing is asked for llama.cpp alone, as it always was. doctor reports the CPU's vector extensions, and warns when AVX2 is missing, with the fix for a VM: the test node for node add was one, on QEMU's default CPU model, where llama.cpp runs several times slower and nothing else looks wrong. node add shows it, since it runs doctor. The changelog's unreleased section was two releases' worth of sections one after the other, with two lists of fixes and the upgrade notes in the middle. It is one entry now: largest changes first, the fixes together, and every upgrade note at the end, with vLLM's default install and how to keep your own. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
251 lines
8.7 KiB
Go
251 lines
8.7 KiB
Go
// Package provision installs the engines a node is asked to have.
|
|
//
|
|
// The control plane says which engines at which versions; everything else is
|
|
// worked out here, on the node, because only the node knows it: which build
|
|
// fits its processor and driver, what its operating system is missing, and
|
|
// whether the result actually runs. Nothing already on the machine is taken
|
|
// on trust. An engine counts as installed when this package installed it,
|
|
// checked every file against the checksum its publisher gave, and watched it
|
|
// start.
|
|
package provision
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
)
|
|
|
|
// Build is one published build of an engine, chosen for one machine.
|
|
type Build struct {
|
|
// Variant names the build: "cpu", "cuda-12.8".
|
|
Variant string
|
|
// Assets are the release files to fetch, unpacked in order into one
|
|
// directory.
|
|
Assets []string
|
|
// Packages are operating system packages the build links against and a
|
|
// minimal install may not carry, by packaging family ("deb", "rpm"): the
|
|
// same library is libgomp1 on one and libgomp on the other.
|
|
Packages map[string][]string
|
|
// Binary is the executable's file name, found anywhere in the unpacked
|
|
// tree.
|
|
Binary string
|
|
// CPUFlags are extensions the build is slow without. Missing ones are
|
|
// reported, not refused: slow is still running.
|
|
CPUFlags []string
|
|
|
|
// Python, when set, makes the build a Python environment rather than an
|
|
// archive: that interpreter version, with Requirements installed from
|
|
// PyPI by uv. Binary is then looked for in the environment's bin.
|
|
Python string
|
|
Requirements []string
|
|
// VersionArgs is what Binary is run with to prove the build works, when
|
|
// --version is not it.
|
|
VersionArgs []string
|
|
// FreeGB is the disk space the installation needs before it starts: a
|
|
// Python engine is several gigabytes, and one that fills the disk halfway
|
|
// leaves the node worse off than one that never began.
|
|
FreeGB int
|
|
}
|
|
|
|
// Recipe knows how to install one engine.
|
|
type Recipe struct {
|
|
// Repo is the GitHub repository whose releases carry the builds; empty
|
|
// for an engine installed from PyPI.
|
|
Repo string
|
|
// PyPI is the project an engine installed from PyPI is published as.
|
|
PyPI string
|
|
// Choose picks the build for a machine, or says why none fits.
|
|
Choose func(version string, h discovery.Host) (Build, error)
|
|
}
|
|
|
|
// Recipes are the engines this agent can install.
|
|
//
|
|
// vLLM and SGLang are Python environments of several gigabytes, installed
|
|
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
|
|
// checked against its published checksum.
|
|
var Recipes = map[string]Recipe{
|
|
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
|
|
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
|
|
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
|
|
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
|
|
}
|
|
|
|
// Installable are the engines this agent can install, as it tells the
|
|
// control plane: every recipe but uv, which is a tool for the others.
|
|
func Installable() []string {
|
|
var out []string
|
|
for name := range Recipes {
|
|
if name != "uv" {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// UVVersion is the uv that installs the Python engines.
|
|
const UVVersion = "0.12.20"
|
|
|
|
// pythonForEngines is the interpreter the Python engines run on. uv fetches
|
|
// it, so the node's own Python — or its absence — does not matter.
|
|
const pythonForEngines = "3.12"
|
|
|
|
// NotHere is the error a recipe gives for a node the engine is not meant
|
|
// for, as opposed to one it should run on and cannot: vLLM on a machine
|
|
// with no GPU is not a fault, and is not reported as one.
|
|
type NotHere struct{ Reason string }
|
|
|
|
func (e NotHere) Error() string { return e.Reason }
|
|
|
|
func chooseUV(version string, h discovery.Host) (Build, error) {
|
|
var arch string
|
|
switch h.Arch {
|
|
case "amd64":
|
|
arch = "x86_64"
|
|
case "arm64":
|
|
arch = "aarch64"
|
|
default:
|
|
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
|
|
}
|
|
return Build{
|
|
Variant: arch,
|
|
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
|
|
Binary: "uv",
|
|
}, nil
|
|
}
|
|
|
|
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
|
|
// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
|
|
const minDriverForCUDA13 = 580
|
|
|
|
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
|
|
// Volta, so a V100 cannot run these builds at all.
|
|
const minComputeForCUDA13 = 7.5
|
|
|
|
// cudaGate says whether a node can run an engine built on CUDA 13.
|
|
func cudaGate(engine string, h discovery.Host) error {
|
|
switch h.Arch {
|
|
case "amd64", "arm64":
|
|
default:
|
|
return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
|
|
}
|
|
if h.NVIDIADriver == "" {
|
|
return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
|
|
}
|
|
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 {
|
|
return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s",
|
|
engine, minDriverForCUDA13, h.NVIDIADriver)
|
|
}
|
|
if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 {
|
|
return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f",
|
|
engine, minComputeForCUDA13, cc)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Python engines need a C compiler at run time: Triton builds a small
|
|
// launcher for its kernels the first time a model loads, and without gcc
|
|
// that load fails long after the installation looked finished.
|
|
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
|
|
|
|
func chooseVLLM(version string, h discovery.Host) (Build, error) {
|
|
if err := cudaGate("vLLM", h); err != nil {
|
|
return Build{}, err
|
|
}
|
|
return Build{
|
|
Variant: "cu13",
|
|
Python: pythonForEngines,
|
|
Requirements: []string{"vllm==" + version},
|
|
Binary: "vllm",
|
|
Packages: compilerPackages,
|
|
FreeGB: 25,
|
|
}, nil
|
|
}
|
|
|
|
func chooseSGLang(version string, h discovery.Host) (Build, error) {
|
|
if err := cudaGate("SGLang", h); err != nil {
|
|
return Build{}, err
|
|
}
|
|
// SGLang is launched as a module, so what the agent is given is the
|
|
// environment's interpreter, and the proof is importing it.
|
|
return Build{
|
|
Variant: "cu13",
|
|
Python: pythonForEngines,
|
|
Requirements: []string{"sglang==" + version},
|
|
Binary: "python",
|
|
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
|
|
Packages: compilerPackages,
|
|
FreeGB: 25,
|
|
}, nil
|
|
}
|
|
|
|
// leadingInt is the number a version string starts with: 580 of "580.65.06".
|
|
func leadingInt(s string) int {
|
|
n := 0
|
|
for _, c := range strings.TrimSpace(s) {
|
|
if c < '0' || c > '9' {
|
|
break
|
|
}
|
|
n = n*10 + int(c-'0')
|
|
}
|
|
return n
|
|
}
|
|
|
|
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
|
|
// carries a matching cudart archive, so the node needs a driver and nothing
|
|
// else — no toolkit.
|
|
const cudaForLlamaCpp = "12.8"
|
|
|
|
func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
|
|
// The published Linux builds are built on Ubuntu, but nothing about them
|
|
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
|
|
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
|
|
// name decides nothing here. The build is tried everywhere, and ldd says
|
|
// whether this system can load it — in the words of the missing symbol.
|
|
var arch string
|
|
switch h.Arch {
|
|
case "amd64":
|
|
arch = "x64"
|
|
case "arm64":
|
|
arch = "arm64"
|
|
default:
|
|
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
|
|
}
|
|
|
|
b := Build{
|
|
Variant: "cpu",
|
|
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
|
|
// Everything else the build links is in the archive or in any current
|
|
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
|
|
// the server does not start without it.
|
|
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
|
|
Binary: "llama-server",
|
|
}
|
|
if arch == "x64" {
|
|
// The archive carries a backend per CPU generation and loads the best
|
|
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
|
|
b.CPUFlags = []string{"avx2"}
|
|
}
|
|
// The CUDA build is published for x64 at this version; on arm64 it is a
|
|
// newer CUDA that the driver may not support, so an arm64 GPU node runs
|
|
// the CPU build until someone decides otherwise.
|
|
if h.NVIDIADriver != "" && arch == "x64" {
|
|
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
|
|
b.Variant = "cuda-" + cudaForLlamaCpp
|
|
b.Assets = []string{
|
|
fmt.Sprintf("llama-%s-%s", version, name),
|
|
fmt.Sprintf("cudart-llama-%s-%s", version, name),
|
|
}
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func orUnknown(s string) string {
|
|
if strings.TrimSpace(s) == "" {
|
|
return "an unrecognised system"
|
|
}
|
|
return s
|
|
}
|