// Package provision installs the engines a node is asked to have. // // The control plane says which engines at which versions; everything else is // worked out here, on the node, because only the node knows it: which build // fits its processor and driver, what its operating system is missing, and // whether the result actually runs. Nothing already on the machine is taken // on trust. An engine counts as installed when this package installed it, // checked every file against the checksum its publisher gave, and watched it // start. package provision import ( "fmt" "slices" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) // Build is one published build of an engine, chosen for one machine. type Build struct { // Variant names the build: "cpu", "cuda-12.8". Variant string // Assets are the release files to fetch, unpacked in order into one // directory. Assets []string // Packages are operating system packages the build links against and a // minimal install may not carry, by packaging family ("deb", "rpm"): the // same library is libgomp1 on one and libgomp on the other. Packages map[string][]string // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string // CPUFlags are extensions the build is slow without. Missing ones are // reported, not refused: slow is still running. CPUFlags []string // Python, when set, makes the build a Python environment rather than an // archive: that interpreter version, with Requirements installed from // PyPI by uv. Binary is then looked for in the environment's bin. Python string Requirements []string // VersionArgs is what Binary is run with to prove the build works, when // --version is not it. VersionArgs []string // GPUBackend, when set, is the shared library the build computes on GPUs // with, loaded at run time rather than linked: "libggml-cuda.so". The // build must list a device with it, or it is not installed — a llama.cpp // whose CUDA backend cannot load runs on the CPU without complaint. GPUBackend string // FreeGB is the disk space the installation needs before it starts: a // Python engine is several gigabytes, and one that fills the disk halfway // leaves the node worse off than one that never began. FreeGB int } // Recipe knows how to install one engine. type Recipe struct { // Repo is the GitHub repository whose releases carry the builds; empty // for an engine installed from PyPI. Repo string // PyPI is the project an engine installed from PyPI is published as. PyPI string // Choose picks the build for a machine, or says why none fits. Choose func(version string, h discovery.Host) (Build, error) } // Recipes are the engines this agent can install. // // vLLM and SGLang are Python environments of several gigabytes, installed // from PyPI by uv, which is itself installed like llama.cpp: a GitHub release // checked against its published checksum. var Recipes = map[string]Recipe{ "llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp}, "vllm": {PyPI: "vllm", Choose: chooseVLLM}, "sglang": {PyPI: "sglang", Choose: chooseSGLang}, "uv": {Repo: "astral-sh/uv", Choose: chooseUV}, } // Installable are the engines this agent can install, as it tells the // control plane: every recipe but uv, which is a tool for the others. func Installable() []string { var out []string for name := range Recipes { if name != "uv" { out = append(out, name) } } slices.Sort(out) return out } // UVVersion is the uv that installs the Python engines. const UVVersion = "0.12.20" // pythonForEngines is the interpreter the Python engines run on. uv fetches // it, so the node's own Python — or its absence — does not matter. const pythonForEngines = "3.12" // NotHere is the error a recipe gives for a node the engine is not meant // for, as opposed to one it should run on and cannot: vLLM on a machine // with no GPU is not a fault, and is not reported as one. type NotHere struct{ Reason string } func (e NotHere) Error() string { return e.Reason } func chooseUV(version string, h discovery.Host) (Build, error) { var arch string switch h.Arch { case "amd64": arch = "x86_64" case "arm64": arch = "aarch64" default: return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch)) } return Build{ Variant: arch, Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"}, Binary: "uv", }, nil } // minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM // since 0.26 and SGLang 0.5 is built on torch for CUDA 13. const minDriverForCUDA13 = 580 // minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped // Volta, so a V100 cannot run these builds at all. const minComputeForCUDA13 = 7.5 // cudaGate says whether a node can run an engine built on CUDA 13. func cudaGate(engine string, h discovery.Host) error { switch h.Arch { case "amd64", "arm64": default: return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch)) } if h.NVIDIADriver == "" { return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"} } if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 { return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s", engine, minDriverForCUDA13, h.NVIDIADriver) } if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 { return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f", engine, minComputeForCUDA13, cc) } return nil } // Python engines need a C compiler at run time: Triton builds a small // launcher for its kernels the first time a model loads, and without gcc // that load fails long after the installation looked finished. var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}} func chooseVLLM(version string, h discovery.Host) (Build, error) { if err := cudaGate("vLLM", h); err != nil { return Build{}, err } return Build{ Variant: "cu13", Python: pythonForEngines, Requirements: []string{"vllm==" + version}, Binary: "vllm", Packages: compilerPackages, FreeGB: 25, }, nil } func chooseSGLang(version string, h discovery.Host) (Build, error) { if err := cudaGate("SGLang", h); err != nil { return Build{}, err } // SGLang is launched as a module, so what the agent is given is the // environment's interpreter, and the proof is importing it. return Build{ Variant: "cu13", Python: pythonForEngines, Requirements: []string{"sglang==" + version}, Binary: "python", VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"}, Packages: compilerPackages, FreeGB: 25, }, nil } // leadingInt is the number a version string starts with: 580 of "580.65.06". func leadingInt(s string) int { n := 0 for _, c := range strings.TrimSpace(s) { if c < '0' || c > '9' { break } n = n*10 + int(c-'0') } return n } // cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also // carries a matching cudart archive, so the node needs a driver and nothing // else — no toolkit. const cudaForLlamaCpp = "12.8" func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { // The published Linux builds are built on Ubuntu, but nothing about them // is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which // Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's // name decides nothing here. The build is tried everywhere, and ldd says // whether this system can load it — in the words of the missing symbol. var arch string switch h.Arch { case "amd64": arch = "x64" case "arm64": arch = "arm64" default: return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch)) } b := Build{ Variant: "cpu", Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)}, // Everything else the build links is in the archive or in any current // glibc system; OpenMP's runtime is the one a cloud image lacks, and // the server does not start without it. Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}}, Binary: "llama-server", } if arch == "x64" { // The archive carries a backend per CPU generation and loads the best // one, down to SSE4.2. Below AVX2 it runs, several times slower. b.CPUFlags = []string{"avx2"} } // The CUDA build is published for x64 at this version; on arm64 it is a // newer CUDA that the driver may not support, so an arm64 GPU node runs // the CPU build until someone decides otherwise. if h.NVIDIADriver != "" && arch == "x64" { name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp) b.Variant = "cuda-" + cudaForLlamaCpp b.GPUBackend = "libggml-cuda.so" b.Assets = []string{ fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name), } } return b, nil } func orUnknown(s string) string { if strings.TrimSpace(s) == "" { return "an unrecognised system" } return s }