llama.cpp whose CUDA backend cannot load runs on the CPU without an error. The CUDA build is now asked for its devices when installed, a shard on GPUs must hold memory on each before it is ready, and engine-check reports the devices. Shards get CUDA_DEVICE_ORDER=PCI_BUS_ID so CUDA numbers GPUs as placement does. Refs #2 (item 2). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
257 lines
9.0 KiB
Go
257 lines
9.0 KiB
Go
// Package provision installs the engines a node is asked to have.
|
|
//
|
|
// The control plane says which engines at which versions; everything else is
|
|
// worked out here, on the node, because only the node knows it: which build
|
|
// fits its processor and driver, what its operating system is missing, and
|
|
// whether the result actually runs. Nothing already on the machine is taken
|
|
// on trust. An engine counts as installed when this package installed it,
|
|
// checked every file against the checksum its publisher gave, and watched it
|
|
// start.
|
|
package provision
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
)
|
|
|
|
// Build is one published build of an engine, chosen for one machine.
|
|
type Build struct {
|
|
// Variant names the build: "cpu", "cuda-12.8".
|
|
Variant string
|
|
// Assets are the release files to fetch, unpacked in order into one
|
|
// directory.
|
|
Assets []string
|
|
// Packages are operating system packages the build links against and a
|
|
// minimal install may not carry, by packaging family ("deb", "rpm"): the
|
|
// same library is libgomp1 on one and libgomp on the other.
|
|
Packages map[string][]string
|
|
// Binary is the executable's file name, found anywhere in the unpacked
|
|
// tree.
|
|
Binary string
|
|
// CPUFlags are extensions the build is slow without. Missing ones are
|
|
// reported, not refused: slow is still running.
|
|
CPUFlags []string
|
|
|
|
// Python, when set, makes the build a Python environment rather than an
|
|
// archive: that interpreter version, with Requirements installed from
|
|
// PyPI by uv. Binary is then looked for in the environment's bin.
|
|
Python string
|
|
Requirements []string
|
|
// VersionArgs is what Binary is run with to prove the build works, when
|
|
// --version is not it.
|
|
VersionArgs []string
|
|
// GPUBackend, when set, is the shared library the build computes on GPUs
|
|
// with, loaded at run time rather than linked: "libggml-cuda.so". The
|
|
// build must list a device with it, or it is not installed — a llama.cpp
|
|
// whose CUDA backend cannot load runs on the CPU without complaint.
|
|
GPUBackend string
|
|
// FreeGB is the disk space the installation needs before it starts: a
|
|
// Python engine is several gigabytes, and one that fills the disk halfway
|
|
// leaves the node worse off than one that never began.
|
|
FreeGB int
|
|
}
|
|
|
|
// Recipe knows how to install one engine.
|
|
type Recipe struct {
|
|
// Repo is the GitHub repository whose releases carry the builds; empty
|
|
// for an engine installed from PyPI.
|
|
Repo string
|
|
// PyPI is the project an engine installed from PyPI is published as.
|
|
PyPI string
|
|
// Choose picks the build for a machine, or says why none fits.
|
|
Choose func(version string, h discovery.Host) (Build, error)
|
|
}
|
|
|
|
// Recipes are the engines this agent can install.
|
|
//
|
|
// vLLM and SGLang are Python environments of several gigabytes, installed
|
|
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
|
|
// checked against its published checksum.
|
|
var Recipes = map[string]Recipe{
|
|
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
|
|
"vllm": {PyPI: "vllm", Choose: chooseVLLM},
|
|
"sglang": {PyPI: "sglang", Choose: chooseSGLang},
|
|
"uv": {Repo: "astral-sh/uv", Choose: chooseUV},
|
|
}
|
|
|
|
// Installable are the engines this agent can install, as it tells the
|
|
// control plane: every recipe but uv, which is a tool for the others.
|
|
func Installable() []string {
|
|
var out []string
|
|
for name := range Recipes {
|
|
if name != "uv" {
|
|
out = append(out, name)
|
|
}
|
|
}
|
|
slices.Sort(out)
|
|
return out
|
|
}
|
|
|
|
// UVVersion is the uv that installs the Python engines.
|
|
const UVVersion = "0.12.20"
|
|
|
|
// pythonForEngines is the interpreter the Python engines run on. uv fetches
|
|
// it, so the node's own Python — or its absence — does not matter.
|
|
const pythonForEngines = "3.12"
|
|
|
|
// NotHere is the error a recipe gives for a node the engine is not meant
|
|
// for, as opposed to one it should run on and cannot: vLLM on a machine
|
|
// with no GPU is not a fault, and is not reported as one.
|
|
type NotHere struct{ Reason string }
|
|
|
|
func (e NotHere) Error() string { return e.Reason }
|
|
|
|
func chooseUV(version string, h discovery.Host) (Build, error) {
|
|
var arch string
|
|
switch h.Arch {
|
|
case "amd64":
|
|
arch = "x86_64"
|
|
case "arm64":
|
|
arch = "aarch64"
|
|
default:
|
|
return Build{}, fmt.Errorf("uv publishes no Linux build for %s", orUnknown(h.Arch))
|
|
}
|
|
return Build{
|
|
Variant: arch,
|
|
Assets: []string{"uv-" + arch + "-unknown-linux-gnu.tar.gz"},
|
|
Binary: "uv",
|
|
}, nil
|
|
}
|
|
|
|
// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
|
|
// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
|
|
const minDriverForCUDA13 = 580
|
|
|
|
// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
|
|
// Volta, so a V100 cannot run these builds at all.
|
|
const minComputeForCUDA13 = 7.5
|
|
|
|
// cudaGate says whether a node can run an engine built on CUDA 13.
|
|
func cudaGate(engine string, h discovery.Host) error {
|
|
switch h.Arch {
|
|
case "amd64", "arm64":
|
|
default:
|
|
return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
|
|
}
|
|
if h.NVIDIADriver == "" {
|
|
return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
|
|
}
|
|
if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 {
|
|
return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s",
|
|
engine, minDriverForCUDA13, h.NVIDIADriver)
|
|
}
|
|
if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 {
|
|
return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f",
|
|
engine, minComputeForCUDA13, cc)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Python engines need a C compiler at run time: Triton builds a small
|
|
// launcher for its kernels the first time a model loads, and without gcc
|
|
// that load fails long after the installation looked finished.
|
|
var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
|
|
|
|
func chooseVLLM(version string, h discovery.Host) (Build, error) {
|
|
if err := cudaGate("vLLM", h); err != nil {
|
|
return Build{}, err
|
|
}
|
|
return Build{
|
|
Variant: "cu13",
|
|
Python: pythonForEngines,
|
|
Requirements: []string{"vllm==" + version},
|
|
Binary: "vllm",
|
|
Packages: compilerPackages,
|
|
FreeGB: 25,
|
|
}, nil
|
|
}
|
|
|
|
func chooseSGLang(version string, h discovery.Host) (Build, error) {
|
|
if err := cudaGate("SGLang", h); err != nil {
|
|
return Build{}, err
|
|
}
|
|
// SGLang is launched as a module, so what the agent is given is the
|
|
// environment's interpreter, and the proof is importing it.
|
|
return Build{
|
|
Variant: "cu13",
|
|
Python: pythonForEngines,
|
|
Requirements: []string{"sglang==" + version},
|
|
Binary: "python",
|
|
VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
|
|
Packages: compilerPackages,
|
|
FreeGB: 25,
|
|
}, nil
|
|
}
|
|
|
|
// leadingInt is the number a version string starts with: 580 of "580.65.06".
|
|
func leadingInt(s string) int {
|
|
n := 0
|
|
for _, c := range strings.TrimSpace(s) {
|
|
if c < '0' || c > '9' {
|
|
break
|
|
}
|
|
n = n*10 + int(c-'0')
|
|
}
|
|
return n
|
|
}
|
|
|
|
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
|
|
// carries a matching cudart archive, so the node needs a driver and nothing
|
|
// else — no toolkit.
|
|
const cudaForLlamaCpp = "12.8"
|
|
|
|
func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
|
|
// The published Linux builds are built on Ubuntu, but nothing about them
|
|
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
|
|
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
|
|
// name decides nothing here. The build is tried everywhere, and ldd says
|
|
// whether this system can load it — in the words of the missing symbol.
|
|
var arch string
|
|
switch h.Arch {
|
|
case "amd64":
|
|
arch = "x64"
|
|
case "arm64":
|
|
arch = "arm64"
|
|
default:
|
|
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
|
|
}
|
|
|
|
b := Build{
|
|
Variant: "cpu",
|
|
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
|
|
// Everything else the build links is in the archive or in any current
|
|
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
|
|
// the server does not start without it.
|
|
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
|
|
Binary: "llama-server",
|
|
}
|
|
if arch == "x64" {
|
|
// The archive carries a backend per CPU generation and loads the best
|
|
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
|
|
b.CPUFlags = []string{"avx2"}
|
|
}
|
|
// The CUDA build is published for x64 at this version; on arm64 it is a
|
|
// newer CUDA that the driver may not support, so an arm64 GPU node runs
|
|
// the CPU build until someone decides otherwise.
|
|
if h.NVIDIADriver != "" && arch == "x64" {
|
|
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
|
|
b.Variant = "cuda-" + cudaForLlamaCpp
|
|
b.GPUBackend = "libggml-cuda.so"
|
|
b.Assets = []string{
|
|
fmt.Sprintf("llama-%s-%s", version, name),
|
|
fmt.Sprintf("cudart-llama-%s-%s", version, name),
|
|
}
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func orUnknown(s string) string {
|
|
if strings.TrimSpace(s) == "" {
|
|
return "an unrecognised system"
|
|
}
|
|
return s
|
|
}
|