Files
llmgw/internal/agent/provision/recipe.go
T
Ilker ManapandClaude Opus 5.5 f5d6210b86 Install engines on any glibc system, with its own package manager
The llama.cpp build is published from Ubuntu but is not Ubuntu's in any way
that matters: it needs glibc 2.34 and GCC 12's libstdc++, which Debian 12
and RHEL 10 have and RHEL 9 does not. Refusing every system not called
"ubuntu" turned away the first two for their name. The build is now tried
everywhere, and ldd decides.

What ldd says is made readable. "/lib64/libstdc++.so.6: version
`GLIBCXX_3.4.30' not found" is a system too old for the build, and is
reported as "libstdc++.so.6 is too old (needs GLIBCXX_3.4.30)", once —
rather than as a missing library that is plainly installed.

Packages are named per family — libgomp1 for Debian's, libgomp for Red
Hat's — and installed with whichever of apt-get, dnf, yum or zypper the
machine has, decided from ID_LIKE rather than a list of distribution names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-09-28 11:10:06 +02:00

110 lines
4.0 KiB
Go

// Package provision installs the engines a node is asked to have.
//
// The control plane says which engines at which versions; everything else is
// worked out here, on the node, because only the node knows it: which build
// fits its processor and driver, what its operating system is missing, and
// whether the result actually runs. Nothing already on the machine is taken
// on trust. An engine counts as installed when this package installed it,
// checked every file against the checksum its publisher gave, and watched it
// start.
package provision
import (
"fmt"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
)
// Build is one published build of an engine, chosen for one machine.
type Build struct {
// Variant names the build: "cpu", "cuda-12.8".
Variant string
// Assets are the release files to fetch, unpacked in order into one
// directory.
Assets []string
// Packages are operating system packages the build links against and a
// minimal install may not carry, by packaging family ("deb", "rpm"): the
// same library is libgomp1 on one and libgomp on the other.
Packages map[string][]string
// Binary is the executable's file name, found anywhere in the unpacked
// tree.
Binary string
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
}
// Recipe knows how to install one engine.
type Recipe struct {
// Repo is the GitHub repository whose releases carry the builds.
Repo string
// Choose picks the build for a machine, or says why none fits.
Choose func(version string, h discovery.Host) (Build, error)
}
// Recipes are the engines this agent can install.
//
// vLLM and SGLang are not here yet: they are Python environments of several
// gigabytes rather than an archive, and are installed by hand and named in the
// agent's configuration until they are.
var Recipes = map[string]Recipe{
"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
}
// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
// carries a matching cudart archive, so the node needs a driver and nothing
// else — no toolkit.
const cudaForLlamaCpp = "12.8"
func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
// The published Linux builds are built on Ubuntu, but nothing about them
// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
// name decides nothing here. The build is tried everywhere, and ldd says
// whether this system can load it — in the words of the missing symbol.
var arch string
switch h.Arch {
case "amd64":
arch = "x64"
case "arm64":
arch = "arm64"
default:
return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
}
b := Build{
Variant: "cpu",
Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
// Everything else the build links is in the archive or in any current
// glibc system; OpenMP's runtime is the one a cloud image lacks, and
// the server does not start without it.
Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
Binary: "llama-server",
}
if arch == "x64" {
// The archive carries a backend per CPU generation and loads the best
// one, down to SSE4.2. Below AVX2 it runs, several times slower.
b.CPUFlags = []string{"avx2"}
}
// The CUDA build is published for x64 at this version; on arm64 it is a
// newer CUDA that the driver may not support, so an arm64 GPU node runs
// the CPU build until someone decides otherwise.
if h.NVIDIADriver != "" && arch == "x64" {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
b.Variant = "cuda-" + cudaForLlamaCpp
b.Assets = []string{
fmt.Sprintf("llama-%s-%s", version, name),
fmt.Sprintf("cudart-llama-%s-%s", version, name),
}
}
return b, nil
}
func orUnknown(s string) string {
if strings.TrimSpace(s) == "" {
return "an unrecognised system"
}
return s
}