// Package provision installs the engines a node is asked to have. // // The control plane says which engines at which versions; everything else is // worked out here, on the node, because only the node knows it: which build // fits its processor and driver, what its operating system is missing, and // whether the result actually runs. Nothing already on the machine is taken // on trust. An engine counts as installed when this package installed it, // checked every file against the checksum its publisher gave, and watched it // start. package provision import ( "fmt" "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" ) // Build is one published build of an engine, chosen for one machine. type Build struct { // Variant names the build: "cpu", "cuda-12.8". Variant string // Assets are the release files to fetch, unpacked in order into one // directory. Assets []string // Packages are operating system packages the build links against and a // minimal install may not carry, by packaging family ("deb", "rpm"): the // same library is libgomp1 on one and libgomp on the other. Packages map[string][]string // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string // CPUFlags are extensions the build is slow without. Missing ones are // reported, not refused: slow is still running. CPUFlags []string } // Recipe knows how to install one engine. type Recipe struct { // Repo is the GitHub repository whose releases carry the builds. Repo string // Choose picks the build for a machine, or says why none fits. Choose func(version string, h discovery.Host) (Build, error) } // Recipes are the engines this agent can install. // // vLLM and SGLang are not here yet: they are Python environments of several // gigabytes rather than an archive, and are installed by hand and named in the // agent's configuration until they are. var Recipes = map[string]Recipe{ "llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp}, } // cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also // carries a matching cudart archive, so the node needs a driver and nothing // else — no toolkit. const cudaForLlamaCpp = "12.8" func chooseLlamaCpp(version string, h discovery.Host) (Build, error) { // The published Linux builds are built on Ubuntu, but nothing about them // is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which // Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's // name decides nothing here. The build is tried everywhere, and ldd says // whether this system can load it — in the words of the missing symbol. var arch string switch h.Arch { case "amd64": arch = "x64" case "arm64": arch = "arm64" default: return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch)) } b := Build{ Variant: "cpu", Assets: []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)}, // Everything else the build links is in the archive or in any current // glibc system; OpenMP's runtime is the one a cloud image lacks, and // the server does not start without it. Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}}, Binary: "llama-server", } if arch == "x64" { // The archive carries a backend per CPU generation and loads the best // one, down to SSE4.2. Below AVX2 it runs, several times slower. b.CPUFlags = []string{"avx2"} } // The CUDA build is published for x64 at this version; on arm64 it is a // newer CUDA that the driver may not support, so an arm64 GPU node runs // the CPU build until someone decides otherwise. if h.NVIDIADriver != "" && arch == "x64" { name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp) b.Variant = "cuda-" + cudaForLlamaCpp b.Assets = []string{ fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name), } } return b, nil } func orUnknown(s string) string { if strings.TrimSpace(s) == "" { return "an unrecognised system" } return s }