llama.cpp's CUDA runtime is unpacked beside the backend, and CUDA 13 follows the build
CI / test (push) Successful in 12m42s

The cudart archive unpacks into a directory of its own, while
libggml-cuda.so finds its libraries only through $ORIGIN, so the runtime
the release carries was never loaded: CUDA builds worked only on nodes
with a CUDA toolkit of the same major version installed. On cuda01
(CUDA 13.3 toolkit) b12606's CUDA 12.8 build saw no GPU and the install
fell through to Vulkan.

The CUDA 13 build's name also moved with llama.cpp's CI: 13.3 up to
b11070, 13.4 from b11071, 13.5 from b12170. A newer version asked for
the 13.3 archive, which does not exist.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-21 13:39:55 +01:00
1 parent 32d4d9c3af
commit 0ffd633bf6
5 files changed
+101 -10

No files matched your search

+2 -2
View File
@@ -787,7 +787,7 @@ Nothing to do: the agent installs them once the node joins, under
| Engine | On | How |
|---|---|---|
| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
| llama.cpp | every node | the release build for its processor and GPU — CUDA 13 (13.3, 13.4 or 13.5, whichever the build was published for), CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
| vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
| SGLang | only when you ask for it | the same way as vLLM |
| TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) |
@@ -3312,7 +3312,7 @@ That carries:
`--version` for another), with `checksums.txt` and its signature, checked
against the project's signing key as `ctl upgrade` checks it.
- **llama.cpp** at the version this llmgw installs, every build for each
architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked
architecture — CPU, CUDA 13 (cuda-13.3 up to b11070, cuda-13.4 to b12169, cuda-13.5 after), CUDA 12.8 (amd64), Vulkan — each checked
against the checksum GitHub publishes, the same check a node does online.
`--variant cpu --variant cuda-13.3` carries only those, for a site whose
hardware you know; `--engine llamacpp=b10991` names the version.
+38 -1
View File
@@ -17,6 +17,7 @@ import (
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"time"
@@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
return err
}
in.progress(st, "unpacking %s", asset)
if err := untarGz(archive, tmp); err != nil {
into := tmp
if slices.Contains(b.Beside, asset) {
into = filepath.Join(tmp, ".beside", asset)
if err := os.MkdirAll(into, 0o755); err != nil {
return err
}
}
if err := untarGz(archive, into); err != nil {
return fmt.Errorf("unpacking %s: %w", asset, err)
}
_ = os.Remove(archive)
got[asset] = want
}
if len(b.Beside) > 0 {
if err := moveBesideBinary(tmp, b.Binary); err != nil {
return err
}
}
in.progress(st, "checking that %s runs", b.Binary)
bin, _, err := in.verify(ctx, tmp, b)
@@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
return nil
}
// moveBesideBinary moves every file the Beside assets unpacked, under
// dir/.beside, into the directory the binary is in, by its own name: the
// directories the archive put them in are its packaging, not where they are
// loaded from. A file the build already has is kept.
func moveBesideBinary(dir, binary string) error {
staged := filepath.Join(dir, ".beside")
defer os.RemoveAll(staged) //nolint:errcheck // scratch
bin, err := findFile(dir, binary)
if err != nil {
return err
}
into := filepath.Dir(bin)
return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
to := filepath.Join(into, d.Name())
if _, err := os.Lstat(to); err == nil {
return nil
}
return os.Rename(p, to)
})
}
// verify finds the binary and runs it.
//
// Two checks, because they fail differently. ldd names a missing library
+29 -2
View File
@@ -13,6 +13,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
@@ -429,7 +430,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
rel := newRelease(map[string][]byte{
"llama-b1-" + cuda: tarGz(t, map[string]string{
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
// As the release packs it: under a directory of its own.
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil),
})
gpu := ubuntu
gpu.NVIDIADriver = "570.86.15"
@@ -446,7 +448,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
// The same build seeing its GPU is installed.
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
if got[0].State != Ready {
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
}
// The runtime is beside the backend, where its $ORIGIN runpath finds it,
// not in the directory its archive gave it.
if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil {
t.Errorf("the CUDA runtime is not beside llama-server: %v", err)
}
if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil {
t.Error("the staging directory was left behind")
}
}
func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) {
for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3",
"b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} {
if got := llamaCUDA13(v); got != want {
t.Errorf("%s: CUDA %s, want %s", v, got, want)
}
}
bs, err := Builds("llamacpp", "b12606", "amd64")
if err != nil {
t.Fatal(err)
}
if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") {
t.Errorf("b12606's CUDA 13 build: %+v", bs[1])
}
}
+31 -4
View File
@@ -12,6 +12,7 @@ package provision
import (
"fmt"
"slices"
"strconv"
"strings"
"time"
@@ -32,6 +33,11 @@ type Build struct {
// Binary is the executable's file name, found anywhere in the unpacked
// tree.
Binary string
// Beside are assets whose files belong in the directory Binary is in,
// wherever the archive puts them: llama.cpp's CUDA runtime, which its
// archive holds under a directory of its own, while the backend looks for
// it only beside itself ($ORIGIN).
Beside []string
// CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running.
CPUFlags []string
@@ -509,9 +515,25 @@ func leadingInt(s string) int {
// compile.
var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the
// archive's name carries it, and it moves on with the project's CI. Taken
// from the releases themselves (the first build of each).
func llamaCUDA13(version string) string {
n, err := strconv.Atoi(strings.TrimPrefix(version, "b"))
switch {
case err != nil:
return "13.3"
case n >= 12170:
return "13.5"
case n >= 11071:
return "13.4"
}
return "13.3"
}
// llamaCppCandidates are the llama.cpp builds for a machine.
//
// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580,
// On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580,
// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
// the GPUs it ships as PTX, 525 for those it ships as machine code), and
// Vulkan, which runs on any card whose driver has Vulkan — the generations
@@ -560,8 +582,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
cuda := func(cv string) Build {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
// The release carries the CUDA runtime beside the build, so the node
// needs a driver and nothing else — no toolkit.
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name))
// needs a driver and nothing else — no toolkit. Its archive unpacks
// into a directory of its own, where the backend never looks; Beside
// moves it next to the backend.
runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name)
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime)
b.Beside = []string{runtime}
b.GPUBackend = "libggml-cuda.so"
return b
}
@@ -573,7 +599,8 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
cuda12Driver = minDriverForCUDA12
}
cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}}
cv := llamaCUDA13(version)
cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}}
if arch == "x64" {
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
}
+1 -1
View File
@@ -131,7 +131,7 @@ func newBundleCreateCmd() *cobra.Command {
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)")
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3 (cuda-13.4 from llama.cpp b11071, cuda-13.5 from b12170), cuda-12.8, vulkan; repeatable (default: every build)")
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
f.StringVar(&registry, "registry", "", "the model registry, when not the public Hugging Face")