From 0ffd633bf63c0daf6d1ce8ef67c43aeb011b392f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 21 Nov 2026 13:39:55 +0100 Subject: [PATCH] llama.cpp's CUDA runtime is unpacked beside the backend, and CUDA 13 follows the build The cudart archive unpacks into a directory of its own, while libggml-cuda.so finds its libraries only through $ORIGIN, so the runtime the release carries was never loaded: CUDA builds worked only on nodes with a CUDA toolkit of the same major version installed. On cuda01 (CUDA 13.3 toolkit) b12606's CUDA 12.8 build saw no GPU and the install fell through to Vulkan. The CUDA 13 build's name also moved with llama.cpp's CI: 13.3 up to b11070, 13.4 from b11071, 13.5 from b12170. A newer version asked for the 13.3 archive, which does not exist. Co-Authored-By: Claude Opus 5.5 --- docs/install.md | 4 +-- internal/agent/provision/install.go | 39 +++++++++++++++++++++++- internal/agent/provision/install_test.go | 31 +++++++++++++++++-- internal/agent/provision/recipe.go | 35 ++++++++++++++++++--- internal/cli/bundle.go | 2 +- 5 files changed, 101 insertions(+), 10 deletions(-) diff --git a/docs/install.md b/docs/install.md index b23c1e9..f88bfc6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -787,7 +787,7 @@ Nothing to do: the agent installs them once the node joins, under | Engine | On | How | |---|---|---| -| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU | +| llama.cpp | every node | the release build for its processor and GPU — CUDA 13 (13.3, 13.4 or 13.5, whichever the build was published for), CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU | | vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB | | SGLang | only when you ask for it | the same way as vLLM | | TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) | @@ -3312,7 +3312,7 @@ That carries: `--version` for another), with `checksums.txt` and its signature, checked against the project's signing key as `ctl upgrade` checks it. - **llama.cpp** at the version this llmgw installs, every build for each - architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked + architecture — CPU, CUDA 13 (cuda-13.3 up to b11070, cuda-13.4 to b12169, cuda-13.5 after), CUDA 12.8 (amd64), Vulkan — each checked against the checksum GitHub publishes, the same check a node does online. `--variant cpu --variant cuda-13.3` carries only those, for a site whose hardware you know; `--engine llamacpp=b10991` names the version. diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 12a24e0..8a3597e 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -17,6 +17,7 @@ import ( "os/exec" "path/filepath" "regexp" + "slices" "strings" "time" @@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir return err } in.progress(st, "unpacking %s", asset) - if err := untarGz(archive, tmp); err != nil { + into := tmp + if slices.Contains(b.Beside, asset) { + into = filepath.Join(tmp, ".beside", asset) + if err := os.MkdirAll(into, 0o755); err != nil { + return err + } + } + if err := untarGz(archive, into); err != nil { return fmt.Errorf("unpacking %s: %w", asset, err) } _ = os.Remove(archive) got[asset] = want } + if len(b.Beside) > 0 { + if err := moveBesideBinary(tmp, b.Binary); err != nil { + return err + } + } in.progress(st, "checking that %s runs", b.Binary) bin, _, err := in.verify(ctx, tmp, b) @@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir return nil } +// moveBesideBinary moves every file the Beside assets unpacked, under +// dir/.beside, into the directory the binary is in, by its own name: the +// directories the archive put them in are its packaging, not where they are +// loaded from. A file the build already has is kept. +func moveBesideBinary(dir, binary string) error { + staged := filepath.Join(dir, ".beside") + defer os.RemoveAll(staged) //nolint:errcheck // scratch + bin, err := findFile(dir, binary) + if err != nil { + return err + } + into := filepath.Dir(bin) + return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + to := filepath.Join(into, d.Name()) + if _, err := os.Lstat(to); err == nil { + return nil + } + return os.Rename(p, to) + }) +} + // verify finds the binary and runs it. // // Two checks, because they fail differently. ldd names a missing library diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index ca9daae..9161b0a 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -429,7 +430,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) { rel := newRelease(map[string][]byte{ "llama-b1-" + cuda: tarGz(t, map[string]string{ "llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil), - "cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil), + // As the release packs it: under a directory of its own. + "cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil), }) gpu := ubuntu gpu.NVIDIADriver = "570.86.15" @@ -446,7 +448,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) { // The same build seeing its GPU is installed. sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" - if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready { + got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Ready { t.Fatalf("a CUDA build that sees its GPU: %+v", got[0]) } + // The runtime is beside the backend, where its $ORIGIN runpath finds it, + // not in the directory its archive gave it. + if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil { + t.Errorf("the CUDA runtime is not beside llama-server: %v", err) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil { + t.Error("the staging directory was left behind") + } +} + +func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) { + for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3", + "b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} { + if got := llamaCUDA13(v); got != want { + t.Errorf("%s: CUDA %s, want %s", v, got, want) + } + } + bs, err := Builds("llamacpp", "b12606", "amd64") + if err != nil { + t.Fatal(err) + } + if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") { + t.Errorf("b12606's CUDA 13 build: %+v", bs[1]) + } } diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index e7f7156..5e80ff4 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -12,6 +12,7 @@ package provision import ( "fmt" "slices" + "strconv" "strings" "time" @@ -32,6 +33,11 @@ type Build struct { // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string + // Beside are assets whose files belong in the directory Binary is in, + // wherever the archive puts them: llama.cpp's CUDA runtime, which its + // archive holds under a directory of its own, while the backend looks for + // it only beside itself ($ORIGIN). + Beside []string // CPUFlags are extensions the build is slow without. Missing ones are // reported, not refused: slow is still running. CPUFlags []string @@ -509,9 +515,25 @@ func leadingInt(s string) int { // compile. var llamaCUDA12Real = []float64{8.6, 8.9, 12.0} +// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the +// archive's name carries it, and it moves on with the project's CI. Taken +// from the releases themselves (the first build of each). +func llamaCUDA13(version string) string { + n, err := strconv.Atoi(strings.TrimPrefix(version, "b")) + switch { + case err != nil: + return "13.3" + case n >= 12170: + return "13.5" + case n >= 11071: + return "13.4" + } + return "13.3" +} + // llamaCppCandidates are the llama.cpp builds for a machine. // -// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580, +// On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580, // Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for // the GPUs it ships as PTX, 525 for those it ships as machine code), and // Vulkan, which runs on any card whose driver has Vulkan — the generations @@ -560,8 +582,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err cuda := func(cv string) Build { name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch) // The release carries the CUDA runtime beside the build, so the node - // needs a driver and nothing else — no toolkit. - b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name)) + // needs a driver and nothing else — no toolkit. Its archive unpacks + // into a directory of its own, where the backend never looks; Beside + // moves it next to the backend. + runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name) + b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime) + b.Beside = []string{runtime} b.GPUBackend = "libggml-cuda.so" return b } @@ -573,7 +599,8 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) { cuda12Driver = minDriverForCUDA12 } - cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}} + cv := llamaCUDA13(version) + cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}} if arch == "x64" { cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"}) } diff --git a/internal/cli/bundle.go b/internal/cli/bundle.go index fc03e53..26ea2e9 100644 --- a/internal/cli/bundle.go +++ b/internal/cli/bundle.go @@ -131,7 +131,7 @@ func newBundleCreateCmd() *cobra.Command { f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published") f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable") f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)") - f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)") + f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3 (cuda-13.4 from llama.cpp b11071, cuda-13.5 from b12170), cuda-12.8, vulkan; repeatable (default: every build)") f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable") f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves") f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face")