llama.cpp's CUDA runtime is unpacked beside the backend, and CUDA 13 follows the build
CI / test (push) Successful in 12m42s
CI / test (push) Successful in 12m42s
The cudart archive unpacks into a directory of its own, while libggml-cuda.so finds its libraries only through $ORIGIN, so the runtime the release carries was never loaded: CUDA builds worked only on nodes with a CUDA toolkit of the same major version installed. On cuda01 (CUDA 13.3 toolkit) b12606's CUDA 12.8 build saw no GPU and the install fell through to Vulkan. The CUDA 13 build's name also moved with llama.cpp's CI: 13.3 up to b11070, 13.4 from b11071, 13.5 from b12170. A newer version asked for the 13.3 archive, which does not exist. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
32d4d9c3af
commit
0ffd633bf6
5 files changed
+101
-10
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
return err
|
||||
}
|
||||
in.progress(st, "unpacking %s", asset)
|
||||
if err := untarGz(archive, tmp); err != nil {
|
||||
into := tmp
|
||||
if slices.Contains(b.Beside, asset) {
|
||||
into = filepath.Join(tmp, ".beside", asset)
|
||||
if err := os.MkdirAll(into, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := untarGz(archive, into); err != nil {
|
||||
return fmt.Errorf("unpacking %s: %w", asset, err)
|
||||
}
|
||||
_ = os.Remove(archive)
|
||||
got[asset] = want
|
||||
}
|
||||
if len(b.Beside) > 0 {
|
||||
if err := moveBesideBinary(tmp, b.Binary); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
in.progress(st, "checking that %s runs", b.Binary)
|
||||
bin, _, err := in.verify(ctx, tmp, b)
|
||||
@@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveBesideBinary moves every file the Beside assets unpacked, under
|
||||
// dir/.beside, into the directory the binary is in, by its own name: the
|
||||
// directories the archive put them in are its packaging, not where they are
|
||||
// loaded from. A file the build already has is kept.
|
||||
func moveBesideBinary(dir, binary string) error {
|
||||
staged := filepath.Join(dir, ".beside")
|
||||
defer os.RemoveAll(staged) //nolint:errcheck // scratch
|
||||
bin, err := findFile(dir, binary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
into := filepath.Dir(bin)
|
||||
return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
to := filepath.Join(into, d.Name())
|
||||
if _, err := os.Lstat(to); err == nil {
|
||||
return nil
|
||||
}
|
||||
return os.Rename(p, to)
|
||||
})
|
||||
}
|
||||
|
||||
// verify finds the binary and runs it.
|
||||
//
|
||||
// Two checks, because they fail differently. ldd names a missing library
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -429,7 +430,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
|
||||
rel := newRelease(map[string][]byte{
|
||||
"llama-b1-" + cuda: tarGz(t, map[string]string{
|
||||
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
|
||||
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
|
||||
// As the release packs it: under a directory of its own.
|
||||
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil),
|
||||
})
|
||||
gpu := ubuntu
|
||||
gpu.NVIDIADriver = "570.86.15"
|
||||
@@ -446,7 +448,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
|
||||
|
||||
// The same build seeing its GPU is installed.
|
||||
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
|
||||
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
|
||||
got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
|
||||
if got[0].State != Ready {
|
||||
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
|
||||
}
|
||||
// The runtime is beside the backend, where its $ORIGIN runpath finds it,
|
||||
// not in the directory its archive gave it.
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil {
|
||||
t.Errorf("the CUDA runtime is not beside llama-server: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil {
|
||||
t.Error("the staging directory was left behind")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) {
|
||||
for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3",
|
||||
"b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} {
|
||||
if got := llamaCUDA13(v); got != want {
|
||||
t.Errorf("%s: CUDA %s, want %s", v, got, want)
|
||||
}
|
||||
}
|
||||
bs, err := Builds("llamacpp", "b12606", "amd64")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") {
|
||||
t.Errorf("b12606's CUDA 13 build: %+v", bs[1])
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ package provision
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -32,6 +33,11 @@ type Build struct {
|
||||
// Binary is the executable's file name, found anywhere in the unpacked
|
||||
// tree.
|
||||
Binary string
|
||||
// Beside are assets whose files belong in the directory Binary is in,
|
||||
// wherever the archive puts them: llama.cpp's CUDA runtime, which its
|
||||
// archive holds under a directory of its own, while the backend looks for
|
||||
// it only beside itself ($ORIGIN).
|
||||
Beside []string
|
||||
// CPUFlags are extensions the build is slow without. Missing ones are
|
||||
// reported, not refused: slow is still running.
|
||||
CPUFlags []string
|
||||
@@ -509,9 +515,25 @@ func leadingInt(s string) int {
|
||||
// compile.
|
||||
var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
|
||||
|
||||
// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the
|
||||
// archive's name carries it, and it moves on with the project's CI. Taken
|
||||
// from the releases themselves (the first build of each).
|
||||
func llamaCUDA13(version string) string {
|
||||
n, err := strconv.Atoi(strings.TrimPrefix(version, "b"))
|
||||
switch {
|
||||
case err != nil:
|
||||
return "13.3"
|
||||
case n >= 12170:
|
||||
return "13.5"
|
||||
case n >= 11071:
|
||||
return "13.4"
|
||||
}
|
||||
return "13.3"
|
||||
}
|
||||
|
||||
// llamaCppCandidates are the llama.cpp builds for a machine.
|
||||
//
|
||||
// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580,
|
||||
// On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580,
|
||||
// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
|
||||
// the GPUs it ships as PTX, 525 for those it ships as machine code), and
|
||||
// Vulkan, which runs on any card whose driver has Vulkan — the generations
|
||||
@@ -560,8 +582,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
|
||||
cuda := func(cv string) Build {
|
||||
name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
|
||||
// The release carries the CUDA runtime beside the build, so the node
|
||||
// needs a driver and nothing else — no toolkit.
|
||||
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name))
|
||||
// needs a driver and nothing else — no toolkit. Its archive unpacks
|
||||
// into a directory of its own, where the backend never looks; Beside
|
||||
// moves it next to the backend.
|
||||
runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name)
|
||||
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime)
|
||||
b.Beside = []string{runtime}
|
||||
b.GPUBackend = "libggml-cuda.so"
|
||||
return b
|
||||
}
|
||||
@@ -573,7 +599,8 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
|
||||
if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
|
||||
cuda12Driver = minDriverForCUDA12
|
||||
}
|
||||
cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}}
|
||||
cv := llamaCUDA13(version)
|
||||
cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}}
|
||||
if arch == "x64" {
|
||||
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user