diff --git a/docs/install.md b/docs/install.md index b23c1e9..f88bfc6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -787,7 +787,7 @@ Nothing to do: the agent installs them once the node joins, under | Engine | On | How | |---|---|---| -| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU | +| llama.cpp | every node | the release build for its processor and GPU — CUDA 13 (13.3, 13.4 or 13.5, whichever the build was published for), CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU | | vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB | | SGLang | only when you ask for it | the same way as vLLM | | TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) | @@ -3312,7 +3312,7 @@ That carries: `--version` for another), with `checksums.txt` and its signature, checked against the project's signing key as `ctl upgrade` checks it. - **llama.cpp** at the version this llmgw installs, every build for each - architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked + architecture — CPU, CUDA 13 (cuda-13.3 up to b11070, cuda-13.4 to b12169, cuda-13.5 after), CUDA 12.8 (amd64), Vulkan — each checked against the checksum GitHub publishes, the same check a node does online. `--variant cpu --variant cuda-13.3` carries only those, for a site whose hardware you know; `--engine llamacpp=b10991` names the version. diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index 12a24e0..8a3597e 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -17,6 +17,7 @@ import ( "os/exec" "path/filepath" "regexp" + "slices" "strings" "time" @@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir return err } in.progress(st, "unpacking %s", asset) - if err := untarGz(archive, tmp); err != nil { + into := tmp + if slices.Contains(b.Beside, asset) { + into = filepath.Join(tmp, ".beside", asset) + if err := os.MkdirAll(into, 0o755); err != nil { + return err + } + } + if err := untarGz(archive, into); err != nil { return fmt.Errorf("unpacking %s: %w", asset, err) } _ = os.Remove(archive) got[asset] = want } + if len(b.Beside) > 0 { + if err := moveBesideBinary(tmp, b.Binary); err != nil { + return err + } + } in.progress(st, "checking that %s runs", b.Binary) bin, _, err := in.verify(ctx, tmp, b) @@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir return nil } +// moveBesideBinary moves every file the Beside assets unpacked, under +// dir/.beside, into the directory the binary is in, by its own name: the +// directories the archive put them in are its packaging, not where they are +// loaded from. A file the build already has is kept. +func moveBesideBinary(dir, binary string) error { + staged := filepath.Join(dir, ".beside") + defer os.RemoveAll(staged) //nolint:errcheck // scratch + bin, err := findFile(dir, binary) + if err != nil { + return err + } + into := filepath.Dir(bin) + return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + to := filepath.Join(into, d.Name()) + if _, err := os.Lstat(to); err == nil { + return nil + } + return os.Rename(p, to) + }) +} + // verify finds the binary and runs it. // // Two checks, because they fail differently. ldd names a missing library diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index ca9daae..9161b0a 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -429,7 +430,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) { rel := newRelease(map[string][]byte{ "llama-b1-" + cuda: tarGz(t, map[string]string{ "llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil), - "cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil), + // As the release packs it: under a directory of its own. + "cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil), }) gpu := ubuntu gpu.NVIDIADriver = "570.86.15" @@ -446,7 +448,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) { // The same build seeing its GPU is installed. sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" - if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready { + got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}) + if got[0].State != Ready { t.Fatalf("a CUDA build that sees its GPU: %+v", got[0]) } + // The runtime is beside the backend, where its $ORIGIN runpath finds it, + // not in the directory its archive gave it. + if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil { + t.Errorf("the CUDA runtime is not beside llama-server: %v", err) + } + if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil { + t.Error("the staging directory was left behind") + } +} + +func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) { + for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3", + "b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} { + if got := llamaCUDA13(v); got != want { + t.Errorf("%s: CUDA %s, want %s", v, got, want) + } + } + bs, err := Builds("llamacpp", "b12606", "amd64") + if err != nil { + t.Fatal(err) + } + if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") { + t.Errorf("b12606's CUDA 13 build: %+v", bs[1]) + } } diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index e7f7156..5e80ff4 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -12,6 +12,7 @@ package provision import ( "fmt" "slices" + "strconv" "strings" "time" @@ -32,6 +33,11 @@ type Build struct { // Binary is the executable's file name, found anywhere in the unpacked // tree. Binary string + // Beside are assets whose files belong in the directory Binary is in, + // wherever the archive puts them: llama.cpp's CUDA runtime, which its + // archive holds under a directory of its own, while the backend looks for + // it only beside itself ($ORIGIN). + Beside []string // CPUFlags are extensions the build is slow without. Missing ones are // reported, not refused: slow is still running. CPUFlags []string @@ -509,9 +515,25 @@ func leadingInt(s string) int { // compile. var llamaCUDA12Real = []float64{8.6, 8.9, 12.0} +// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the +// archive's name carries it, and it moves on with the project's CI. Taken +// from the releases themselves (the first build of each). +func llamaCUDA13(version string) string { + n, err := strconv.Atoi(strings.TrimPrefix(version, "b")) + switch { + case err != nil: + return "13.3" + case n >= 12170: + return "13.5" + case n >= 11071: + return "13.4" + } + return "13.3" +} + // llamaCppCandidates are the llama.cpp builds for a machine. // -// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580, +// On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580, // Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for // the GPUs it ships as PTX, 525 for those it ships as machine code), and // Vulkan, which runs on any card whose driver has Vulkan — the generations @@ -560,8 +582,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err cuda := func(cv string) Build { name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch) // The release carries the CUDA runtime beside the build, so the node - // needs a driver and nothing else — no toolkit. - b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name)) + // needs a driver and nothing else — no toolkit. Its archive unpacks + // into a directory of its own, where the backend never looks; Beside + // moves it next to the backend. + runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name) + b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime) + b.Beside = []string{runtime} b.GPUBackend = "libggml-cuda.so" return b } @@ -573,7 +599,8 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) { cuda12Driver = minDriverForCUDA12 } - cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}} + cv := llamaCUDA13(version) + cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}} if arch == "x64" { cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"}) } diff --git a/internal/cli/bundle.go b/internal/cli/bundle.go index fc03e53..26ea2e9 100644 --- a/internal/cli/bundle.go +++ b/internal/cli/bundle.go @@ -131,7 +131,7 @@ func newBundleCreateCmd() *cobra.Command { f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published") f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable") f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)") - f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)") + f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3 (cuda-13.4 from llama.cpp b11071, cuda-13.5 from b12170), cuda-12.8, vulkan; repeatable (default: every build)") f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable") f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves") f.StringVar(®istry, "registry", "", "the model registry, when not the public Hugging Face")