Compare commits

...
5 Commits
Author SHA1 Message Date
Ilker ManapandClaude Opus 5.5 754ab256a2 The driver's CUDA is read from nvidia-smi --version on the portable build
CI / test (push) Failing after 10m13s
Only the NVML build filled it, and the release's portable binary, the
one nodes run, never did: the CUDA 13.x check that came before this saw
nothing on cuda01 and installed b12606's CUDA 13.5 build again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 14:02:27 +01:00
Ilker ManapandClaude Opus 5.5 6765d430d8 A CUDA 13.x build of llama.cpp needs a driver that runs CUDA 13.x
CI / test (push) Failing after 9m48s
The release carries some GPUs' code as PTX, which the driver compiles,
and a driver older than the toolkit refuses it at the first kernel,
after the build has already listed the GPU. On cuda01 (Turing, driver
615 = CUDA 13.4) b12606's CUDA 13.5 build installed and then aborted
loading Inkling 2 Mini with "the provided PTX was compiled with an
unsupported toolchain". The driver's CUDA, which NVML already reports
per GPU, is now carried on the host and the CUDA 13.x candidate is
passed over for CUDA 12.8 when the driver is older.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 14:01:16 +01:00
Ilker ManapandClaude Opus 5.5 09ebe77513 A GGUF is servable when a llama.cpp that engine_overrides give some nodes loads it
CI / test (push) Failing after 9m58s
`ctl model serve`, its plan and model updates judged a file only against
the cluster's own llama.cpp, so a GPU node given a newer build could not
be served an architecture only that build knows: Inkling 2 Mini
(inkling2_loop) was refused for b10991 with cuda01 on b12606.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 13:43:40 +01:00
Ilker ManapandClaude Opus 5.5 0ffd633bf6 llama.cpp's CUDA runtime is unpacked beside the backend, and CUDA 13 follows the build
CI / test (push) Successful in 12m42s
The cudart archive unpacks into a directory of its own, while
libggml-cuda.so finds its libraries only through $ORIGIN, so the runtime
the release carries was never loaded: CUDA builds worked only on nodes
with a CUDA toolkit of the same major version installed. On cuda01
(CUDA 13.3 toolkit) b12606's CUDA 12.8 build saw no GPU and the install
fell through to Vulkan.

The CUDA 13 build's name also moved with llama.cpp's CI: 13.3 up to
b11070, 13.4 from b11071, 13.5 from b12170. A newer version asked for
the 13.3 archive, which does not exist.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-21 13:39:55 +01:00
Ilker ManapandClaude Fable 5.1 32d4d9c3af The admin API document regenerated for Slurm's jobs and the training page
CI / test (push) Failing after 8m24s
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-11-19 12:41:57 +01:00
19 changed files with 334 additions and 31 deletions

No files matched your search

+16 -2
View File
@@ -9944,6 +9944,12 @@
"created_by": { "created_by": {
"type": "string" "type": "string"
}, },
"project": {
"type": "string"
},
"publish": {
"type": "boolean"
},
"artifacts": { "artifacts": {
"type": "array", "type": "array",
"items": { "items": {
@@ -9989,18 +9995,26 @@
}, },
"base_repo": { "base_repo": {
"type": "string", "type": "string",
"description": "BaseRepo and BaseRevision are the weights trained on: the model's own on its registry, in safetensors (Hugging Face), not a GGUF." "description": "BaseRepo and BaseRevision are the weights trained on: the model's own on its registry, in safetensors (Hugging Face), not a GGUF. Empty finds them from where the model's came: the repository itself, or the one its card says it was made from."
}, },
"base_revision": { "base_revision": {
"type": "string" "type": "string"
}, },
"node": { "node": {
"type": "string", "type": "string",
"description": "Node trains it; GPUMemoryMiB is what it needs of a card, which can be what an engine there leaves free." "description": "Node trains it; empty is the first with a card that has enough free. GPUMemoryMiB is what it needs of a card, which can be what an engine there leaves free; 0 is worked out from the weights' size."
}, },
"gpu_memory_mib": { "gpu_memory_mib": {
"type": "integer" "type": "integer"
}, },
"project": {
"type": "string",
"description": "Project is the project the job is started in: its members see and manage it, and only one of its jobs trains at a time."
},
"publish": {
"type": "boolean",
"description": "Publish puts what the job makes on the mirror and makes it the model's adapter by itself when the job is done, restarting a llama.cpp model's replicas to load it."
},
"owner": { "owner": {
"type": "string", "type": "string",
"description": "Owner is who the adapter will belong to: org:, team:, project: (by path or ID), user: or key:. Only callers there see it. When the pairs come from one knowledge base, it defaults to that collection's owner; it must be able to read every collection they come from." "description": "Owner is who the adapter will belong to: org:, team:, project: (by path or ID), user: or key:. Only callers there see it. When the pairs come from one knowledge base, it defaults to that collection's owner; it must be able to read every collection they come from."
+2 -2
View File
@@ -787,7 +787,7 @@ Nothing to do: the agent installs them once the node joins, under
| Engine | On | How | | Engine | On | How |
|---|---|---| |---|---|---|
| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU | | llama.cpp | every node | the release build for its processor and GPU — CUDA 13 (13.3, 13.4 or 13.5, whichever the build was published for), CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
| vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB | | vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
| SGLang | only when you ask for it | the same way as vLLM | | SGLang | only when you ask for it | the same way as vLLM |
| TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) | | TensorRT-LLM | only nodes where you installed it | not installed by llmgw: see [TensorRT-LLM, which you bring](#tensorrt-llm-which-you-bring) |
@@ -3312,7 +3312,7 @@ That carries:
`--version` for another), with `checksums.txt` and its signature, checked `--version` for another), with `checksums.txt` and its signature, checked
against the project's signing key as `ctl upgrade` checks it. against the project's signing key as `ctl upgrade` checks it.
- **llama.cpp** at the version this llmgw installs, every build for each - **llama.cpp** at the version this llmgw installs, every build for each
architecture — CPU, CUDA 13.3, CUDA 12.8 (amd64), Vulkan — each checked architecture — CPU, CUDA 13 (cuda-13.3 up to b11070, cuda-13.4 to b12169, cuda-13.5 after), CUDA 12.8 (amd64), Vulkan — each checked
against the checksum GitHub publishes, the same check a node does online. against the checksum GitHub publishes, the same check a node does online.
`--variant cpu --variant cuda-13.3` carries only those, for a site whose `--variant cpu --variant cuda-13.3` carries only those, for a site whose
hardware you know; `--engine llamacpp=b10991` names the version. hardware you know; `--engine llamacpp=b10991` names the version.
+1
View File
@@ -529,6 +529,7 @@ func (a *Agent) scan(ctx context.Context) error {
inv, err := discovery.Discover(ctx, opts, a.log) inv, err := discovery.Discover(ctx, opts, a.log)
host := discovery.DiscoverHost() host := discovery.DiscoverHost()
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs) host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
host.DriverCUDA = discovery.DriverCUDA(inv.GPUs)
bw := a.memBandwidth() bw := a.memBandwidth()
host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core
a.mu.Lock() a.mu.Lock()
+42
View File
@@ -65,6 +65,11 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
return cards, err return cards, err
} }
readPCIe(ctx, binary, cards) readPCIe(ctx, binary, cards)
if cuda := driverCUDAFromSMI(ctx, binary); cuda != "" {
for i := range cards {
cards[i].CUDAVersion = cuda
}
}
// The query lists whole cards, MIG mode or not. A card in MIG mode is // The query lists whole cards, MIG mode or not. A card in MIG mode is
// replaced by its instances, which only -L names and only the full // replaced by its instances, which only -L names and only the full
// report sizes; asked only when some card could be in MIG mode, since // report sizes; asked only when some card could be in MIG mode, since
@@ -173,3 +178,40 @@ func assessHealth(g *GPU) {
g.Health.Problems = append(g.Health.Problems, p) g.Health.Problems = append(g.Health.Problems, p)
} }
} }
// driverCUDAFromSMI is the newest CUDA the driver runs, from `nvidia-smi
// --version`: "CUDA version : 12.4" on older drivers, "CUDA UMD version :
// 13.4" on newer ones, where the old line says "Deprecated". Empty when
// neither says; the query fields have no such column.
func driverCUDAFromSMI(ctx context.Context, binary string) string {
out, err := exec.CommandContext(ctx, binary, "--version").Output()
if err != nil {
return ""
}
return parseSMIDriverCUDA(string(out))
}
func parseSMIDriverCUDA(out string) string {
found := ""
for _, line := range strings.Split(out, "\n") {
key, val, ok := strings.Cut(line, ":")
if !ok {
continue
}
key, val = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(val)
if !strings.HasPrefix(key, "cuda") || !strings.HasSuffix(key, "version") {
continue
}
major, minor, ok := strings.Cut(val, ".")
if _, err := strconv.Atoi(major); err != nil || !ok {
continue
}
if _, err := strconv.Atoi(minor); err != nil {
continue
}
if strings.Contains(key, "umd") || found == "" {
found = val
}
}
return found
}
+15
View File
@@ -177,3 +177,18 @@ func TestUtilisationIsReadWhenPresent(t *testing.T) {
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent) t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
} }
} }
func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) {
for out, want := range map[string]string{
// cuda01, driver 615.
"NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" +
"CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4",
// An older driver.
"NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4",
"nothing useful": "",
} {
if got := parseSMIDriverCUDA(out); got != want {
t.Errorf("%q: %q, want %q", out, got, want)
}
}
}
+16
View File
@@ -36,6 +36,11 @@ type Host struct {
// MinComputeCapability is the oldest GPU's, 0 when there are none or it // MinComputeCapability is the oldest GPU's, 0 when there are none or it
// is not known. Not read here but from the GPU inventory, by the agent. // is not known. Not read here but from the GPU inventory, by the agent.
MinComputeCapability float64 `json:"min_compute_capability,omitempty"` MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
// DriverCUDA is the newest CUDA the loaded driver runs ("13.4"), empty
// when not known. Not read here but from the GPU inventory, by the agent.
// A build carrying a GPU's code as PTX needs a driver at least this new
// to compile it.
DriverCUDA string `json:"driver_cuda,omitempty"`
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not // MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
// measured: what decides a model's speed on the CPU. Not read here but // measured: what decides a model's speed on the CPU. Not read here but
// measured by the agent when it starts (MeasureMemoryBandwidth). // measured by the agent when it starts (MeasureMemoryBandwidth).
@@ -209,6 +214,17 @@ func nvidiaDriverFrom(path string) string {
return "" return ""
} }
// DriverCUDA is the newest CUDA the GPUs' driver runs, as NVML reports it,
// empty when none says.
func DriverCUDA(gpus []GPU) string {
for _, g := range gpus {
if v := strings.TrimSpace(g.CUDAVersion); v != "" {
return v
}
}
return ""
}
// MinComputeCapability is the oldest GPU's compute capability, 0 when none // MinComputeCapability is the oldest GPU's compute capability, 0 when none
// is known: an engine built for newer GPUs is refused by the oldest. // is known: an engine built for newer GPUs is refused by the oldest.
func MinComputeCapability(gpus []GPU) float64 { func MinComputeCapability(gpus []GPU) float64 {
+38 -1
View File
@@ -17,6 +17,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"regexp" "regexp"
"slices"
"strings" "strings"
"time" "time"
@@ -341,12 +342,24 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
return err return err
} }
in.progress(st, "unpacking %s", asset) in.progress(st, "unpacking %s", asset)
if err := untarGz(archive, tmp); err != nil { into := tmp
if slices.Contains(b.Beside, asset) {
into = filepath.Join(tmp, ".beside", asset)
if err := os.MkdirAll(into, 0o755); err != nil {
return err
}
}
if err := untarGz(archive, into); err != nil {
return fmt.Errorf("unpacking %s: %w", asset, err) return fmt.Errorf("unpacking %s: %w", asset, err)
} }
_ = os.Remove(archive) _ = os.Remove(archive)
got[asset] = want got[asset] = want
} }
if len(b.Beside) > 0 {
if err := moveBesideBinary(tmp, b.Binary); err != nil {
return err
}
}
in.progress(st, "checking that %s runs", b.Binary) in.progress(st, "checking that %s runs", b.Binary)
bin, _, err := in.verify(ctx, tmp, b) bin, _, err := in.verify(ctx, tmp, b)
@@ -370,6 +383,30 @@ func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir
return nil return nil
} }
// moveBesideBinary moves every file the Beside assets unpacked, under
// dir/.beside, into the directory the binary is in, by its own name: the
// directories the archive put them in are its packaging, not where they are
// loaded from. A file the build already has is kept.
func moveBesideBinary(dir, binary string) error {
staged := filepath.Join(dir, ".beside")
defer os.RemoveAll(staged) //nolint:errcheck // scratch
bin, err := findFile(dir, binary)
if err != nil {
return err
}
into := filepath.Dir(bin)
return filepath.WalkDir(staged, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
to := filepath.Join(into, d.Name())
if _, err := os.Lstat(to); err == nil {
return nil
}
return os.Rename(p, to)
})
}
// verify finds the binary and runs it. // verify finds the binary and runs it.
// //
// Two checks, because they fail differently. ldd names a missing library // Two checks, because they fail differently. ldd names a missing library
+44 -2
View File
@@ -13,6 +13,7 @@ import (
"net/http/httptest" "net/http/httptest"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings" "strings"
"sync" "sync"
"testing" "testing"
@@ -288,6 +289,21 @@ func TestTheBuildFollowsTheHardware(t *testing.T) {
} }
} }
// Found on cuda01 (Turing, driver 615, which runs CUDA 13.4): b12606's
// CUDA 13.5 build listed the GPU, then failed its first kernel with "the
// provided PTX was compiled with an unsupported toolchain".
turing := discovery.Host{Arch: "amd64", NVIDIADriver: "615.71.09", MinComputeCapability: 7.5}
for _, c := range []struct {
driverCUDA string
want string
}{{"13.4", "cuda-12.8"}, {"13.5", "cuda-13.5"}, {"14.0", "cuda-13.5"}, {"", "cuda-13.5"}} {
turing.DriverCUDA = c.driverCUDA
got, err := llamaCppCandidates("b12606", "", turing)
if err != nil || got[0].Variant != c.want {
t.Errorf("driver for CUDA %q: %+v, %v; want %s first", c.driverCUDA, got, err, c.want)
}
}
slow := ubuntu slow := ubuntu
slow.CPUFlags = nil slow.CPUFlags = nil
st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "") st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
@@ -429,7 +445,8 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
rel := newRelease(map[string][]byte{ rel := newRelease(map[string][]byte{
"llama-b1-" + cuda: tarGz(t, map[string]string{ "llama-b1-" + cuda: tarGz(t, map[string]string{
"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil), "llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil), // As the release packs it: under a directory of its own.
"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"cudart-llama-b1-" + strings.TrimSuffix(cuda, ".tar.gz") + "/libcudart.so.12": "x"}, nil),
}) })
gpu := ubuntu gpu := ubuntu
gpu.NVIDIADriver = "570.86.15" gpu.NVIDIADriver = "570.86.15"
@@ -446,7 +463,32 @@ func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
// The same build seeing its GPU is installed. // The same build seeing its GPU is installed.
sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n" sys.devices = "Available devices:\n CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready { got = in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
if got[0].State != Ready {
t.Fatalf("a CUDA build that sees its GPU: %+v", got[0]) t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
} }
// The runtime is beside the backend, where its $ORIGIN runpath finds it,
// not in the directory its archive gave it.
if _, err := os.Stat(filepath.Join(filepath.Dir(got[0].Binary), "libcudart.so.12")); err != nil {
t.Errorf("the CUDA runtime is not beside llama-server: %v", err)
}
if _, err := os.Stat(filepath.Join(filepath.Dir(filepath.Dir(got[0].Binary)), ".beside")); err == nil {
t.Error("the staging directory was left behind")
}
}
func TestLlamaCppsCUDA13FollowsTheBuild(t *testing.T) {
for v, want := range map[string]string{"b1": "13.3", "b10991": "13.3", "b11070": "13.3",
"b11071": "13.4", "b12167": "13.4", "b12170": "13.5", "b12606": "13.5", "latest": "13.3"} {
if got := llamaCUDA13(v); got != want {
t.Errorf("%s: CUDA %s, want %s", v, got, want)
}
}
bs, err := Builds("llamacpp", "b12606", "amd64")
if err != nil {
t.Fatal(err)
}
if bs[1].Variant != "cuda-13.5" || !slices.Contains(bs[1].Assets, "llama-b12606-bin-ubuntu-cuda-13.5-x64.tar.gz") {
t.Errorf("b12606's CUDA 13 build: %+v", bs[1])
}
} }
+62 -16
View File
@@ -12,6 +12,7 @@ package provision
import ( import (
"fmt" "fmt"
"slices" "slices"
"strconv"
"strings" "strings"
"time" "time"
@@ -32,6 +33,11 @@ type Build struct {
// Binary is the executable's file name, found anywhere in the unpacked // Binary is the executable's file name, found anywhere in the unpacked
// tree. // tree.
Binary string Binary string
// Beside are assets whose files belong in the directory Binary is in,
// wherever the archive puts them: llama.cpp's CUDA runtime, which its
// archive holds under a directory of its own, while the backend looks for
// it only beside itself ($ORIGIN).
Beside []string
// CPUFlags are extensions the build is slow without. Missing ones are // CPUFlags are extensions the build is slow without. Missing ones are
// reported, not refused: slow is still running. // reported, not refused: slow is still running.
CPUFlags []string CPUFlags []string
@@ -251,6 +257,11 @@ const (
type gpuNeeds struct { type gpuNeeds struct {
driver int driver int
cc float64 cc float64
// cuda, when set, is the CUDA the driver must run: a build that carries
// a GPU's code as PTX has the driver compile it, and a driver older than
// the toolkit refuses ("the provided PTX was compiled with an unsupported
// toolchain") at the first kernel — after the build has listed the GPU.
cuda string
} }
// fits says why a machine cannot run a build, or nil. // fits says why a machine cannot run a build, or nil.
@@ -258,12 +269,26 @@ func (n gpuNeeds) fits(what string, h discovery.Host) error {
if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver { if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver) return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
} }
if n.cuda != "" && h.DriverCUDA != "" && cudaBefore(h.DriverCUDA, n.cuda) {
return fmt.Errorf("%s needs a driver for CUDA %s or newer; this node's driver %s runs CUDA %s",
what, n.cuda, h.NVIDIADriver, h.DriverCUDA)
}
if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc { if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc) return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
} }
return nil return nil
} }
// cudaBefore reports whether CUDA version a ("13.4") is older than b.
func cudaBefore(a, b string) bool {
am, an, _ := strings.Cut(a, ".")
bm, bn, _ := strings.Cut(b, ".")
if x, y := leadingInt(am), leadingInt(bm); x != y {
return x < y
}
return leadingInt(an) < leadingInt(bn)
}
// candidate is a build and what it needs. // candidate is a build and what it needs.
type candidate struct { type candidate struct {
b Build b Build
@@ -400,7 +425,7 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
variant = "cu12" variant = "cu12"
} }
return pick("vLLM "+v, variant, []candidate{ return pick("vLLM "+v, variant, []candidate{
{pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v}, {pypi("cu12"), gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v},
}, h) }, h)
} }
cu129 := pypi("cu129") cu129 := pypi("cu129")
@@ -414,8 +439,8 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
cu129.ExtraIndex = pytorchCU129 cu129.ExtraIndex = pytorchCU129
cu129.IndexPackages = pytorchPackages cu129.IndexPackages = pytorchPackages
return pick("vLLM "+v, variant, []candidate{ return pick("vLLM "+v, variant, []candidate{
{pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"}, {pypi("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: minCC}, "vLLM " + v + " built for CUDA 13"},
{cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"}, {cu129, gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v + " built for CUDA 12.9"},
}, h) }, h)
} }
@@ -454,9 +479,9 @@ func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error
FreeGB: 25, FreeGB: 25,
GPUCheck: true, GPUCheck: true,
} }
needs := gpuNeeds{minDriverForCUDA13, 7.5} needs := gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}
if versionBefore(version, sglangCUDA13Since) { if versionBefore(version, sglangCUDA13Since) {
b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0} b.Variant, needs = "cu12", gpuNeeds{driver: minDriverForCUDA12, cc: 7.0}
} }
return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h) return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
} }
@@ -509,9 +534,25 @@ func leadingInt(s string) int {
// compile. // compile.
var llamaCUDA12Real = []float64{8.6, 8.9, 12.0} var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
// llamaCUDA13 is the CUDA 13 release a llama.cpp build is published for: the
// archive's name carries it, and it moves on with the project's CI. Taken
// from the releases themselves (the first build of each).
func llamaCUDA13(version string) string {
n, err := strconv.Atoi(strings.TrimPrefix(version, "b"))
switch {
case err != nil:
return "13.3"
case n >= 12170:
return "13.5"
case n >= 11071:
return "13.4"
}
return "13.3"
}
// llamaCppCandidates are the llama.cpp builds for a machine. // llamaCppCandidates are the llama.cpp builds for a machine.
// //
// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580, // On an NVIDIA node, the GPU builds best first: CUDA 13 (driver 580,
// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for // Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
// the GPUs it ships as PTX, 525 for those it ships as machine code), and // the GPUs it ships as PTX, 525 for those it ships as machine code), and
// Vulkan, which runs on any card whose driver has Vulkan — the generations // Vulkan, which runs on any card whose driver has Vulkan — the generations
@@ -560,8 +601,12 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
cuda := func(cv string) Build { cuda := func(cv string) Build {
name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch) name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
// The release carries the CUDA runtime beside the build, so the node // The release carries the CUDA runtime beside the build, so the node
// needs a driver and nothing else — no toolkit. // needs a driver and nothing else — no toolkit. Its archive unpacks
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name)) // into a directory of its own, where the backend never looks; Beside
// moves it next to the backend.
runtime := fmt.Sprintf("cudart-llama-%s-%s", version, name)
b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), runtime)
b.Beside = []string{runtime}
b.GPUBackend = "libggml-cuda.so" b.GPUBackend = "libggml-cuda.so"
return b return b
} }
@@ -573,9 +618,10 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) { if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
cuda12Driver = minDriverForCUDA12 cuda12Driver = minDriverForCUDA12
} }
cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}} cv := llamaCUDA13(version)
cands := []candidate{{cuda(cv), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5, cuda: cv}, "llama.cpp built for CUDA " + cv}}
if arch == "x64" { if arch == "x64" {
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"}) cands = append(cands, candidate{cuda("12.8"), gpuNeeds{driver: cuda12Driver, cc: 5.0}, "llama.cpp built for CUDA 12.8"})
} }
cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"}) cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
return pick("llama.cpp", variant, cands, h) return pick("llama.cpp", variant, cands, h)
@@ -677,9 +723,9 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`},
cu126 := build("cu126") cu126 := build("cu126")
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
return pick("ComfyUI "+version, variant, []candidate{ return pick("ComfyUI "+version, variant, []candidate{
{build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, {build("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
{cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"}, {cu129, gpuNeeds{driver: minDriverForCUDA12, cc: 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
{cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"}, {cu126, gpuNeeds{driver: minDriverForCUDA12, cc: 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
}, h) }, h)
} }
@@ -753,10 +799,10 @@ print("strata version", "` + version + `", "engine", "` + rel.engine + `")`},
} }
} }
return pick("Strata "+version, variant, []candidate{ return pick("Strata "+version, variant, []candidate{
{build("cuda13"), gpuNeeds{minDriverForCUDA13, 7.5}, "Strata " + version + " built for CUDA 13"}, {build("cuda13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "Strata " + version + " built for CUDA 13"},
// 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its // 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its
// PTX was compiled by 12.9, so the driver must be 575. // PTX was compiled by 12.9, so the driver must be 575.
{build("cuda12"), gpuNeeds{575, 6.0}, "Strata " + version + " built for CUDA 12.9"}, {build("cuda12"), gpuNeeds{driver: 575, cc: 6.0}, "Strata " + version + " built for CUDA 12.9"},
}, h) }, h)
} }
@@ -809,6 +855,6 @@ func freetokenCandidates(version, variant string, h discovery.Host) ([]Build, er
GPUCheck: true, GPUCheck: true,
} }
return pick("FreeToken "+version, variant, []candidate{ return pick("FreeToken "+version, variant, []candidate{
{b, gpuNeeds{minDriverForCUDA13, freetokenMinCompute}, "FreeToken " + version}, {b, gpuNeeds{driver: minDriverForCUDA13, cc: freetokenMinCompute}, "FreeToken " + version},
}, h) }, h)
} }
+1 -1
View File
@@ -131,7 +131,7 @@ func newBundleCreateCmd() *cobra.Command {
f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published") f.StringVar(&releases, "releases", release.DefaultBase, "where llmgw's releases are published")
f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable") f.StringArrayVar(&arches, "arch", []string{"amd64"}, "an architecture of the site's machines, amd64 or arm64; repeatable")
f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)") f.StringArrayVar(&engines, "engine", nil, "an engine version to carry, engine=version; repeatable (default: llama.cpp at the version this llmgw installs)")
f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3, cuda-12.8, vulkan; repeatable (default: every build)") f.StringArrayVar(&variants, "variant", nil, "a build to carry: cpu, cuda-13.3 (cuda-13.4 from llama.cpp b11071, cuda-13.5 from b12170), cuda-12.8, vulkan; repeatable (default: every build)")
f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable") f.StringArrayVar(&models, "model", nil, "a model file to carry, name/version=hf:owner/repo/file or name/version=https://...; repeatable")
f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves") f.BoolVar(&noStarter, "no-starter-models", false, "leave out the starter model and the guardrails' judge that server init serves")
f.StringVar(&registry, "registry", "", "the model registry, when not the public Hugging Face") f.StringVar(&registry, "registry", "", "the model registry, when not the public Hugging Face")
+1 -1
View File
@@ -679,7 +679,7 @@ func newServerCmd() *cobra.Command {
if adminAPI != nil && catalogSvc != nil { if adminAPI != nil && catalogSvc != nil {
serve := modelserve.New(modelserve.Deps{ serve := modelserve.New(modelserve.Deps{
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin, Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log, LlamaVersion: cfg.WantedEngines()["llamacpp"], LlamaOverrides: cfg.OverrideVersions("llamacpp"), Log: log,
// Updates: the catalog in force, the service that // Updates: the catalog in force, the service that
// changes it, and what replaces replicas. // changes it, and what replaces replicas.
Current: func() []catalog.Model { return admin.Catalog().Models() }, Current: func() []catalog.Model { return admin.Catalog().Models() },
+12
View File
@@ -1164,6 +1164,18 @@ func (s Server) EnginesFor(node string, classes []string, labels map[string]stri
return out return out
} }
// OverrideVersions are the versions of an engine that engine_overrides give
// some nodes, in the order they are set.
func (s Server) OverrideVersions(engine string) []string {
var out []string
for _, o := range s.EngineOverrides {
if v := o.Engines[engine]; v != "" && !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
// WantedEngines is DefaultEngines with the configuration laid over it. // WantedEngines is DefaultEngines with the configuration laid over it.
func (s Server) WantedEngines() map[string]string { func (s Server) WantedEngines() map[string]string {
out := map[string]string{} out := map[string]string{}
+15
View File
@@ -1,6 +1,7 @@
package config package config
import ( import (
"slices"
"strings" "strings"
"testing" "testing"
) )
@@ -54,3 +55,17 @@ func TestOIDCSettingsAreCheckedBeforeAProviderIsAsked(t *testing.T) {
} }
} }
} }
func TestOverrideVersionsAreTheEnginesNodesAreGivenInstead(t *testing.T) {
s := Server{EngineOverrides: []EngineOverride{
{Nodes: []string{"cuda01"}, Engines: map[string]string{"strata": "0.2.7", "llamacpp": "b12606"}},
{GPUClass: "v100-32g", Engines: map[string]string{"llamacpp": ""}},
{Labels: map[string]string{"site": "b"}, Engines: map[string]string{"llamacpp": "b12606"}},
}}
if got := s.OverrideVersions("llamacpp"); !slices.Equal(got, []string{"b12606"}) {
t.Errorf("llamacpp: %v", got)
}
if got := s.OverrideVersions("vllm"); got != nil {
t.Errorf("vllm: %v", got)
}
}
+17 -3
View File
@@ -68,8 +68,10 @@ type Deps struct {
Catalog Catalog Catalog Catalog
Fleet Fleet Fleet Fleet
// LlamaVersion is the llama.cpp build the nodes install, which decides // LlamaVersion is the llama.cpp build the nodes install, which decides
// which architectures load. // which architectures load. LlamaOverrides are the builds some nodes are
// given instead (engine_overrides): a file one of them loads is servable.
LlamaVersion string LlamaVersion string
LlamaOverrides []string
HTTP *http.Client HTTP *http.Client
Log *slog.Logger Log *slog.Logger
// Poll is how often the job looks at the mirror and the fleet. // Poll is how often the job looks at the mirror and the fleet.
@@ -240,6 +242,18 @@ type Server struct {
} }
// New builds a server. // New builds a server.
// llamaVersions are the llama.cpp builds the nodes install, the cluster's own
// first.
func (s *Server) llamaVersions() []string {
out := []string{s.deps.LlamaVersion}
for _, v := range s.deps.LlamaOverrides {
if !slices.Contains(out, v) {
out = append(out, v)
}
}
return out
}
func New(d Deps) *Server { func New(d Deps) *Server {
if d.Poll <= 0 { if d.Poll <= 0 {
d.Poll = 2 * time.Second d.Poll = 2 * time.Second
@@ -553,9 +567,9 @@ func (s *Server) run(j *Job) {
s.step(j, "check", Running, "reading its header from the mirror") s.step(j, "check", Running, "reading its header from the mirror")
var header gguf.Header var header gguf.Header
if u := headerURL(source, set); u != "" { if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion) v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable { if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason)) s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run this: %s", v.Reason))
return return
} }
header = v.Header header = v.Header
+1 -1
View File
@@ -147,7 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
if b, ok := s.deps.Registry.(interface{ Base() string }); ok { if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
base = b.Base() base = b.Base()
} }
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion) v := gguf.CheckAny(ctx, repo.DownloadURL(base, small.Files[0].Path), s.llamaVersions())
if v.Checked && !v.Servable { if v.Checked && !v.Servable {
p.Servable, p.Reason = false, v.Reason p.Servable, p.Reason = false, v.Reason
} }
+2 -2
View File
@@ -585,9 +585,9 @@ func (s *Server) prepare(ctx context.Context, j *Job, name string) (prepared, bo
s.step(j, "check", Running, "reading its header from the mirror") s.step(j, "check", Running, "reading its header from the mirror")
detail := "no header to read" detail := "no header to read"
if u := headerURL(source, set); u != "" { if u := headerURL(source, set); u != "" {
v := gguf.Check(ctx, u, s.deps.LlamaVersion) v := gguf.CheckAny(ctx, u, s.llamaVersions())
if !v.Servable { if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run the new weights: %s", s.deps.LlamaVersion, v.Reason)) s.fail(j, "check", fmt.Errorf("llama.cpp on the nodes cannot run the new weights: %s", v.Reason))
return prepared{}, false return prepared{}, false
} }
detail = orUnknown(v.Header.Architecture) detail = orUnknown(v.Header.Architecture)
+1
View File
@@ -222,6 +222,7 @@ func Collect(ctx context.Context, o Options) Facts {
f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) f.Fabrics = discovery.CheckFabrics(declared, f.HCAs)
} }
f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs) f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs)
f.Host.DriverCUDA = discovery.DriverCUDA(f.GPUs)
if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" { if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" {
if f.EnginesDir == "" { if f.EnginesDir == "" {
f.EnginesDir = provision.DefaultRoot f.EnginesDir = provision.DefaultRoot
+26
View File
@@ -404,6 +404,32 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
return v return v
} }
// CheckAny is Check against several llama.cpp builds, the header read once:
// servable when any of them loads it. A cluster whose GPU nodes are given a
// newer build than the rest (engine_overrides) can serve an architecture
// only that build knows. The first version is the cluster's own; its verdict
// is the one given when none loads it.
func CheckAny(ctx context.Context, url string, versions []string) Verdict {
if len(versions) == 0 {
versions = []string{""}
}
v := Check(ctx, url, versions[0])
if v.Servable || v.Header.Architecture == "" {
return v
}
for _, other := range versions[1:] {
if o := LlamaCpp(other, v.Header.Architecture); o.Servable {
o.Header = v.Header
return o
}
}
if len(versions) > 1 {
v.Reason = "its architecture is \"" + v.Header.Architecture + "\", which no llama.cpp the nodes install (" +
strings.Join(versions, ", ") + ") can load"
}
return v
}
// CardUsable is the share of a card's memory a model may take: the rest is // CardUsable is the share of a card's memory a model may take: the rest is
// CUDA's own and the engine's compute buffers. // CUDA's own and the engine's compute buffers.
const CardUsable = 0.9 const CardUsable = 0.9
+22
View File
@@ -221,3 +221,25 @@ func TestAModelOfExpertsReadsAShareOfItsWeights(t *testing.T) {
t.Error("a dense model reads all its weights") t.Error("a dense model reads all its weights")
} }
} }
func TestAnArchitectureOnlyAnOverriddenBuildKnowsIsServable(t *testing.T) {
// Found on the PoC: Inkling 2 Mini (inkling2_loop) refused by `ctl model
// serve` for the cluster's b10991, though the GPU node had been given
// b12606 through engine_overrides.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, "m.gguf", timeZero, bytes.NewReader(header("inkling2_loop")))
}))
defer srv.Close()
url := srv.URL + "/m.gguf"
if v := CheckAny(context.Background(), url, []string{"b10991"}); v.Servable {
t.Errorf("b10991 alone: %+v", v)
}
if v := CheckAny(context.Background(), url, []string{"b10991", "b12606"}); !v.Servable ||
v.Header.Architecture != "inkling2_loop" {
t.Errorf("with b12606 on some nodes: %+v", v)
}
if v := CheckAny(context.Background(), url, []string{"b10991", "b10991"}); v.Servable ||
!strings.Contains(v.Reason, "no llama.cpp the nodes install (b10991, b10991)") {
t.Errorf("refused by every build: %+v", v)
}
}