A CUDA 13.x build of llama.cpp needs a driver that runs CUDA 13.x
CI / test (push) Failing after 9m48s
CI / test (push) Failing after 9m48s
The release carries some GPUs' code as PTX, which the driver compiles, and a driver older than the toolkit refuses it at the first kernel, after the build has already listed the GPU. On cuda01 (Turing, driver 615 = CUDA 13.4) b12606's CUDA 13.5 build installed and then aborted loading Inkling 2 Mini with "the provided PTX was compiled with an unsupported toolchain". The driver's CUDA, which NVML already reports per GPU, is now carried on the host and the CUDA 13.x candidate is passed over for CUDA 12.8 when the driver is older. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
09ebe77513
commit
6765d430d8
5 files changed
+65
-13
No files matched your search
@@ -529,6 +529,7 @@ func (a *Agent) scan(ctx context.Context) error {
|
||||
inv, err := discovery.Discover(ctx, opts, a.log)
|
||||
host := discovery.DiscoverHost()
|
||||
host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs)
|
||||
host.DriverCUDA = discovery.DriverCUDA(inv.GPUs)
|
||||
bw := a.memBandwidth()
|
||||
host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core
|
||||
a.mu.Lock()
|
||||
|
||||
@@ -36,6 +36,11 @@ type Host struct {
|
||||
// MinComputeCapability is the oldest GPU's, 0 when there are none or it
|
||||
// is not known. Not read here but from the GPU inventory, by the agent.
|
||||
MinComputeCapability float64 `json:"min_compute_capability,omitempty"`
|
||||
// DriverCUDA is the newest CUDA the loaded driver runs ("13.4"), empty
|
||||
// when not known. Not read here but from the GPU inventory, by the agent.
|
||||
// A build carrying a GPU's code as PTX needs a driver at least this new
|
||||
// to compile it.
|
||||
DriverCUDA string `json:"driver_cuda,omitempty"`
|
||||
// MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not
|
||||
// measured: what decides a model's speed on the CPU. Not read here but
|
||||
// measured by the agent when it starts (MeasureMemoryBandwidth).
|
||||
@@ -209,6 +214,17 @@ func nvidiaDriverFrom(path string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DriverCUDA is the newest CUDA the GPUs' driver runs, as NVML reports it,
|
||||
// empty when none says.
|
||||
func DriverCUDA(gpus []GPU) string {
|
||||
for _, g := range gpus {
|
||||
if v := strings.TrimSpace(g.CUDAVersion); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// MinComputeCapability is the oldest GPU's compute capability, 0 when none
|
||||
// is known: an engine built for newer GPUs is refused by the oldest.
|
||||
func MinComputeCapability(gpus []GPU) float64 {
|
||||
|
||||
@@ -289,6 +289,21 @@ func TestTheBuildFollowsTheHardware(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Found on cuda01 (Turing, driver 615, which runs CUDA 13.4): b12606's
|
||||
// CUDA 13.5 build listed the GPU, then failed its first kernel with "the
|
||||
// provided PTX was compiled with an unsupported toolchain".
|
||||
turing := discovery.Host{Arch: "amd64", NVIDIADriver: "615.71.09", MinComputeCapability: 7.5}
|
||||
for _, c := range []struct {
|
||||
driverCUDA string
|
||||
want string
|
||||
}{{"13.4", "cuda-12.8"}, {"13.5", "cuda-13.5"}, {"14.0", "cuda-13.5"}, {"", "cuda-13.5"}} {
|
||||
turing.DriverCUDA = c.driverCUDA
|
||||
got, err := llamaCppCandidates("b12606", "", turing)
|
||||
if err != nil || got[0].Variant != c.want {
|
||||
t.Errorf("driver for CUDA %q: %+v, %v; want %s first", c.driverCUDA, got, err, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
slow := ubuntu
|
||||
slow.CPUFlags = nil
|
||||
st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "")
|
||||
|
||||
@@ -257,6 +257,11 @@ const (
|
||||
type gpuNeeds struct {
|
||||
driver int
|
||||
cc float64
|
||||
// cuda, when set, is the CUDA the driver must run: a build that carries
|
||||
// a GPU's code as PTX has the driver compile it, and a driver older than
|
||||
// the toolkit refuses ("the provided PTX was compiled with an unsupported
|
||||
// toolchain") at the first kernel — after the build has listed the GPU.
|
||||
cuda string
|
||||
}
|
||||
|
||||
// fits says why a machine cannot run a build, or nil.
|
||||
@@ -264,12 +269,26 @@ func (n gpuNeeds) fits(what string, h discovery.Host) error {
|
||||
if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
|
||||
return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
|
||||
}
|
||||
if n.cuda != "" && h.DriverCUDA != "" && cudaBefore(h.DriverCUDA, n.cuda) {
|
||||
return fmt.Errorf("%s needs a driver for CUDA %s or newer; this node's driver %s runs CUDA %s",
|
||||
what, n.cuda, h.NVIDIADriver, h.DriverCUDA)
|
||||
}
|
||||
if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
|
||||
return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cudaBefore reports whether CUDA version a ("13.4") is older than b.
|
||||
func cudaBefore(a, b string) bool {
|
||||
am, an, _ := strings.Cut(a, ".")
|
||||
bm, bn, _ := strings.Cut(b, ".")
|
||||
if x, y := leadingInt(am), leadingInt(bm); x != y {
|
||||
return x < y
|
||||
}
|
||||
return leadingInt(an) < leadingInt(bn)
|
||||
}
|
||||
|
||||
// candidate is a build and what it needs.
|
||||
type candidate struct {
|
||||
b Build
|
||||
@@ -406,7 +425,7 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
|
||||
variant = "cu12"
|
||||
}
|
||||
return pick("vLLM "+v, variant, []candidate{
|
||||
{pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v},
|
||||
{pypi("cu12"), gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v},
|
||||
}, h)
|
||||
}
|
||||
cu129 := pypi("cu129")
|
||||
@@ -420,8 +439,8 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error)
|
||||
cu129.ExtraIndex = pytorchCU129
|
||||
cu129.IndexPackages = pytorchPackages
|
||||
return pick("vLLM "+v, variant, []candidate{
|
||||
{pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"},
|
||||
{cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"},
|
||||
{pypi("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: minCC}, "vLLM " + v + " built for CUDA 13"},
|
||||
{cu129, gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v + " built for CUDA 12.9"},
|
||||
}, h)
|
||||
}
|
||||
|
||||
@@ -460,9 +479,9 @@ func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error
|
||||
FreeGB: 25,
|
||||
GPUCheck: true,
|
||||
}
|
||||
needs := gpuNeeds{minDriverForCUDA13, 7.5}
|
||||
needs := gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}
|
||||
if versionBefore(version, sglangCUDA13Since) {
|
||||
b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0}
|
||||
b.Variant, needs = "cu12", gpuNeeds{driver: minDriverForCUDA12, cc: 7.0}
|
||||
}
|
||||
return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
|
||||
}
|
||||
@@ -600,9 +619,9 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err
|
||||
cuda12Driver = minDriverForCUDA12
|
||||
}
|
||||
cv := llamaCUDA13(version)
|
||||
cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}}
|
||||
cands := []candidate{{cuda(cv), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5, cuda: cv}, "llama.cpp built for CUDA " + cv}}
|
||||
if arch == "x64" {
|
||||
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
|
||||
cands = append(cands, candidate{cuda("12.8"), gpuNeeds{driver: cuda12Driver, cc: 5.0}, "llama.cpp built for CUDA 12.8"})
|
||||
}
|
||||
cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
|
||||
return pick("llama.cpp", variant, cands, h)
|
||||
@@ -704,9 +723,9 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`},
|
||||
cu126 := build("cu126")
|
||||
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
|
||||
return pick("ComfyUI "+version, variant, []candidate{
|
||||
{build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
||||
{cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||
{cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
||||
{build("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
||||
{cu129, gpuNeeds{driver: minDriverForCUDA12, cc: 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||
{cu126, gpuNeeds{driver: minDriverForCUDA12, cc: 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
||||
}, h)
|
||||
}
|
||||
|
||||
@@ -780,10 +799,10 @@ print("strata version", "` + version + `", "engine", "` + rel.engine + `")`},
|
||||
}
|
||||
}
|
||||
return pick("Strata "+version, variant, []candidate{
|
||||
{build("cuda13"), gpuNeeds{minDriverForCUDA13, 7.5}, "Strata " + version + " built for CUDA 13"},
|
||||
{build("cuda13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "Strata " + version + " built for CUDA 13"},
|
||||
// 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its
|
||||
// PTX was compiled by 12.9, so the driver must be 575.
|
||||
{build("cuda12"), gpuNeeds{575, 6.0}, "Strata " + version + " built for CUDA 12.9"},
|
||||
{build("cuda12"), gpuNeeds{driver: 575, cc: 6.0}, "Strata " + version + " built for CUDA 12.9"},
|
||||
}, h)
|
||||
}
|
||||
|
||||
@@ -836,6 +855,6 @@ func freetokenCandidates(version, variant string, h discovery.Host) ([]Build, er
|
||||
GPUCheck: true,
|
||||
}
|
||||
return pick("FreeToken "+version, variant, []candidate{
|
||||
{b, gpuNeeds{minDriverForCUDA13, freetokenMinCompute}, "FreeToken " + version},
|
||||
{b, gpuNeeds{driver: minDriverForCUDA13, cc: freetokenMinCompute}, "FreeToken " + version},
|
||||
}, h)
|
||||
}
|
||||
Reference in new issue
Block a user