diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 993c19c..f4581e0 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -529,6 +529,7 @@ func (a *Agent) scan(ctx context.Context) error { inv, err := discovery.Discover(ctx, opts, a.log) host := discovery.DiscoverHost() host.MinComputeCapability = discovery.MinComputeCapability(inv.GPUs) + host.DriverCUDA = discovery.DriverCUDA(inv.GPUs) bw := a.memBandwidth() host.MemBandwidthGBs, host.MemBandwidthCoreGBs = bw.All, bw.Core a.mu.Lock() diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index 6ac2451..5fde6e3 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -36,6 +36,11 @@ type Host struct { // MinComputeCapability is the oldest GPU's, 0 when there are none or it // is not known. Not read here but from the GPU inventory, by the agent. MinComputeCapability float64 `json:"min_compute_capability,omitempty"` + // DriverCUDA is the newest CUDA the loaded driver runs ("13.4"), empty + // when not known. Not read here but from the GPU inventory, by the agent. + // A build carrying a GPU's code as PTX needs a driver at least this new + // to compile it. + DriverCUDA string `json:"driver_cuda,omitempty"` // MemBandwidthGBs is how fast the memory reads, in GB/s, 0 when not // measured: what decides a model's speed on the CPU. Not read here but // measured by the agent when it starts (MeasureMemoryBandwidth). @@ -209,6 +214,17 @@ func nvidiaDriverFrom(path string) string { return "" } +// DriverCUDA is the newest CUDA the GPUs' driver runs, as NVML reports it, +// empty when none says. +func DriverCUDA(gpus []GPU) string { + for _, g := range gpus { + if v := strings.TrimSpace(g.CUDAVersion); v != "" { + return v + } + } + return "" +} + // MinComputeCapability is the oldest GPU's compute capability, 0 when none // is known: an engine built for newer GPUs is refused by the oldest. func MinComputeCapability(gpus []GPU) float64 { diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index 9161b0a..fd347e9 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -289,6 +289,21 @@ func TestTheBuildFollowsTheHardware(t *testing.T) { } } + // Found on cuda01 (Turing, driver 615, which runs CUDA 13.4): b12606's + // CUDA 13.5 build listed the GPU, then failed its first kernel with "the + // provided PTX was compiled with an unsupported toolchain". + turing := discovery.Host{Arch: "amd64", NVIDIADriver: "615.71.09", MinComputeCapability: 7.5} + for _, c := range []struct { + driverCUDA string + want string + }{{"13.4", "cuda-12.8"}, {"13.5", "cuda-13.5"}, {"14.0", "cuda-13.5"}, {"", "cuda-13.5"}} { + turing.DriverCUDA = c.driverCUDA + got, err := llamaCppCandidates("b12606", "", turing) + if err != nil || got[0].Variant != c.want { + t.Errorf("driver for CUDA %q: %+v, %v; want %s first", c.driverCUDA, got, err, c.want) + } + } + slow := ubuntu slow.CPUFlags = nil st := (&Installer{Host: slow}).ready(Status{}, Build{CPUFlags: []string{"avx2"}}, "/x/llama-server", "") diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 5e80ff4..59b75cd 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -257,6 +257,11 @@ const ( type gpuNeeds struct { driver int cc float64 + // cuda, when set, is the CUDA the driver must run: a build that carries + // a GPU's code as PTX has the driver compile it, and a driver older than + // the toolkit refuses ("the provided PTX was compiled with an unsupported + // toolchain") at the first kernel — after the build has listed the GPU. + cuda string } // fits says why a machine cannot run a build, or nil. @@ -264,12 +269,26 @@ func (n gpuNeeds) fits(what string, h discovery.Host) error { if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver { return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver) } + if n.cuda != "" && h.DriverCUDA != "" && cudaBefore(h.DriverCUDA, n.cuda) { + return fmt.Errorf("%s needs a driver for CUDA %s or newer; this node's driver %s runs CUDA %s", + what, n.cuda, h.NVIDIADriver, h.DriverCUDA) + } if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc { return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc) } return nil } +// cudaBefore reports whether CUDA version a ("13.4") is older than b. +func cudaBefore(a, b string) bool { + am, an, _ := strings.Cut(a, ".") + bm, bn, _ := strings.Cut(b, ".") + if x, y := leadingInt(am), leadingInt(bm); x != y { + return x < y + } + return leadingInt(an) < leadingInt(bn) +} + // candidate is a build and what it needs. type candidate struct { b Build @@ -406,7 +425,7 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error) variant = "cu12" } return pick("vLLM "+v, variant, []candidate{ - {pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v}, + {pypi("cu12"), gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v}, }, h) } cu129 := pypi("cu129") @@ -420,8 +439,8 @@ func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error) cu129.ExtraIndex = pytorchCU129 cu129.IndexPackages = pytorchPackages return pick("vLLM "+v, variant, []candidate{ - {pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"}, - {cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"}, + {pypi("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: minCC}, "vLLM " + v + " built for CUDA 13"}, + {cu129, gpuNeeds{driver: minDriverForCUDA12, cc: minCC}, "vLLM " + v + " built for CUDA 12.9"}, }, h) } @@ -460,9 +479,9 @@ func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error FreeGB: 25, GPUCheck: true, } - needs := gpuNeeds{minDriverForCUDA13, 7.5} + needs := gpuNeeds{driver: minDriverForCUDA13, cc: 7.5} if versionBefore(version, sglangCUDA13Since) { - b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0} + b.Variant, needs = "cu12", gpuNeeds{driver: minDriverForCUDA12, cc: 7.0} } return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h) } @@ -600,9 +619,9 @@ func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, err cuda12Driver = minDriverForCUDA12 } cv := llamaCUDA13(version) - cands := []candidate{{cuda(cv), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA " + cv}} + cands := []candidate{{cuda(cv), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5, cuda: cv}, "llama.cpp built for CUDA " + cv}} if arch == "x64" { - cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"}) + cands = append(cands, candidate{cuda("12.8"), gpuNeeds{driver: cuda12Driver, cc: 5.0}, "llama.cpp built for CUDA 12.8"}) } cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"}) return pick("llama.cpp", variant, cands, h) @@ -704,9 +723,9 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`}, cu126 := build("cu126") cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages return pick("ComfyUI "+version, variant, []candidate{ - {build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, - {cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"}, - {cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"}, + {build("cu13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, + {cu129, gpuNeeds{driver: minDriverForCUDA12, cc: 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + {cu126, gpuNeeds{driver: minDriverForCUDA12, cc: 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"}, }, h) } @@ -780,10 +799,10 @@ print("strata version", "` + version + `", "engine", "` + rel.engine + `")`}, } } return pick("Strata "+version, variant, []candidate{ - {build("cuda13"), gpuNeeds{minDriverForCUDA13, 7.5}, "Strata " + version + " built for CUDA 13"}, + {build("cuda13"), gpuNeeds{driver: minDriverForCUDA13, cc: 7.5}, "Strata " + version + " built for CUDA 13"}, // 12.9 rather than 12: its machine code is for 6.0 to 8.9 and its // PTX was compiled by 12.9, so the driver must be 575. - {build("cuda12"), gpuNeeds{575, 6.0}, "Strata " + version + " built for CUDA 12.9"}, + {build("cuda12"), gpuNeeds{driver: 575, cc: 6.0}, "Strata " + version + " built for CUDA 12.9"}, }, h) } @@ -836,6 +855,6 @@ func freetokenCandidates(version, variant string, h discovery.Host) ([]Build, er GPUCheck: true, } return pick("FreeToken "+version, variant, []candidate{ - {b, gpuNeeds{minDriverForCUDA13, freetokenMinCompute}, "FreeToken " + version}, + {b, gpuNeeds{driver: minDriverForCUDA13, cc: freetokenMinCompute}, "FreeToken " + version}, }, h) } diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 0f75d86..bae6dce 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -222,6 +222,7 @@ func Collect(ctx context.Context, o Options) Facts { f.Fabrics = discovery.CheckFabrics(declared, f.HCAs) } f.Host.MinComputeCapability = discovery.MinComputeCapability(f.GPUs) + f.Host.DriverCUDA = discovery.DriverCUDA(f.GPUs) if f.VLLMVersion = o.VLLMVersion; f.VLLMVersion != "" { if f.EnginesDir == "" { f.EnginesDir = provision.DefaultRoot