diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index bb96c30..2e99e7c 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -3,6 +3,7 @@ package discovery import ( "bufio" "os" + "regexp" "runtime" "strconv" "strings" @@ -142,17 +143,29 @@ func cpuFlagsFrom(path string) []string { // nvidiaDriverFrom reads the loaded kernel module's version, which exists // exactly when a driver is loaded — unlike nvidia-smi, which can be installed // on a machine whose module failed to load. +var driverVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`) + func nvidiaDriverFrom(path string) string { b, err := os.ReadFile(path) //nolint:gosec // a fixed system path if err != nil { return "" } - // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..." + // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ...", or + // for the open module, the default since 560, "NVIDIA UNIX Open Kernel + // Module for x86_64 580.82.07 Release Build ...": the first version + // after "Module", not the word after it. Read as "for", the driver was + // unknown, and every build was tried whatever driver it needs. fields := strings.Fields(string(b)) for i, f := range fields { - if f == "Module" && i+1 < len(fields) { - return fields[i+1] + if f != "Module" { + continue } + for _, v := range fields[i+1:] { + if driverVersion.MatchString(v) { + return v + } + } + return "" } return "" } diff --git a/internal/agent/discovery/host_driver_test.go b/internal/agent/discovery/host_driver_test.go new file mode 100644 index 0000000..c05bdee --- /dev/null +++ b/internal/agent/discovery/host_driver_test.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNVIDIADriverFrom(t *testing.T) { + for _, tc := range []struct{ text, want string }{ + {"NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu Jan 23 21:30:38 UTC 2025\nGCC version: gcc version 12.2.0\n", "570.86.15"}, + {"NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 580.82.07 Release Build (dvs-builder@U16-I3-B03-4-3) Wed Aug 27 00:51:12 UTC 2025\n", "580.82.07"}, + {"NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.95.05 Release Build\n", "580.95.05"}, + {"NVRM version: NVIDIA UNIX x86_64 Kernel Module 535.54 Mon\n", "535.54"}, + {"something else\n", ""}, + } { + p := filepath.Join(t.TempDir(), "version") + if err := os.WriteFile(p, []byte(tc.text), 0o600); err != nil { + t.Fatal(err) + } + if got := nvidiaDriverFrom(p); got != tc.want { + t.Errorf("%q: got %q, want %q", tc.text, got, tc.want) + } + } +} diff --git a/internal/agent/provision/comfyui_test.go b/internal/agent/provision/comfyui_test.go index 4dab243..1c16461 100644 --- a/internal/agent/provision/comfyui_test.go +++ b/internal/agent/provision/comfyui_test.go @@ -91,6 +91,25 @@ func TestComfyUIOnAnOlderDriverTakesPyTorchsCUDA12Torch(t *testing.T) { } } +// A V100 on driver 580: CUDA 13 runs on the driver, but neither its torch +// nor CUDA 12.9's has kernels for Volta. Tried in turn, each was installed, +// 10 GB at a time, and failed on the card. +func TestComfyUIOnAVoltaTakesPyTorchsCUDA126Torch(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + host := turingNode + host.MinComputeCapability = 7.0 + in, rel := pythonInstaller(t, host, sys) + comfyuiSource(t, in, rel, "") + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}) + if got[0].State != Ready || got[0].Variant != "cu126" { + t.Fatalf("got %+v", got[0]) + } + all := strings.Join(sys.commands, "\n") + if !strings.Contains(all, pytorchCU126) || strings.Contains(all, pytorchCU129) { + t.Errorf("torch is not CUDA 12.6's:\n%s", all) + } +} + func TestComfyUIWhoseSourceIsNotTheOneTheRecipeNamesIsNotInstalled(t *testing.T) { sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} in, rel := pythonInstaller(t, turingNode, sys) diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 2f7f77c..78b3b8a 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -599,10 +599,15 @@ const ComfyUIRepo = "Comfy-Org/ComfyUI" // /comfyui/main.py, beside bin/python. const comfyuiDir = "comfyui" +// pytorchCU126 is PyTorch's CUDA 12.6 build: the last that carries kernels +// for Volta and Pascal. From torch 2.8 its CUDA 12.8 and 12.9 builds start +// at Turing (7.5), as CUDA 13 does, so a V100 has no torch but this one. +const pytorchCU126 = "https://download.pytorch.org/whl/cu126" + // comfyuiCandidates are the ComfyUI builds for a machine: its source, in a -// Python environment with the torch for the node's driver — PyPI's, built -// for CUDA 13, or PyTorch's CUDA 12.9 build for a driver CUDA 13 does not -// run on. +// Python environment with the torch for the node's driver and cards — +// PyPI's, built for CUDA 13; PyTorch's CUDA 12.9 build for a driver CUDA 13 +// does not run on; and its CUDA 12.6 build for cards older than Turing. func comfyuiCandidates(version, variant string, h discovery.Host) ([]Build, error) { if _, err := gpuArch("ComfyUI", h); err != nil { return nil, err @@ -647,8 +652,11 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`}, } cu129 := build("cu129") cu129.ExtraIndex, cu129.IndexPackages = pytorchCU129, pytorchPackages + cu126 := build("cu126") + cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages return pick("ComfyUI "+version, variant, []candidate{ {build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, - {cu129, gpuNeeds{minDriverForCUDA12, 7.0}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + {cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + {cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"}, }, h) } diff --git a/internal/web/admin_nodeengine.go b/internal/web/admin_nodeengine.go index 04ffdcc..9d96926 100644 --- a/internal/web/admin_nodeengine.go +++ b/internal/web/admin_nodeengine.go @@ -50,7 +50,7 @@ func buildNotice(l *Lang, node, engine, build string) string { case "none": return l.T("nodeengine.notice.none", node, engine) } - name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu12": "CUDA 12"}[build] + name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu126": "CUDA 12.6", "cu12": "CUDA 12"}[build] if name == "" { name = build }