From 52898fbb8016de5bb1c05cbeac1f157da7ad5087 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 13 Oct 2026 10:10:20 +0200 Subject: [PATCH] A V100 gets a ComfyUI it can run, and an open driver is read by its version PyTorch's CUDA 12.8 and 12.9 builds start at Turing since torch 2.8, as CUDA 13 does, so on a V100 both ComfyUI builds were installed, 10 GB each, and failed on the card with "no kernel image". The cu129 build now needs 7.5, and a third, on PyTorch's CUDA 12.6 torch, carries Volta and Pascal. The open kernel module, NVIDIA's default since 560, writes "Open Kernel Module for x86_64 580.82.07" in /proc/driver/nvidia/version, and the word after "Module" was taken as the driver: "for". With the driver unknown, every build was tried whatever driver it needs. Co-Authored-By: Claude Opus 5.5 --- internal/agent/discovery/host.go | 19 ++++++++++++--- internal/agent/discovery/host_driver_test.go | 25 ++++++++++++++++++++ internal/agent/provision/comfyui_test.go | 19 +++++++++++++++ internal/agent/provision/recipe.go | 16 +++++++++---- internal/web/admin_nodeengine.go | 2 +- 5 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 internal/agent/discovery/host_driver_test.go diff --git a/internal/agent/discovery/host.go b/internal/agent/discovery/host.go index bb96c30..2e99e7c 100644 --- a/internal/agent/discovery/host.go +++ b/internal/agent/discovery/host.go @@ -3,6 +3,7 @@ package discovery import ( "bufio" "os" + "regexp" "runtime" "strconv" "strings" @@ -142,17 +143,29 @@ func cpuFlagsFrom(path string) []string { // nvidiaDriverFrom reads the loaded kernel module's version, which exists // exactly when a driver is loaded — unlike nvidia-smi, which can be installed // on a machine whose module failed to load. +var driverVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`) + func nvidiaDriverFrom(path string) string { b, err := os.ReadFile(path) //nolint:gosec // a fixed system path if err != nil { return "" } - // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..." + // "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ...", or + // for the open module, the default since 560, "NVIDIA UNIX Open Kernel + // Module for x86_64 580.82.07 Release Build ...": the first version + // after "Module", not the word after it. Read as "for", the driver was + // unknown, and every build was tried whatever driver it needs. fields := strings.Fields(string(b)) for i, f := range fields { - if f == "Module" && i+1 < len(fields) { - return fields[i+1] + if f != "Module" { + continue } + for _, v := range fields[i+1:] { + if driverVersion.MatchString(v) { + return v + } + } + return "" } return "" } diff --git a/internal/agent/discovery/host_driver_test.go b/internal/agent/discovery/host_driver_test.go new file mode 100644 index 0000000..c05bdee --- /dev/null +++ b/internal/agent/discovery/host_driver_test.go @@ -0,0 +1,25 @@ +package discovery + +import ( + "os" + "path/filepath" + "testing" +) + +func TestNVIDIADriverFrom(t *testing.T) { + for _, tc := range []struct{ text, want string }{ + {"NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu Jan 23 21:30:38 UTC 2025\nGCC version: gcc version 12.2.0\n", "570.86.15"}, + {"NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 580.82.07 Release Build (dvs-builder@U16-I3-B03-4-3) Wed Aug 27 00:51:12 UTC 2025\n", "580.82.07"}, + {"NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.95.05 Release Build\n", "580.95.05"}, + {"NVRM version: NVIDIA UNIX x86_64 Kernel Module 535.54 Mon\n", "535.54"}, + {"something else\n", ""}, + } { + p := filepath.Join(t.TempDir(), "version") + if err := os.WriteFile(p, []byte(tc.text), 0o600); err != nil { + t.Fatal(err) + } + if got := nvidiaDriverFrom(p); got != tc.want { + t.Errorf("%q: got %q, want %q", tc.text, got, tc.want) + } + } +} diff --git a/internal/agent/provision/comfyui_test.go b/internal/agent/provision/comfyui_test.go index 4dab243..1c16461 100644 --- a/internal/agent/provision/comfyui_test.go +++ b/internal/agent/provision/comfyui_test.go @@ -91,6 +91,25 @@ func TestComfyUIOnAnOlderDriverTakesPyTorchsCUDA12Torch(t *testing.T) { } } +// A V100 on driver 580: CUDA 13 runs on the driver, but neither its torch +// nor CUDA 12.9's has kernels for Volta. Tried in turn, each was installed, +// 10 GB at a time, and failed on the card. +func TestComfyUIOnAVoltaTakesPyTorchsCUDA126Torch(t *testing.T) { + sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} + host := turingNode + host.MinComputeCapability = 7.0 + in, rel := pythonInstaller(t, host, sys) + comfyuiSource(t, in, rel, "") + got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}}) + if got[0].State != Ready || got[0].Variant != "cu126" { + t.Fatalf("got %+v", got[0]) + } + all := strings.Join(sys.commands, "\n") + if !strings.Contains(all, pytorchCU126) || strings.Contains(all, pytorchCU129) { + t.Errorf("torch is not CUDA 12.6's:\n%s", all) + } +} + func TestComfyUIWhoseSourceIsNotTheOneTheRecipeNamesIsNotInstalled(t *testing.T) { sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}} in, rel := pythonInstaller(t, turingNode, sys) diff --git a/internal/agent/provision/recipe.go b/internal/agent/provision/recipe.go index 2f7f77c..78b3b8a 100644 --- a/internal/agent/provision/recipe.go +++ b/internal/agent/provision/recipe.go @@ -599,10 +599,15 @@ const ComfyUIRepo = "Comfy-Org/ComfyUI" // /comfyui/main.py, beside bin/python. const comfyuiDir = "comfyui" +// pytorchCU126 is PyTorch's CUDA 12.6 build: the last that carries kernels +// for Volta and Pascal. From torch 2.8 its CUDA 12.8 and 12.9 builds start +// at Turing (7.5), as CUDA 13 does, so a V100 has no torch but this one. +const pytorchCU126 = "https://download.pytorch.org/whl/cu126" + // comfyuiCandidates are the ComfyUI builds for a machine: its source, in a -// Python environment with the torch for the node's driver — PyPI's, built -// for CUDA 13, or PyTorch's CUDA 12.9 build for a driver CUDA 13 does not -// run on. +// Python environment with the torch for the node's driver and cards — +// PyPI's, built for CUDA 13; PyTorch's CUDA 12.9 build for a driver CUDA 13 +// does not run on; and its CUDA 12.6 build for cards older than Turing. func comfyuiCandidates(version, variant string, h discovery.Host) ([]Build, error) { if _, err := gpuArch("ComfyUI", h); err != nil { return nil, err @@ -647,8 +652,11 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`}, } cu129 := build("cu129") cu129.ExtraIndex, cu129.IndexPackages = pytorchCU129, pytorchPackages + cu126 := build("cu126") + cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages return pick("ComfyUI "+version, variant, []candidate{ {build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"}, - {cu129, gpuNeeds{minDriverForCUDA12, 7.0}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + {cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"}, + {cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"}, }, h) } diff --git a/internal/web/admin_nodeengine.go b/internal/web/admin_nodeengine.go index 04ffdcc..9d96926 100644 --- a/internal/web/admin_nodeengine.go +++ b/internal/web/admin_nodeengine.go @@ -50,7 +50,7 @@ func buildNotice(l *Lang, node, engine, build string) string { case "none": return l.T("nodeengine.notice.none", node, engine) } - name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu12": "CUDA 12"}[build] + name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu126": "CUDA 12.6", "cu12": "CUDA 12"}[build] if name == "" { name = build }