A V100 gets a ComfyUI it can run, and an open driver is read by its version
PyTorch's CUDA 12.8 and 12.9 builds start at Turing since torch 2.8, as CUDA 13 does, so on a V100 both ComfyUI builds were installed, 10 GB each, and failed on the card with "no kernel image". The cu129 build now needs 7.5, and a third, on PyTorch's CUDA 12.6 torch, carries Volta and Pascal. The open kernel module, NVIDIA's default since 560, writes "Open Kernel Module for x86_64 580.82.07" in /proc/driver/nvidia/version, and the word after "Module" was taken as the driver: "for". With the driver unknown, every build was tried whatever driver it needs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7781b6a308
commit
52898fbb80
5 files changed
+73
-8
No files matched your search
@@ -3,6 +3,7 @@ package discovery
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -142,17 +143,29 @@ func cpuFlagsFrom(path string) []string {
|
||||
// nvidiaDriverFrom reads the loaded kernel module's version, which exists
|
||||
// exactly when a driver is loaded — unlike nvidia-smi, which can be installed
|
||||
// on a machine whose module failed to load.
|
||||
var driverVersion = regexp.MustCompile(`^[0-9]+\.[0-9]+(\.[0-9]+)?$`)
|
||||
|
||||
func nvidiaDriverFrom(path string) string {
|
||||
b, err := os.ReadFile(path) //nolint:gosec // a fixed system path
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
// "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ..."
|
||||
// "NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu ...", or
|
||||
// for the open module, the default since 560, "NVIDIA UNIX Open Kernel
|
||||
// Module for x86_64 580.82.07 Release Build ...": the first version
|
||||
// after "Module", not the word after it. Read as "for", the driver was
|
||||
// unknown, and every build was tried whatever driver it needs.
|
||||
fields := strings.Fields(string(b))
|
||||
for i, f := range fields {
|
||||
if f == "Module" && i+1 < len(fields) {
|
||||
return fields[i+1]
|
||||
if f != "Module" {
|
||||
continue
|
||||
}
|
||||
for _, v := range fields[i+1:] {
|
||||
if driverVersion.MatchString(v) {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNVIDIADriverFrom(t *testing.T) {
|
||||
for _, tc := range []struct{ text, want string }{
|
||||
{"NVRM version: NVIDIA UNIX x86_64 Kernel Module 570.86.15 Thu Jan 23 21:30:38 UTC 2025\nGCC version: gcc version 12.2.0\n", "570.86.15"},
|
||||
{"NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 580.82.07 Release Build (dvs-builder@U16-I3-B03-4-3) Wed Aug 27 00:51:12 UTC 2025\n", "580.82.07"},
|
||||
{"NVRM version: NVIDIA UNIX Open Kernel Module for aarch64 580.95.05 Release Build\n", "580.95.05"},
|
||||
{"NVRM version: NVIDIA UNIX x86_64 Kernel Module 535.54 Mon\n", "535.54"},
|
||||
{"something else\n", ""},
|
||||
} {
|
||||
p := filepath.Join(t.TempDir(), "version")
|
||||
if err := os.WriteFile(p, []byte(tc.text), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := nvidiaDriverFrom(p); got != tc.want {
|
||||
t.Errorf("%q: got %q, want %q", tc.text, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,25 @@ func TestComfyUIOnAnOlderDriverTakesPyTorchsCUDA12Torch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A V100 on driver 580: CUDA 13 runs on the driver, but neither its torch
|
||||
// nor CUDA 12.9's has kernels for Volta. Tried in turn, each was installed,
|
||||
// 10 GB at a time, and failed on the card.
|
||||
func TestComfyUIOnAVoltaTakesPyTorchsCUDA126Torch(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
host := turingNode
|
||||
host.MinComputeCapability = 7.0
|
||||
in, rel := pythonInstaller(t, host, sys)
|
||||
comfyuiSource(t, in, rel, "")
|
||||
got := in.Converge(t.Context(), []Want{{Engine: "comfyui", Version: "0.39.2"}})
|
||||
if got[0].State != Ready || got[0].Variant != "cu126" {
|
||||
t.Fatalf("got %+v", got[0])
|
||||
}
|
||||
all := strings.Join(sys.commands, "\n")
|
||||
if !strings.Contains(all, pytorchCU126) || strings.Contains(all, pytorchCU129) {
|
||||
t.Errorf("torch is not CUDA 12.6's:\n%s", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComfyUIWhoseSourceIsNotTheOneTheRecipeNamesIsNotInstalled(t *testing.T) {
|
||||
sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
|
||||
in, rel := pythonInstaller(t, turingNode, sys)
|
||||
|
||||
@@ -599,10 +599,15 @@ const ComfyUIRepo = "Comfy-Org/ComfyUI"
|
||||
// <environment>/comfyui/main.py, beside bin/python.
|
||||
const comfyuiDir = "comfyui"
|
||||
|
||||
// pytorchCU126 is PyTorch's CUDA 12.6 build: the last that carries kernels
|
||||
// for Volta and Pascal. From torch 2.8 its CUDA 12.8 and 12.9 builds start
|
||||
// at Turing (7.5), as CUDA 13 does, so a V100 has no torch but this one.
|
||||
const pytorchCU126 = "https://download.pytorch.org/whl/cu126"
|
||||
|
||||
// comfyuiCandidates are the ComfyUI builds for a machine: its source, in a
|
||||
// Python environment with the torch for the node's driver — PyPI's, built
|
||||
// for CUDA 13, or PyTorch's CUDA 12.9 build for a driver CUDA 13 does not
|
||||
// run on.
|
||||
// Python environment with the torch for the node's driver and cards —
|
||||
// PyPI's, built for CUDA 13; PyTorch's CUDA 12.9 build for a driver CUDA 13
|
||||
// does not run on; and its CUDA 12.6 build for cards older than Turing.
|
||||
func comfyuiCandidates(version, variant string, h discovery.Host) ([]Build, error) {
|
||||
if _, err := gpuArch("ComfyUI", h); err != nil {
|
||||
return nil, err
|
||||
@@ -647,8 +652,11 @@ print("comfyui version", "` + version + `", "on torch", torch.__version__)`},
|
||||
}
|
||||
cu129 := build("cu129")
|
||||
cu129.ExtraIndex, cu129.IndexPackages = pytorchCU129, pytorchPackages
|
||||
cu126 := build("cu126")
|
||||
cu126.ExtraIndex, cu126.IndexPackages = pytorchCU126, pytorchPackages
|
||||
return pick("ComfyUI "+version, variant, []candidate{
|
||||
{build("cu13"), gpuNeeds{minDriverForCUDA13, 7.5}, "ComfyUI " + version + " on CUDA 13's torch"},
|
||||
{cu129, gpuNeeds{minDriverForCUDA12, 7.0}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||
{cu129, gpuNeeds{minDriverForCUDA12, 7.5}, "ComfyUI " + version + " on CUDA 12.9's torch"},
|
||||
{cu126, gpuNeeds{minDriverForCUDA12, 5.0}, "ComfyUI " + version + " on CUDA 12.6's torch"},
|
||||
}, h)
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func buildNotice(l *Lang, node, engine, build string) string {
|
||||
case "none":
|
||||
return l.T("nodeengine.notice.none", node, engine)
|
||||
}
|
||||
name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu12": "CUDA 12"}[build]
|
||||
name := map[string]string{"cu13": "CUDA 13", "cu129": "CUDA 12.9", "cu126": "CUDA 12.6", "cu12": "CUDA 12"}[build]
|
||||
if name == "" {
|
||||
name = build
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user