The driver's CUDA is read from nvidia-smi --version on the portable build
CI / test (push) Failing after 10m13s

Only the NVML build filled it, and the release's portable binary, the
one nodes run, never did: the CUDA 13.x check that came before this saw
nothing on cuda01 and installed b12606's CUDA 13.5 build again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-21 14:02:27 +01:00
1 parent 6765d430d8
commit 754ab256a2
2 files changed
+57

No files matched your search

+42
View File
@@ -65,6 +65,11 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
return cards, err
}
readPCIe(ctx, binary, cards)
if cuda := driverCUDAFromSMI(ctx, binary); cuda != "" {
for i := range cards {
cards[i].CUDAVersion = cuda
}
}
// The query lists whole cards, MIG mode or not. A card in MIG mode is
// replaced by its instances, which only -L names and only the full
// report sizes; asked only when some card could be in MIG mode, since
@@ -173,3 +178,40 @@ func assessHealth(g *GPU) {
g.Health.Problems = append(g.Health.Problems, p)
}
}
// driverCUDAFromSMI is the newest CUDA the driver runs, from `nvidia-smi
// --version`: "CUDA version : 12.4" on older drivers, "CUDA UMD version :
// 13.4" on newer ones, where the old line says "Deprecated". Empty when
// neither says; the query fields have no such column.
func driverCUDAFromSMI(ctx context.Context, binary string) string {
out, err := exec.CommandContext(ctx, binary, "--version").Output()
if err != nil {
return ""
}
return parseSMIDriverCUDA(string(out))
}
func parseSMIDriverCUDA(out string) string {
found := ""
for _, line := range strings.Split(out, "\n") {
key, val, ok := strings.Cut(line, ":")
if !ok {
continue
}
key, val = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(val)
if !strings.HasPrefix(key, "cuda") || !strings.HasSuffix(key, "version") {
continue
}
major, minor, ok := strings.Cut(val, ".")
if _, err := strconv.Atoi(major); err != nil || !ok {
continue
}
if _, err := strconv.Atoi(minor); err != nil {
continue
}
if strings.Contains(key, "umd") || found == "" {
found = val
}
}
return found
}
+15
View File
@@ -177,3 +177,18 @@ func TestUtilisationIsReadWhenPresent(t *testing.T) {
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
}
}
func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) {
for out, want := range map[string]string{
// cuda01, driver 615.
"NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" +
"CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4",
// An older driver.
"NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4",
"nothing useful": "",
} {
if got := parseSMIDriverCUDA(out); got != want {
t.Errorf("%q: %q, want %q", out, got, want)
}
}
}