diff --git a/internal/agent/discovery/gpu_smi.go b/internal/agent/discovery/gpu_smi.go index b8b2b9b..8b807f9 100644 --- a/internal/agent/discovery/gpu_smi.go +++ b/internal/agent/discovery/gpu_smi.go @@ -65,6 +65,11 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { return cards, err } readPCIe(ctx, binary, cards) + if cuda := driverCUDAFromSMI(ctx, binary); cuda != "" { + for i := range cards { + cards[i].CUDAVersion = cuda + } + } // The query lists whole cards, MIG mode or not. A card in MIG mode is // replaced by its instances, which only -L names and only the full // report sizes; asked only when some card could be in MIG mode, since @@ -173,3 +178,40 @@ func assessHealth(g *GPU) { g.Health.Problems = append(g.Health.Problems, p) } } + +// driverCUDAFromSMI is the newest CUDA the driver runs, from `nvidia-smi +// --version`: "CUDA version : 12.4" on older drivers, "CUDA UMD version : +// 13.4" on newer ones, where the old line says "Deprecated". Empty when +// neither says; the query fields have no such column. +func driverCUDAFromSMI(ctx context.Context, binary string) string { + out, err := exec.CommandContext(ctx, binary, "--version").Output() + if err != nil { + return "" + } + return parseSMIDriverCUDA(string(out)) +} + +func parseSMIDriverCUDA(out string) string { + found := "" + for _, line := range strings.Split(out, "\n") { + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + key, val = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(val) + if !strings.HasPrefix(key, "cuda") || !strings.HasSuffix(key, "version") { + continue + } + major, minor, ok := strings.Cut(val, ".") + if _, err := strconv.Atoi(major); err != nil || !ok { + continue + } + if _, err := strconv.Atoi(minor); err != nil { + continue + } + if strings.Contains(key, "umd") || found == "" { + found = val + } + } + return found +} diff --git a/internal/agent/discovery/gpu_test.go b/internal/agent/discovery/gpu_test.go index 034d233..3766223 100644 --- a/internal/agent/discovery/gpu_test.go +++ b/internal/agent/discovery/gpu_test.go @@ -177,3 +177,18 @@ func TestUtilisationIsReadWhenPresent(t *testing.T) { t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent) } } + +func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) { + for out, want := range map[string]string{ + // cuda01, driver 615. + "NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" + + "CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4", + // An older driver. + "NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4", + "nothing useful": "", + } { + if got := parseSMIDriverCUDA(out); got != want { + t.Errorf("%q: %q, want %q", out, got, want) + } + } +}