The driver's CUDA is read from nvidia-smi --version on the portable build
CI / test (push) Failing after 10m13s
CI / test (push) Failing after 10m13s
Only the NVML build filled it, and the release's portable binary, the one nodes run, never did: the CUDA 13.x check that came before this saw nothing on cuda01 and installed b12606's CUDA 13.5 build again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6765d430d8
commit
754ab256a2
2 files changed
+57
No files matched your search
@@ -65,6 +65,11 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
|
||||
return cards, err
|
||||
}
|
||||
readPCIe(ctx, binary, cards)
|
||||
if cuda := driverCUDAFromSMI(ctx, binary); cuda != "" {
|
||||
for i := range cards {
|
||||
cards[i].CUDAVersion = cuda
|
||||
}
|
||||
}
|
||||
// The query lists whole cards, MIG mode or not. A card in MIG mode is
|
||||
// replaced by its instances, which only -L names and only the full
|
||||
// report sizes; asked only when some card could be in MIG mode, since
|
||||
@@ -173,3 +178,40 @@ func assessHealth(g *GPU) {
|
||||
g.Health.Problems = append(g.Health.Problems, p)
|
||||
}
|
||||
}
|
||||
|
||||
// driverCUDAFromSMI is the newest CUDA the driver runs, from `nvidia-smi
|
||||
// --version`: "CUDA version : 12.4" on older drivers, "CUDA UMD version :
|
||||
// 13.4" on newer ones, where the old line says "Deprecated". Empty when
|
||||
// neither says; the query fields have no such column.
|
||||
func driverCUDAFromSMI(ctx context.Context, binary string) string {
|
||||
out, err := exec.CommandContext(ctx, binary, "--version").Output()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return parseSMIDriverCUDA(string(out))
|
||||
}
|
||||
|
||||
func parseSMIDriverCUDA(out string) string {
|
||||
found := ""
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
key, val, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key, val = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(val)
|
||||
if !strings.HasPrefix(key, "cuda") || !strings.HasSuffix(key, "version") {
|
||||
continue
|
||||
}
|
||||
major, minor, ok := strings.Cut(val, ".")
|
||||
if _, err := strconv.Atoi(major); err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
if _, err := strconv.Atoi(minor); err != nil {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(key, "umd") || found == "" {
|
||||
found = val
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
@@ -177,3 +177,18 @@ func TestUtilisationIsReadWhenPresent(t *testing.T) {
|
||||
t.Errorf("utilisation = %d, want 87", gpus[0].UtilizationPercent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheDriversCUDAIsReadFromNvidiaSMIVersion(t *testing.T) {
|
||||
for out, want := range map[string]string{
|
||||
// cuda01, driver 615.
|
||||
"NVIDIA-SMI version : 615.71.09\nNVML version : 615.71\nDRIVER version : Deprecated, see \"KMD version\" instead\n" +
|
||||
"CUDA version : Deprecated, see \"CUDA UMD version\" instead\nKMD version : 615.71.09\nCUDA UMD version : 13.4\n": "13.4",
|
||||
// An older driver.
|
||||
"NVIDIA-SMI version : 550.54.15\nNVML version : 550.54\nDRIVER version : 550.54.15\nCUDA Version : 12.4\n": "12.4",
|
||||
"nothing useful": "",
|
||||
} {
|
||||
if got := parseSMIDriverCUDA(out); got != want {
|
||||
t.Errorf("%q: %q, want %q", out, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user