CI / test (push) Successful in 9m17s
At the user's asking: we know what a card's link could be, so a card linked slower is reported. Each agent reads, apart from the GPU query so that a driver not knowing a field costs nothing else, the fastest PCIe generation card and slot allow together, the card's own, and the lanes in use and the card's most (nvidia-smi; NVML in the nvml build). The current generation is not compared: it drops while a card idles. A card on fewer lanes, or a slot of an older generation, is a node warning with the GB/s it carries against what it could — on the dashboard, the console and in llmgw doctor, which points at lspci's LnkCap and LnkSta. MIG instances are said of their card. GPU proto fields 23-26. The PoC's Quadro RTX 6000 reads Gen3 of Gen3, x16 of x16, idle at Gen1: nothing to say. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
3.8 KiB
Go
117 lines
3.8 KiB
Go
package discovery
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"fmt"
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// A card in the wrong slot: an x16 card in a slot wired x8 or x4, behind a
|
|
// riser, or in a slot of an older PCIe generation than the card's, links
|
|
// slower than it was built to, and nothing says so. A model on the card
|
|
// still decodes at the card's own memory speed, but it loads slower, a
|
|
// tensor-parallel replica over PCIe is slower, and so is anything that
|
|
// spills to the host. The driver says how the link stands and what the
|
|
// card could do; the difference is reported where the node's warnings are.
|
|
|
|
// PCIeLink is how a card is linked to its host.
|
|
type PCIeLink struct {
|
|
// Gen is the fastest generation the card and the slot together allow —
|
|
// not the current one, which drops while the card idles, to save power.
|
|
// GPUGen is the card's own fastest, 0 when the driver does not say.
|
|
Gen int `json:"gen,omitempty"`
|
|
GPUGen int `json:"gpu_gen,omitempty"`
|
|
// Width is the lanes in use now, WidthMax the card's most.
|
|
Width int `json:"width,omitempty"`
|
|
WidthMax int `json:"width_max,omitempty"`
|
|
}
|
|
|
|
// laneGBs is one lane's usable rate in GB/s, by generation.
|
|
var laneGBs = map[int]float64{1: 0.25, 2: 0.5, 3: 0.985, 4: 1.969, 5: 3.938, 6: 7.877}
|
|
|
|
// GBs is what a link of gen and width carries, in GB/s each way.
|
|
func pcieGBs(gen, width int) float64 { return laneGBs[gen] * float64(width) }
|
|
|
|
// Problem says how a card's link falls short of what the card could do,
|
|
// empty when it does not or the driver did not say.
|
|
func (p PCIeLink) Problem() string {
|
|
gen := p.Gen
|
|
if p.GPUGen > 0 && gen == 0 {
|
|
gen = p.GPUGen
|
|
}
|
|
slowGen := p.GPUGen > 0 && p.Gen > 0 && p.Gen < p.GPUGen
|
|
narrow := p.WidthMax > 0 && p.Width > 0 && p.Width < p.WidthMax
|
|
if !slowGen && !narrow {
|
|
return ""
|
|
}
|
|
could := pcieGBs(max(p.GPUGen, gen), p.WidthMax)
|
|
has := pcieGBs(gen, p.Width)
|
|
var why []string
|
|
if narrow {
|
|
why = append(why, fmt.Sprintf("x%d of its x%d lanes", p.Width, p.WidthMax))
|
|
}
|
|
if slowGen {
|
|
why = append(why, fmt.Sprintf("PCIe Gen%d of its Gen%d", p.Gen, p.GPUGen))
|
|
}
|
|
return fmt.Sprintf("linked at %s: %.1f GB/s where it could have %.1f. "+
|
|
"Check the slot (wired fewer lanes, or an older generation), a riser, or the BIOS's PCIe setting",
|
|
strings.Join(why, " and "), has, could)
|
|
}
|
|
|
|
// pcieFields are asked apart from the rest, and the card's own fastest
|
|
// generation apart again: a driver that does not know a field refuses the
|
|
// whole query, and these must not cost a node its cards.
|
|
var pcieFields = [][]string{
|
|
{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max", "pcie.link.gen.gpumax"},
|
|
{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max"},
|
|
}
|
|
|
|
// readPCIe fills each card's link from nvidia-smi, leaving it empty when
|
|
// the driver says nothing of it.
|
|
func readPCIe(ctx context.Context, binary string, cards []GPU) {
|
|
for _, fields := range pcieFields {
|
|
out, err := exec.CommandContext(ctx, binary, "--query-gpu="+strings.Join(fields, ","),
|
|
"--format=csv,noheader,nounits").Output()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
links := parsePCIe(string(out))
|
|
for i := range cards {
|
|
if l, ok := links[cards[i].Index]; ok && cards[i].MIG == nil {
|
|
cards[i].PCIe = l
|
|
}
|
|
}
|
|
return
|
|
}
|
|
}
|
|
|
|
func parsePCIe(out string) map[int]PCIeLink {
|
|
r := csv.NewReader(strings.NewReader(out))
|
|
r.TrimLeadingSpace = true
|
|
r.FieldsPerRecord = -1
|
|
rows, _ := r.ReadAll()
|
|
num := func(s string) int {
|
|
n, _ := strconv.Atoi(strings.TrimSpace(s))
|
|
return n
|
|
}
|
|
links := map[int]PCIeLink{}
|
|
for _, row := range rows {
|
|
if len(row) < 4 {
|
|
continue
|
|
}
|
|
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
l := PCIeLink{Gen: num(row[1]), Width: num(row[2]), WidthMax: num(row[3])}
|
|
if len(row) > 4 {
|
|
l.GPUGen = num(row[4])
|
|
}
|
|
links[idx] = l
|
|
}
|
|
return links
|
|
}
|