Files
llmgw/internal/agent/discovery/pcie.go
T
Ilker ManapandClaude Opus 5.5 721705b29d A card in the wrong PCIe slot is said so
At the user's asking: we know what a card's link could be, so a card
linked slower is reported. Each agent reads, apart from the GPU query so
that a driver not knowing a field costs nothing else, the fastest PCIe
generation card and slot allow together, the card's own, and the lanes in
use and the card's most (nvidia-smi; NVML in the nvml build). The current
generation is not compared: it drops while a card idles. A card on fewer
lanes, or a slot of an older generation, is a node warning with the GB/s
it carries against what it could — on the dashboard, the console and in
llmgw doctor, which points at lspci's LnkCap and LnkSta. MIG instances are
said of their card. GPU proto fields 23-26.

The PoC's Quadro RTX 6000 reads Gen3 of Gen3, x16 of x16, idle at Gen1:
nothing to say.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:49:16 +01:00

117 lines
3.8 KiB
Go

package discovery
import (
"context"
"encoding/csv"
"fmt"
"os/exec"
"strconv"
"strings"
)
// A card in the wrong slot: an x16 card in a slot wired x8 or x4, behind a
// riser, or in a slot of an older PCIe generation than the card's, links
// slower than it was built to, and nothing says so. A model on the card
// still decodes at the card's own memory speed, but it loads slower, a
// tensor-parallel replica over PCIe is slower, and so is anything that
// spills to the host. The driver says how the link stands and what the
// card could do; the difference is reported where the node's warnings are.
// PCIeLink is how a card is linked to its host.
type PCIeLink struct {
// Gen is the fastest generation the card and the slot together allow —
// not the current one, which drops while the card idles, to save power.
// GPUGen is the card's own fastest, 0 when the driver does not say.
Gen int `json:"gen,omitempty"`
GPUGen int `json:"gpu_gen,omitempty"`
// Width is the lanes in use now, WidthMax the card's most.
Width int `json:"width,omitempty"`
WidthMax int `json:"width_max,omitempty"`
}
// laneGBs is one lane's usable rate in GB/s, by generation.
var laneGBs = map[int]float64{1: 0.25, 2: 0.5, 3: 0.985, 4: 1.969, 5: 3.938, 6: 7.877}
// GBs is what a link of gen and width carries, in GB/s each way.
func pcieGBs(gen, width int) float64 { return laneGBs[gen] * float64(width) }
// Problem says how a card's link falls short of what the card could do,
// empty when it does not or the driver did not say.
func (p PCIeLink) Problem() string {
gen := p.Gen
if p.GPUGen > 0 && gen == 0 {
gen = p.GPUGen
}
slowGen := p.GPUGen > 0 && p.Gen > 0 && p.Gen < p.GPUGen
narrow := p.WidthMax > 0 && p.Width > 0 && p.Width < p.WidthMax
if !slowGen && !narrow {
return ""
}
could := pcieGBs(max(p.GPUGen, gen), p.WidthMax)
has := pcieGBs(gen, p.Width)
var why []string
if narrow {
why = append(why, fmt.Sprintf("x%d of its x%d lanes", p.Width, p.WidthMax))
}
if slowGen {
why = append(why, fmt.Sprintf("PCIe Gen%d of its Gen%d", p.Gen, p.GPUGen))
}
return fmt.Sprintf("linked at %s: %.1f GB/s where it could have %.1f. "+
"Check the slot (wired fewer lanes, or an older generation), a riser, or the BIOS's PCIe setting",
strings.Join(why, " and "), has, could)
}
// pcieFields are asked apart from the rest, and the card's own fastest
// generation apart again: a driver that does not know a field refuses the
// whole query, and these must not cost a node its cards.
var pcieFields = [][]string{
{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max", "pcie.link.gen.gpumax"},
{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max"},
}
// readPCIe fills each card's link from nvidia-smi, leaving it empty when
// the driver says nothing of it.
func readPCIe(ctx context.Context, binary string, cards []GPU) {
for _, fields := range pcieFields {
out, err := exec.CommandContext(ctx, binary, "--query-gpu="+strings.Join(fields, ","),
"--format=csv,noheader,nounits").Output()
if err != nil {
continue
}
links := parsePCIe(string(out))
for i := range cards {
if l, ok := links[cards[i].Index]; ok && cards[i].MIG == nil {
cards[i].PCIe = l
}
}
return
}
}
func parsePCIe(out string) map[int]PCIeLink {
r := csv.NewReader(strings.NewReader(out))
r.TrimLeadingSpace = true
r.FieldsPerRecord = -1
rows, _ := r.ReadAll()
num := func(s string) int {
n, _ := strconv.Atoi(strings.TrimSpace(s))
return n
}
links := map[int]PCIeLink{}
for _, row := range rows {
if len(row) < 4 {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
if err != nil {
continue
}
l := PCIeLink{Gen: num(row[1]), Width: num(row[2]), WidthMax: num(row[3])}
if len(row) > 4 {
l.GPUGen = num(row[4])
}
links[idx] = l
}
return links
}