A card in the wrong PCIe slot is said so

At the user's asking: we know what a card's link could be, so a card
linked slower is reported. Each agent reads, apart from the GPU query so
that a driver not knowing a field costs nothing else, the fastest PCIe
generation card and slot allow together, the card's own, and the lanes in
use and the card's most (nvidia-smi; NVML in the nvml build). The current
generation is not compared: it drops while a card idles. A card on fewer
lanes, or a slot of an older generation, is a node warning with the GB/s
it carries against what it could — on the dashboard, the console and in
llmgw doctor, which points at lspci's LnkCap and LnkSta. MIG instances are
said of their card. GPU proto fields 23-26.

The PoC's Quadro RTX 6000 reads Gen3 of Gen3, x16 of x16, idle at Gen1:
nothing to say.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -95,6 +95,11 @@ APIs change between minors without a deprecation period.
 
 ### Added
 
+- **A card in the wrong PCIe slot is said so.** Each agent reads how every
+  card is linked — the fastest generation card and slot allow together, and
+  the lanes in use — against the card's own most; a card on fewer lanes or
+  an older generation is a node warning, with what it carries and what it
+  could, on the dashboard, the console and in `llmgw doctor`.
 - **And on a web page, `/admin/mcp`.** For administrators: the same list,
   adding a server with its credential sealed, testing it and listing its
   tools, granting them in the tree beside that level's other grants and

api/gen/llmgw/v1/agent.pb.go

@@ -1589,7 +1589,14 @@ type GPU struct {
 	// is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and
 	// memory the instance's share; utilization_percent stays zero, because no
 	// driver measures it per instance.
-	Mig           *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
+	Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
+	// How the card is linked to its host: the fastest generation card and
+	// slot allow together, the card's own fastest, and the lanes in use and
+	// the card's most. Zero when the driver did not say.
+	PcieGen       int32 `protobuf:"varint,23,opt,name=pcie_gen,json=pcieGen,proto3" json:"pcie_gen,omitempty"`
+	PcieGpuGen    int32 `protobuf:"varint,24,opt,name=pcie_gpu_gen,json=pcieGpuGen,proto3" json:"pcie_gpu_gen,omitempty"`
+	PcieWidth     int32 `protobuf:"varint,25,opt,name=pcie_width,json=pcieWidth,proto3" json:"pcie_width,omitempty"`
+	PcieWidthMax  int32 `protobuf:"varint,26,opt,name=pcie_width_max,json=pcieWidthMax,proto3" json:"pcie_width_max,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -1778,6 +1785,34 @@ func (x *GPU) GetMig() *MIGDevice {
 	return nil
 }
 
+func (x *GPU) GetPcieGen() int32 {
+	if x != nil {
+		return x.PcieGen
+	}
+	return 0
+}
+
+func (x *GPU) GetPcieGpuGen() int32 {
+	if x != nil {
+		return x.PcieGpuGen
+	}
+	return 0
+}
+
+func (x *GPU) GetPcieWidth() int32 {
+	if x != nil {
+		return x.PcieWidth
+	}
+	return 0
+}
+
+func (x *GPU) GetPcieWidthMax() int32 {
+	if x != nil {
+		return x.PcieWidthMax
+	}
+	return 0
+}
+
 // MIGDevice is which card a MIG instance is carved from, and how.
 type MIGDevice struct {
 	state       protoimpl.MessageState `protogen:"open.v1"`
@@ -5251,7 +5286,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"leaderPort\x12\x1d\n" +
 	"\n" +
 	"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12+\n" +
-	"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xca\x06\n" +
+	"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xcc\a\n" +
 	"\x03GPU\x12\x14\n" +
 	"\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" +
 	"\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" +
@@ -5277,7 +5312,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" +
 	"\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" +
 	"\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x12%\n" +
-	"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x1a;\n" +
+	"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x12\x19\n" +
+	"\bpcie_gen\x18\x17 \x01(\x05R\apcieGen\x12 \n" +
+	"\fpcie_gpu_gen\x18\x18 \x01(\x05R\n" +
+	"pcieGpuGen\x12\x1d\n" +
+	"\n" +
+	"pcie_width\x18\x19 \x01(\x05R\tpcieWidth\x12$\n" +
+	"\x0epcie_width_max\x18\x1a \x01(\x05R\fpcieWidthMax\x1a;\n" +
 	"\rNicPathsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
 	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x84\x02\n" +

api/proto/llmgw/v1/agent.proto

@@ -337,6 +337,13 @@ message GPU {
   // memory the instance's share; utilization_percent stays zero, because no
   // driver measures it per instance.
   MIGDevice mig = 22;
+  // How the card is linked to its host: the fastest generation card and
+  // slot allow together, the card's own fastest, and the lanes in use and
+  // the card's most. Zero when the driver did not say.
+  int32 pcie_gen = 23;
+  int32 pcie_gpu_gen = 24;
+  int32 pcie_width = 25;
+  int32 pcie_width_max = 26;
 }
 
 // MIGDevice is which card a MIG instance is carved from, and how.

docs/install.md

@@ -1053,6 +1053,27 @@ A node with no location is placed as before, as a rack of its own.
 See [facility-aware-placement.md](facility-aware-placement.md) for power caps
 and holds.
 
+### A card in the wrong slot
+
+An x16 card in a slot wired x8 or x4, behind a riser, or in a slot of an
+older PCIe generation than the card's, links slower than it was built to and
+works otherwise. A model on the card still decodes at the card's own memory
+speed, but it loads slower, a replica split over several cards by PCIe is
+slower, and so is anything that spills to the host. Each card's agent reads
+how its link stands — the fastest generation card and slot allow together,
+and the lanes in use — against what the card could do, and the node says so
+among its warnings (the dashboard, the console's Nodes, `llmgw doctor`):
+
+```
+GPU 1 (NVIDIA H100 80GB HBM3) linked at x8 of its x16 lanes: 31.5 GB/s where it
+could have 63.0. Check the slot (wired fewer lanes, or an older generation), a
+riser, or the BIOS's PCIe setting
+```
+
+The generation a card reports now drops while it idles, to save power; what is
+compared is the fastest it may reach, which does not. `lspci -vv -s BUS-ID`
+shows the same: `LnkCap` is the card's, `LnkSta` the link's.
+
 ## 6. Setting the node's CPU model, if it is a VM
 
 Proxmox defaults a VM to `kvm64`, which hides AVX, AVX2, FMA and even SSE4.2

internal/agent/discovery/gpu.go

@@ -52,6 +52,8 @@ type GPU struct {
 	// memory the instance's share. Utilisation stays unreported, because no
 	// driver measures it per instance.
 	MIG *MIGDevice `json:"mig,omitempty"`
+	// PCIe is how the card is linked to its host, zero when not read.
+	PCIe PCIeLink `json:"pcie,omitempty"`
 }
 
 // GPUHealth records conditions that make a GPU unsafe to schedule onto.

internal/agent/discovery/gpu_nvml.go

@@ -67,6 +67,18 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
 		if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS {
 			g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor)
 		}
+		if v, ret := dev.GetMaxPcieLinkGeneration(); ret == nvml.SUCCESS {
+			g.PCIe.Gen = v
+		}
+		if v, ret := dev.GetGpuMaxPcieLinkGeneration(); ret == nvml.SUCCESS {
+			g.PCIe.GPUGen = v
+		}
+		if v, ret := dev.GetCurrPcieLinkWidth(); ret == nvml.SUCCESS {
+			g.PCIe.Width = v
+		}
+		if v, ret := dev.GetMaxPcieLinkWidth(); ret == nvml.SUCCESS {
+			g.PCIe.WidthMax = v
+		}
 		if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS {
 			g.PCIBusID = busIDString(pci.BusId[:])
 			byBusID[normalizeBusID(g.PCIBusID)] = i

internal/agent/discovery/gpu_smi.go

@@ -64,6 +64,7 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) {
 	if err != nil || len(cards) == 0 {
 		return cards, err
 	}
+	readPCIe(ctx, binary, cards)
 	// The query lists whole cards, MIG mode or not. A card in MIG mode is
 	// replaced by its instances, which only -L names and only the full
 	// report sizes; asked only when some card could be in MIG mode, since

internal/agent/discovery/pcie.go

@@ -0,0 +1,116 @@
+package discovery
+
+import (
+	"context"
+	"encoding/csv"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+// A card in the wrong slot: an x16 card in a slot wired x8 or x4, behind a
+// riser, or in a slot of an older PCIe generation than the card's, links
+// slower than it was built to, and nothing says so. A model on the card
+// still decodes at the card's own memory speed, but it loads slower, a
+// tensor-parallel replica over PCIe is slower, and so is anything that
+// spills to the host. The driver says how the link stands and what the
+// card could do; the difference is reported where the node's warnings are.
+
+// PCIeLink is how a card is linked to its host.
+type PCIeLink struct {
+	// Gen is the fastest generation the card and the slot together allow —
+	// not the current one, which drops while the card idles, to save power.
+	// GPUGen is the card's own fastest, 0 when the driver does not say.
+	Gen    int `json:"gen,omitempty"`
+	GPUGen int `json:"gpu_gen,omitempty"`
+	// Width is the lanes in use now, WidthMax the card's most.
+	Width    int `json:"width,omitempty"`
+	WidthMax int `json:"width_max,omitempty"`
+}
+
+// laneGBs is one lane's usable rate in GB/s, by generation.
+var laneGBs = map[int]float64{1: 0.25, 2: 0.5, 3: 0.985, 4: 1.969, 5: 3.938, 6: 7.877}
+
+// GBs is what a link of gen and width carries, in GB/s each way.
+func pcieGBs(gen, width int) float64 { return laneGBs[gen] * float64(width) }
+
+// Problem says how a card's link falls short of what the card could do,
+// empty when it does not or the driver did not say.
+func (p PCIeLink) Problem() string {
+	gen := p.Gen
+	if p.GPUGen > 0 && gen == 0 {
+		gen = p.GPUGen
+	}
+	slowGen := p.GPUGen > 0 && p.Gen > 0 && p.Gen < p.GPUGen
+	narrow := p.WidthMax > 0 && p.Width > 0 && p.Width < p.WidthMax
+	if !slowGen && !narrow {
+		return ""
+	}
+	could := pcieGBs(max(p.GPUGen, gen), p.WidthMax)
+	has := pcieGBs(gen, p.Width)
+	var why []string
+	if narrow {
+		why = append(why, fmt.Sprintf("x%d of its x%d lanes", p.Width, p.WidthMax))
+	}
+	if slowGen {
+		why = append(why, fmt.Sprintf("PCIe Gen%d of its Gen%d", p.Gen, p.GPUGen))
+	}
+	return fmt.Sprintf("linked at %s: %.1f GB/s where it could have %.1f. "+
+		"Check the slot (wired fewer lanes, or an older generation), a riser, or the BIOS's PCIe setting",
+		strings.Join(why, " and "), has, could)
+}
+
+// pcieFields are asked apart from the rest, and the card's own fastest
+// generation apart again: a driver that does not know a field refuses the
+// whole query, and these must not cost a node its cards.
+var pcieFields = [][]string{
+	{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max", "pcie.link.gen.gpumax"},
+	{"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max"},
+}
+
+// readPCIe fills each card's link from nvidia-smi, leaving it empty when
+// the driver says nothing of it.
+func readPCIe(ctx context.Context, binary string, cards []GPU) {
+	for _, fields := range pcieFields {
+		out, err := exec.CommandContext(ctx, binary, "--query-gpu="+strings.Join(fields, ","),
+			"--format=csv,noheader,nounits").Output()
+		if err != nil {
+			continue
+		}
+		links := parsePCIe(string(out))
+		for i := range cards {
+			if l, ok := links[cards[i].Index]; ok && cards[i].MIG == nil {
+				cards[i].PCIe = l
+			}
+		}
+		return
+	}
+}
+
+func parsePCIe(out string) map[int]PCIeLink {
+	r := csv.NewReader(strings.NewReader(out))
+	r.TrimLeadingSpace = true
+	r.FieldsPerRecord = -1
+	rows, _ := r.ReadAll()
+	num := func(s string) int {
+		n, _ := strconv.Atoi(strings.TrimSpace(s))
+		return n
+	}
+	links := map[int]PCIeLink{}
+	for _, row := range rows {
+		if len(row) < 4 {
+			continue
+		}
+		idx, err := strconv.Atoi(strings.TrimSpace(row[0]))
+		if err != nil {
+			continue
+		}
+		l := PCIeLink{Gen: num(row[1]), Width: num(row[2]), WidthMax: num(row[3])}
+		if len(row) > 4 {
+			l.GPUGen = num(row[4])
+		}
+		links[idx] = l
+	}
+	return links
+}

internal/agent/discovery/pcie_test.go

@@ -0,0 +1,37 @@
+package discovery
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestACardInTheWrongSlotIsSaidSo(t *testing.T) {
+	for _, c := range []struct {
+		link PCIeLink
+		want string // empty: nothing to say
+	}{
+		// The PoC's Quadro RTX 6000, idle: current Gen1, but its fastest is 3,
+		// as the card's is, on all 16 lanes. Nothing wrong.
+		{PCIeLink{Gen: 3, GPUGen: 3, Width: 16, WidthMax: 16}, ""},
+		// An x16 card in a slot wired x8.
+		{PCIeLink{Gen: 4, GPUGen: 4, Width: 8, WidthMax: 16}, "x8 of its x16 lanes: 15.8 GB/s where it could have 31.5"},
+		// A Gen5 card in a Gen4 slot.
+		{PCIeLink{Gen: 4, GPUGen: 5, Width: 16, WidthMax: 16}, "PCIe Gen4 of its Gen5: 31.5 GB/s where it could have 63.0"},
+		// Both, behind a riser.
+		{PCIeLink{Gen: 3, GPUGen: 4, Width: 4, WidthMax: 16}, "x4 of its x16 lanes and PCIe Gen3 of its Gen4"},
+		// A driver that says nothing.
+		{PCIeLink{}, ""},
+	} {
+		got := c.link.Problem()
+		if (c.want == "") != (got == "") || !strings.Contains(got, c.want) {
+			t.Errorf("%+v: %q, want %q", c.link, got, c.want)
+		}
+	}
+}
+
+func TestTheLinkIsReadFromNvidiaSMI(t *testing.T) {
+	links := parsePCIe("0, 3, 16, 16, 3\n1, 4, 8, 16, [N/A]\n")
+	if links[0] != (PCIeLink{Gen: 3, GPUGen: 3, Width: 16, WidthMax: 16}) || links[1] != (PCIeLink{Gen: 4, Width: 8, WidthMax: 16}) {
+		t.Errorf("%+v", links)
+	}
+}

internal/control/adminapi/api.go

@@ -707,6 +707,14 @@ func (a *API) Nodes() []NodeView {
 			}
 		}
 		v.MemBandwidthGBs = n.Host.MemBandwidthGBs
+		for _, g := range n.GPUs {
+			if g.MIG != nil {
+				continue // said of the card it is carved from
+			}
+			if p := g.PCIe.Problem(); p != "" {
+				v.Warnings = append(v.Warnings, fmt.Sprintf("GPU %d (%s) %s", g.Index, g.Name, p))
+			}
+		}
 		if h := n.Host; h.OSID != "" {
 			v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch)
 			if h.Arch == "amd64" && !h.HasCPUFlag("avx2") {

internal/control/adminapi/api_test.go

@@ -735,3 +735,17 @@ func TestRemovingANodeRevokesItsCertificate(t *testing.T) {
 		t.Error("the node was removed though its certificate was not revoked")
 	}
 }
+
+func TestANodesCardInTheWrongSlotIsAWarning(t *testing.T) {
+	n := gpuNode("g1", 2)
+	n.GPUs[0].PCIe = discovery.PCIeLink{Gen: 5, GPUGen: 5, Width: 16, WidthMax: 16}
+	n.GPUs[1].PCIe = discovery.PCIeLink{Gen: 5, GPUGen: 5, Width: 8, WidthMax: 16}
+	api, _ := setup(t, testModels(), n)
+	var warned []string
+	for _, v := range api.Nodes() {
+		warned = append(warned, v.Warnings...)
+	}
+	if len(warned) != 1 || !strings.Contains(warned[0], "GPU 1 (NVIDIA H100 80GB HBM3) linked at x8 of its x16 lanes") {
+		t.Errorf("warnings %q", warned)
+	}
+}

internal/control/agentsvc/convert.go

@@ -73,6 +73,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node {
 				Problems:     g.GetProblems(),
 			},
 			MIG: migFrom(g.GetMig()),
+			PCIe: discovery.PCIeLink{Gen: int(g.GetPcieGen()), GPUGen: int(g.GetPcieGpuGen()),
+				Width: int(g.GetPcieWidth()), WidthMax: int(g.GetPcieWidthMax())},
 		})
 	}
 	n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())}
@@ -127,6 +129,10 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string,
 			Problems:           g.Health.Problems,
 			TemperatureC:       int32(g.Health.TemperatureC),
 			EccErrors:          int32(g.Health.ECCErrors),
+			PcieGen:            int32(g.PCIe.Gen),
+			PcieGpuGen:         int32(g.PCIe.GPUGen),
+			PcieWidth:          int32(g.PCIe.Width),
+			PcieWidthMax:       int32(g.PCIe.WidthMax),
 		})
 		if f := g.Fabric; f != nil {
 			last := r.Gpus[len(r.Gpus)-1]

internal/doctor/doctor.go

@@ -385,6 +385,12 @@ func Evaluate(f Facts) []Finding {
 				add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s",
 					discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "")
 			}
+			if g.MIG == nil {
+				if p := g.PCIe.Problem(); p != "" {
+					add(Warn, "gpu", fmt.Sprintf("GPU %d (%s) %s", g.Index, g.Name, p),
+						"lspci -vv -s "+g.PCIBusID+" shows LnkCap (the card) against LnkSta (the link)")
+				}
+			}
 		}
 	}
 	evaluateMIG(f, add)