A card in the wrong PCIe slot is said so
CI / test (push) Successful in 9m17s

At the user's asking: we know what a card's link could be, so a card
linked slower is reported. Each agent reads, apart from the GPU query so
that a driver not knowing a field costs nothing else, the fastest PCIe
generation card and slot allow together, the card's own, and the lanes in
use and the card's most (nvidia-smi; NVML in the nvml build). The current
generation is not compared: it drops while a card idles. A card on fewer
lanes, or a slot of an older generation, is a node warning with the GB/s
it carries against what it could — on the dashboard, the console and in
llmgw doctor, which points at lspci's LnkCap and LnkSta. MIG instances are
said of their card. GPU proto fields 23-26.

The PoC's Quadro RTX 6000 reads Gen3 of Gen3, x16 of x16, idle at Gen1:
nothing to say.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 08:49:16 +01:00
1 parent 6a870a8c15
commit 721705b29d
13 files changed
+279 -3

No files matched your search

+44 -3
View File
@@ -1589,7 +1589,14 @@ type GPU struct {
// is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and
// memory the instance's share; utilization_percent stays zero, because no
// driver measures it per instance.
Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
// How the card is linked to its host: the fastest generation card and
// slot allow together, the card's own fastest, and the lanes in use and
// the card's most. Zero when the driver did not say.
PcieGen int32 `protobuf:"varint,23,opt,name=pcie_gen,json=pcieGen,proto3" json:"pcie_gen,omitempty"`
PcieGpuGen int32 `protobuf:"varint,24,opt,name=pcie_gpu_gen,json=pcieGpuGen,proto3" json:"pcie_gpu_gen,omitempty"`
PcieWidth int32 `protobuf:"varint,25,opt,name=pcie_width,json=pcieWidth,proto3" json:"pcie_width,omitempty"`
PcieWidthMax int32 `protobuf:"varint,26,opt,name=pcie_width_max,json=pcieWidthMax,proto3" json:"pcie_width_max,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -1778,6 +1785,34 @@ func (x *GPU) GetMig() *MIGDevice {
return nil
}
func (x *GPU) GetPcieGen() int32 {
if x != nil {
return x.PcieGen
}
return 0
}
func (x *GPU) GetPcieGpuGen() int32 {
if x != nil {
return x.PcieGpuGen
}
return 0
}
func (x *GPU) GetPcieWidth() int32 {
if x != nil {
return x.PcieWidth
}
return 0
}
func (x *GPU) GetPcieWidthMax() int32 {
if x != nil {
return x.PcieWidthMax
}
return 0
}
// MIGDevice is which card a MIG instance is carved from, and how.
type MIGDevice struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -5251,7 +5286,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"leaderPort\x12\x1d\n" +
"\n" +
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12+\n" +
"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xca\x06\n" +
"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xcc\a\n" +
"\x03GPU\x12\x14\n" +
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" +
"\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" +
@@ -5277,7 +5312,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" +
"\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" +
"\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x12%\n" +
"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x1a;\n" +
"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x12\x19\n" +
"\bpcie_gen\x18\x17 \x01(\x05R\apcieGen\x12 \n" +
"\fpcie_gpu_gen\x18\x18 \x01(\x05R\n" +
"pcieGpuGen\x12\x1d\n" +
"\n" +
"pcie_width\x18\x19 \x01(\x05R\tpcieWidth\x12$\n" +
"\x0epcie_width_max\x18\x1a \x01(\x05R\fpcieWidthMax\x1a;\n" +
"\rNicPathsEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x84\x02\n" +
+7
View File
@@ -337,6 +337,13 @@ message GPU {
// memory the instance's share; utilization_percent stays zero, because no
// driver measures it per instance.
MIGDevice mig = 22;
// How the card is linked to its host: the fastest generation card and
// slot allow together, the card's own fastest, and the lanes in use and
// the card's most. Zero when the driver did not say.
int32 pcie_gen = 23;
int32 pcie_gpu_gen = 24;
int32 pcie_width = 25;
int32 pcie_width_max = 26;
}
// MIGDevice is which card a MIG instance is carved from, and how.