At the user's asking: we know what a card's link could be, so a card linked slower is reported. Each agent reads, apart from the GPU query so that a driver not knowing a field costs nothing else, the fastest PCIe generation card and slot allow together, the card's own, and the lanes in use and the card's most (nvidia-smi; NVML in the nvml build). The current generation is not compared: it drops while a card idles. A card on fewer lanes, or a slot of an older generation, is a node warning with the GB/s it carries against what it could — on the dashboard, the console and in llmgw doctor, which points at lspci's LnkCap and LnkSta. MIG instances are said of their card. GPU proto fields 23-26. The PoC's Quadro RTX 6000 reads Gen3 of Gen3, x16 of x16, idle at Gen1: nothing to say. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6a870a8c15
commit
721705b29d
13 files changed
+279
-3
No files matched your search
@@ -1589,7 +1589,14 @@ type GPU struct {
|
||||
// is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and
|
||||
// memory the instance's share; utilization_percent stays zero, because no
|
||||
// driver measures it per instance.
|
||||
Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
|
||||
Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"`
|
||||
// How the card is linked to its host: the fastest generation card and
|
||||
// slot allow together, the card's own fastest, and the lanes in use and
|
||||
// the card's most. Zero when the driver did not say.
|
||||
PcieGen int32 `protobuf:"varint,23,opt,name=pcie_gen,json=pcieGen,proto3" json:"pcie_gen,omitempty"`
|
||||
PcieGpuGen int32 `protobuf:"varint,24,opt,name=pcie_gpu_gen,json=pcieGpuGen,proto3" json:"pcie_gpu_gen,omitempty"`
|
||||
PcieWidth int32 `protobuf:"varint,25,opt,name=pcie_width,json=pcieWidth,proto3" json:"pcie_width,omitempty"`
|
||||
PcieWidthMax int32 `protobuf:"varint,26,opt,name=pcie_width_max,json=pcieWidthMax,proto3" json:"pcie_width_max,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -1778,6 +1785,34 @@ func (x *GPU) GetMig() *MIGDevice {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *GPU) GetPcieGen() int32 {
|
||||
if x != nil {
|
||||
return x.PcieGen
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPU) GetPcieGpuGen() int32 {
|
||||
if x != nil {
|
||||
return x.PcieGpuGen
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPU) GetPcieWidth() int32 {
|
||||
if x != nil {
|
||||
return x.PcieWidth
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *GPU) GetPcieWidthMax() int32 {
|
||||
if x != nil {
|
||||
return x.PcieWidthMax
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// MIGDevice is which card a MIG instance is carved from, and how.
|
||||
type MIGDevice struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -5251,7 +5286,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"leaderPort\x12\x1d\n" +
|
||||
"\n" +
|
||||
"profile_fp\x18\x0f \x01(\tR\tprofileFp\x12+\n" +
|
||||
"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xca\x06\n" +
|
||||
"\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xcc\a\n" +
|
||||
"\x03GPU\x12\x14\n" +
|
||||
"\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" +
|
||||
"\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" +
|
||||
@@ -5277,7 +5312,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" +
|
||||
"\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" +
|
||||
"\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x12%\n" +
|
||||
"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x1a;\n" +
|
||||
"\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x12\x19\n" +
|
||||
"\bpcie_gen\x18\x17 \x01(\x05R\apcieGen\x12 \n" +
|
||||
"\fpcie_gpu_gen\x18\x18 \x01(\x05R\n" +
|
||||
"pcieGpuGen\x12\x1d\n" +
|
||||
"\n" +
|
||||
"pcie_width\x18\x19 \x01(\x05R\tpcieWidth\x12$\n" +
|
||||
"\x0epcie_width_max\x18\x1a \x01(\x05R\fpcieWidthMax\x1a;\n" +
|
||||
"\rNicPathsEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x84\x02\n" +
|
||||
|
||||
@@ -337,6 +337,13 @@ message GPU {
|
||||
// memory the instance's share; utilization_percent stays zero, because no
|
||||
// driver measures it per instance.
|
||||
MIGDevice mig = 22;
|
||||
// How the card is linked to its host: the fastest generation card and
|
||||
// slot allow together, the card's own fastest, and the lanes in use and
|
||||
// the card's most. Zero when the driver did not say.
|
||||
int32 pcie_gen = 23;
|
||||
int32 pcie_gpu_gen = 24;
|
||||
int32 pcie_width = 25;
|
||||
int32 pcie_width_max = 26;
|
||||
}
|
||||
|
||||
// MIGDevice is which card a MIG instance is carved from, and how.
|
||||
|
||||
Reference in new issue
Block a user