diff --git a/CHANGELOG.md b/CHANGELOG.md index cfa7590..0bed9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,11 @@ APIs change between minors without a deprecation period. ### Added +- **A card in the wrong PCIe slot is said so.** Each agent reads how every + card is linked — the fastest generation card and slot allow together, and + the lanes in use — against the card's own most; a card on fewer lanes or + an older generation is a node warning, with what it carries and what it + could, on the dashboard, the console and in `llmgw doctor`. - **And on a web page, `/admin/mcp`.** For administrators: the same list, adding a server with its credential sealed, testing it and listing its tools, granting them in the tree beside that level's other grants and diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 32acd6a..fc3419a 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -1589,7 +1589,14 @@ type GPU struct { // is then 1000 + 100 × card + device, uuid the instance's (MIG-…), and // memory the instance's share; utilization_percent stays zero, because no // driver measures it per instance. - Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"` + Mig *MIGDevice `protobuf:"bytes,22,opt,name=mig,proto3" json:"mig,omitempty"` + // How the card is linked to its host: the fastest generation card and + // slot allow together, the card's own fastest, and the lanes in use and + // the card's most. Zero when the driver did not say. + PcieGen int32 `protobuf:"varint,23,opt,name=pcie_gen,json=pcieGen,proto3" json:"pcie_gen,omitempty"` + PcieGpuGen int32 `protobuf:"varint,24,opt,name=pcie_gpu_gen,json=pcieGpuGen,proto3" json:"pcie_gpu_gen,omitempty"` + PcieWidth int32 `protobuf:"varint,25,opt,name=pcie_width,json=pcieWidth,proto3" json:"pcie_width,omitempty"` + PcieWidthMax int32 `protobuf:"varint,26,opt,name=pcie_width_max,json=pcieWidthMax,proto3" json:"pcie_width_max,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1778,6 +1785,34 @@ func (x *GPU) GetMig() *MIGDevice { return nil } +func (x *GPU) GetPcieGen() int32 { + if x != nil { + return x.PcieGen + } + return 0 +} + +func (x *GPU) GetPcieGpuGen() int32 { + if x != nil { + return x.PcieGpuGen + } + return 0 +} + +func (x *GPU) GetPcieWidth() int32 { + if x != nil { + return x.PcieWidth + } + return 0 +} + +func (x *GPU) GetPcieWidthMax() int32 { + if x != nil { + return x.PcieWidthMax + } + return 0 +} + // MIGDevice is which card a MIG instance is carved from, and how. type MIGDevice struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -5251,7 +5286,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "leaderPort\x12\x1d\n" + "\n" + "profile_fp\x18\x0f \x01(\tR\tprofileFp\x12+\n" + - "\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xca\x06\n" + + "\acompute\x18\x10 \x01(\v2\x11.llmgw.v1.ComputeR\acompute\"\xcc\a\n" + "\x03GPU\x12\x14\n" + "\x05index\x18\x01 \x01(\x05R\x05index\x12\x12\n" + "\x04uuid\x18\x02 \x01(\tR\x04uuid\x12\x12\n" + @@ -5277,7 +5312,13 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\fcluster_uuid\x18\x13 \x01(\tR\vclusterUuid\x12\x1b\n" + "\tclique_id\x18\x14 \x01(\tR\bcliqueId\x12)\n" + "\x10fabric_bandwidth\x18\x15 \x01(\tR\x0ffabricBandwidth\x12%\n" + - "\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x1a;\n" + + "\x03mig\x18\x16 \x01(\v2\x13.llmgw.v1.MIGDeviceR\x03mig\x12\x19\n" + + "\bpcie_gen\x18\x17 \x01(\x05R\apcieGen\x12 \n" + + "\fpcie_gpu_gen\x18\x18 \x01(\x05R\n" + + "pcieGpuGen\x12\x1d\n" + + "\n" + + "pcie_width\x18\x19 \x01(\x05R\tpcieWidth\x12$\n" + + "\x0epcie_width_max\x18\x1a \x01(\x05R\fpcieWidthMax\x1a;\n" + "\rNicPathsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x84\x02\n" + diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index c0c5814..2d89258 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -337,6 +337,13 @@ message GPU { // memory the instance's share; utilization_percent stays zero, because no // driver measures it per instance. MIGDevice mig = 22; + // How the card is linked to its host: the fastest generation card and + // slot allow together, the card's own fastest, and the lanes in use and + // the card's most. Zero when the driver did not say. + int32 pcie_gen = 23; + int32 pcie_gpu_gen = 24; + int32 pcie_width = 25; + int32 pcie_width_max = 26; } // MIGDevice is which card a MIG instance is carved from, and how. diff --git a/docs/install.md b/docs/install.md index 55c2816..385c698 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1053,6 +1053,27 @@ A node with no location is placed as before, as a rack of its own. See [facility-aware-placement.md](facility-aware-placement.md) for power caps and holds. +### A card in the wrong slot + +An x16 card in a slot wired x8 or x4, behind a riser, or in a slot of an +older PCIe generation than the card's, links slower than it was built to and +works otherwise. A model on the card still decodes at the card's own memory +speed, but it loads slower, a replica split over several cards by PCIe is +slower, and so is anything that spills to the host. Each card's agent reads +how its link stands — the fastest generation card and slot allow together, +and the lanes in use — against what the card could do, and the node says so +among its warnings (the dashboard, the console's Nodes, `llmgw doctor`): + +``` +GPU 1 (NVIDIA H100 80GB HBM3) linked at x8 of its x16 lanes: 31.5 GB/s where it +could have 63.0. Check the slot (wired fewer lanes, or an older generation), a +riser, or the BIOS's PCIe setting +``` + +The generation a card reports now drops while it idles, to save power; what is +compared is the fastest it may reach, which does not. `lspci -vv -s BUS-ID` +shows the same: `LnkCap` is the card's, `LnkSta` the link's. + ## 6. Setting the node's CPU model, if it is a VM Proxmox defaults a VM to `kvm64`, which hides AVX, AVX2, FMA and even SSE4.2 diff --git a/internal/agent/discovery/gpu.go b/internal/agent/discovery/gpu.go index 6495d38..5f25e52 100644 --- a/internal/agent/discovery/gpu.go +++ b/internal/agent/discovery/gpu.go @@ -52,6 +52,8 @@ type GPU struct { // memory the instance's share. Utilisation stays unreported, because no // driver measures it per instance. MIG *MIGDevice `json:"mig,omitempty"` + // PCIe is how the card is linked to its host, zero when not read. + PCIe PCIeLink `json:"pcie,omitempty"` } // GPUHealth records conditions that make a GPU unsafe to schedule onto. diff --git a/internal/agent/discovery/gpu_nvml.go b/internal/agent/discovery/gpu_nvml.go index aa52220..65529e2 100644 --- a/internal/agent/discovery/gpu_nvml.go +++ b/internal/agent/discovery/gpu_nvml.go @@ -67,6 +67,18 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) { if major, minor, ret := dev.GetCudaComputeCapability(); ret == nvml.SUCCESS { g.ComputeCapability = fmt.Sprintf("%d.%d", major, minor) } + if v, ret := dev.GetMaxPcieLinkGeneration(); ret == nvml.SUCCESS { + g.PCIe.Gen = v + } + if v, ret := dev.GetGpuMaxPcieLinkGeneration(); ret == nvml.SUCCESS { + g.PCIe.GPUGen = v + } + if v, ret := dev.GetCurrPcieLinkWidth(); ret == nvml.SUCCESS { + g.PCIe.Width = v + } + if v, ret := dev.GetMaxPcieLinkWidth(); ret == nvml.SUCCESS { + g.PCIe.WidthMax = v + } if pci, ret := dev.GetPciInfo(); ret == nvml.SUCCESS { g.PCIBusID = busIDString(pci.BusId[:]) byBusID[normalizeBusID(g.PCIBusID)] = i diff --git a/internal/agent/discovery/gpu_smi.go b/internal/agent/discovery/gpu_smi.go index 44c9490..b8b2b9b 100644 --- a/internal/agent/discovery/gpu_smi.go +++ b/internal/agent/discovery/gpu_smi.go @@ -64,6 +64,7 @@ func DiscoverGPUsSMI(ctx context.Context, binary string) ([]GPU, error) { if err != nil || len(cards) == 0 { return cards, err } + readPCIe(ctx, binary, cards) // The query lists whole cards, MIG mode or not. A card in MIG mode is // replaced by its instances, which only -L names and only the full // report sizes; asked only when some card could be in MIG mode, since diff --git a/internal/agent/discovery/pcie.go b/internal/agent/discovery/pcie.go new file mode 100644 index 0000000..721e067 --- /dev/null +++ b/internal/agent/discovery/pcie.go @@ -0,0 +1,116 @@ +package discovery + +import ( + "context" + "encoding/csv" + "fmt" + "os/exec" + "strconv" + "strings" +) + +// A card in the wrong slot: an x16 card in a slot wired x8 or x4, behind a +// riser, or in a slot of an older PCIe generation than the card's, links +// slower than it was built to, and nothing says so. A model on the card +// still decodes at the card's own memory speed, but it loads slower, a +// tensor-parallel replica over PCIe is slower, and so is anything that +// spills to the host. The driver says how the link stands and what the +// card could do; the difference is reported where the node's warnings are. + +// PCIeLink is how a card is linked to its host. +type PCIeLink struct { + // Gen is the fastest generation the card and the slot together allow — + // not the current one, which drops while the card idles, to save power. + // GPUGen is the card's own fastest, 0 when the driver does not say. + Gen int `json:"gen,omitempty"` + GPUGen int `json:"gpu_gen,omitempty"` + // Width is the lanes in use now, WidthMax the card's most. + Width int `json:"width,omitempty"` + WidthMax int `json:"width_max,omitempty"` +} + +// laneGBs is one lane's usable rate in GB/s, by generation. +var laneGBs = map[int]float64{1: 0.25, 2: 0.5, 3: 0.985, 4: 1.969, 5: 3.938, 6: 7.877} + +// GBs is what a link of gen and width carries, in GB/s each way. +func pcieGBs(gen, width int) float64 { return laneGBs[gen] * float64(width) } + +// Problem says how a card's link falls short of what the card could do, +// empty when it does not or the driver did not say. +func (p PCIeLink) Problem() string { + gen := p.Gen + if p.GPUGen > 0 && gen == 0 { + gen = p.GPUGen + } + slowGen := p.GPUGen > 0 && p.Gen > 0 && p.Gen < p.GPUGen + narrow := p.WidthMax > 0 && p.Width > 0 && p.Width < p.WidthMax + if !slowGen && !narrow { + return "" + } + could := pcieGBs(max(p.GPUGen, gen), p.WidthMax) + has := pcieGBs(gen, p.Width) + var why []string + if narrow { + why = append(why, fmt.Sprintf("x%d of its x%d lanes", p.Width, p.WidthMax)) + } + if slowGen { + why = append(why, fmt.Sprintf("PCIe Gen%d of its Gen%d", p.Gen, p.GPUGen)) + } + return fmt.Sprintf("linked at %s: %.1f GB/s where it could have %.1f. "+ + "Check the slot (wired fewer lanes, or an older generation), a riser, or the BIOS's PCIe setting", + strings.Join(why, " and "), has, could) +} + +// pcieFields are asked apart from the rest, and the card's own fastest +// generation apart again: a driver that does not know a field refuses the +// whole query, and these must not cost a node its cards. +var pcieFields = [][]string{ + {"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max", "pcie.link.gen.gpumax"}, + {"index", "pcie.link.gen.max", "pcie.link.width.current", "pcie.link.width.max"}, +} + +// readPCIe fills each card's link from nvidia-smi, leaving it empty when +// the driver says nothing of it. +func readPCIe(ctx context.Context, binary string, cards []GPU) { + for _, fields := range pcieFields { + out, err := exec.CommandContext(ctx, binary, "--query-gpu="+strings.Join(fields, ","), + "--format=csv,noheader,nounits").Output() + if err != nil { + continue + } + links := parsePCIe(string(out)) + for i := range cards { + if l, ok := links[cards[i].Index]; ok && cards[i].MIG == nil { + cards[i].PCIe = l + } + } + return + } +} + +func parsePCIe(out string) map[int]PCIeLink { + r := csv.NewReader(strings.NewReader(out)) + r.TrimLeadingSpace = true + r.FieldsPerRecord = -1 + rows, _ := r.ReadAll() + num := func(s string) int { + n, _ := strconv.Atoi(strings.TrimSpace(s)) + return n + } + links := map[int]PCIeLink{} + for _, row := range rows { + if len(row) < 4 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(row[0])) + if err != nil { + continue + } + l := PCIeLink{Gen: num(row[1]), Width: num(row[2]), WidthMax: num(row[3])} + if len(row) > 4 { + l.GPUGen = num(row[4]) + } + links[idx] = l + } + return links +} diff --git a/internal/agent/discovery/pcie_test.go b/internal/agent/discovery/pcie_test.go new file mode 100644 index 0000000..6504ddf --- /dev/null +++ b/internal/agent/discovery/pcie_test.go @@ -0,0 +1,37 @@ +package discovery + +import ( + "strings" + "testing" +) + +func TestACardInTheWrongSlotIsSaidSo(t *testing.T) { + for _, c := range []struct { + link PCIeLink + want string // empty: nothing to say + }{ + // The PoC's Quadro RTX 6000, idle: current Gen1, but its fastest is 3, + // as the card's is, on all 16 lanes. Nothing wrong. + {PCIeLink{Gen: 3, GPUGen: 3, Width: 16, WidthMax: 16}, ""}, + // An x16 card in a slot wired x8. + {PCIeLink{Gen: 4, GPUGen: 4, Width: 8, WidthMax: 16}, "x8 of its x16 lanes: 15.8 GB/s where it could have 31.5"}, + // A Gen5 card in a Gen4 slot. + {PCIeLink{Gen: 4, GPUGen: 5, Width: 16, WidthMax: 16}, "PCIe Gen4 of its Gen5: 31.5 GB/s where it could have 63.0"}, + // Both, behind a riser. + {PCIeLink{Gen: 3, GPUGen: 4, Width: 4, WidthMax: 16}, "x4 of its x16 lanes and PCIe Gen3 of its Gen4"}, + // A driver that says nothing. + {PCIeLink{}, ""}, + } { + got := c.link.Problem() + if (c.want == "") != (got == "") || !strings.Contains(got, c.want) { + t.Errorf("%+v: %q, want %q", c.link, got, c.want) + } + } +} + +func TestTheLinkIsReadFromNvidiaSMI(t *testing.T) { + links := parsePCIe("0, 3, 16, 16, 3\n1, 4, 8, 16, [N/A]\n") + if links[0] != (PCIeLink{Gen: 3, GPUGen: 3, Width: 16, WidthMax: 16}) || links[1] != (PCIeLink{Gen: 4, Width: 8, WidthMax: 16}) { + t.Errorf("%+v", links) + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 4aeb1fe..ad491cf 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -707,6 +707,14 @@ func (a *API) Nodes() []NodeView { } } v.MemBandwidthGBs = n.Host.MemBandwidthGBs + for _, g := range n.GPUs { + if g.MIG != nil { + continue // said of the card it is carved from + } + if p := g.PCIe.Problem(); p != "" { + v.Warnings = append(v.Warnings, fmt.Sprintf("GPU %d (%s) %s", g.Index, g.Name, p)) + } + } if h := n.Host; h.OSID != "" { v.OS = strings.TrimSpace(h.OSID + " " + h.OSVersion + " " + h.Arch) if h.Arch == "amd64" && !h.HasCPUFlag("avx2") { diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index 839c60d..6931785 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -735,3 +735,17 @@ func TestRemovingANodeRevokesItsCertificate(t *testing.T) { t.Error("the node was removed though its certificate was not revoked") } } + +func TestANodesCardInTheWrongSlotIsAWarning(t *testing.T) { + n := gpuNode("g1", 2) + n.GPUs[0].PCIe = discovery.PCIeLink{Gen: 5, GPUGen: 5, Width: 16, WidthMax: 16} + n.GPUs[1].PCIe = discovery.PCIeLink{Gen: 5, GPUGen: 5, Width: 8, WidthMax: 16} + api, _ := setup(t, testModels(), n) + var warned []string + for _, v := range api.Nodes() { + warned = append(warned, v.Warnings...) + } + if len(warned) != 1 || !strings.Contains(warned[0], "GPU 1 (NVIDIA H100 80GB HBM3) linked at x8 of its x16 lanes") { + t.Errorf("warnings %q", warned) + } +} diff --git a/internal/control/agentsvc/convert.go b/internal/control/agentsvc/convert.go index 6639f54..e07617a 100644 --- a/internal/control/agentsvc/convert.go +++ b/internal/control/agentsvc/convert.go @@ -73,6 +73,8 @@ func NodeFromRegister(r *llmgwv1.Register) inventory.Node { Problems: g.GetProblems(), }, MIG: migFrom(g.GetMig()), + PCIe: discovery.PCIeLink{Gen: int(g.GetPcieGen()), GPUGen: int(g.GetPcieGpuGen()), + Width: int(g.GetPcieWidth()), WidthMax: int(g.GetPcieWidthMax())}, }) } n.IMEX = discovery.IMEX{State: r.GetImex().GetState(), Channels: int(r.GetImex().GetChannels())} @@ -127,6 +129,10 @@ func RegisterFromInventory(nodeID, hostname, mgmtAddr, agentVersion string, Problems: g.Health.Problems, TemperatureC: int32(g.Health.TemperatureC), EccErrors: int32(g.Health.ECCErrors), + PcieGen: int32(g.PCIe.Gen), + PcieGpuGen: int32(g.PCIe.GPUGen), + PcieWidth: int32(g.PCIe.Width), + PcieWidthMax: int32(g.PCIe.WidthMax), }) if f := g.Fabric; f != nil { last := r.Gpus[len(r.Gpus)-1] diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index e4e1616..53c9592 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -385,6 +385,12 @@ func Evaluate(f Facts) []Finding { add(Warn, "gpu", fmt.Sprintf("GPU %s is kept out of placement: %s", discovery.DeviceLabel(g.Index), strings.Join(g.Health.Problems, "; ")), "") } + if g.MIG == nil { + if p := g.PCIe.Problem(); p != "" { + add(Warn, "gpu", fmt.Sprintf("GPU %d (%s) %s", g.Index, g.Name, p), + "lspci -vv -s "+g.PCIBusID+" shows LnkCap (the card) against LnkSta (the link)") + } + } } } evaluateMIG(f, add)