Outside services read answers redacted and whole; a card's own lanes
CI / test (push) Successful in 9m16s
CI / test (push) Successful in 9m16s
From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1948a787ae
commit
c09d5ad4ee
12 files changed
+220
-30
No files matched your search
@@ -83,6 +83,7 @@ func DiscoverGPUsNVML(_ context.Context) ([]GPU, error) {
|
||||
g.PCIBusID = busIDString(pci.BusId[:])
|
||||
byBusID[normalizeBusID(g.PCIBusID)] = i
|
||||
}
|
||||
g.PCIe = withSysfs(g.PCIe, "/sys/bus/pci/devices", g.PCIBusID)
|
||||
if t, ret := dev.GetTemperature(nvml.TEMPERATURE_GPU); ret == nvml.SUCCESS {
|
||||
g.Health.TemperatureC = int(t)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -84,8 +86,71 @@ func readPCIe(ctx context.Context, binary string, cards []GPU) {
|
||||
cards[i].PCIe = l
|
||||
}
|
||||
}
|
||||
return
|
||||
break
|
||||
}
|
||||
for i := range cards {
|
||||
if cards[i].MIG == nil {
|
||||
cards[i].PCIe = withSysfs(cards[i].PCIe, "/sys/bus/pci/devices", cards[i].PCIBusID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSysfs takes the card's own lane count, and its own fastest
|
||||
// generation, from the kernel: the driver's width.max is what card and slot
|
||||
// allow together, so a x16 card in a slot wired x8 reads x8 of x8 there,
|
||||
// while the device's max_link_width is the card's, 16.
|
||||
func withSysfs(l PCIeLink, root, busID string) PCIeLink {
|
||||
dev := sysfsBusID(busID)
|
||||
if dev == "" {
|
||||
return l
|
||||
}
|
||||
read := func(name string) string {
|
||||
b, err := os.ReadFile(filepath.Join(root, dev, name)) //nolint:gosec // under /sys
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
if n, err := strconv.Atoi(read("max_link_width")); err == nil && n > 0 {
|
||||
l.WidthMax = n
|
||||
}
|
||||
if n, err := strconv.Atoi(read("current_link_width")); err == nil && n > 0 {
|
||||
l.Width = n
|
||||
}
|
||||
if g := genOfSpeed(read("max_link_speed")); g > 0 && l.GPUGen == 0 {
|
||||
l.GPUGen = g
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// sysfsBusID is nvidia-smi's 00000000:3B:00.0 as the kernel names it,
|
||||
// 0000:3b:00.0.
|
||||
func sysfsBusID(id string) string {
|
||||
id = strings.ToLower(strings.TrimSpace(id))
|
||||
dom, rest, ok := strings.Cut(id, ":")
|
||||
if !ok || len(dom) < 4 {
|
||||
return ""
|
||||
}
|
||||
return dom[len(dom)-4:] + ":" + rest
|
||||
}
|
||||
|
||||
// genOfSpeed is the PCIe generation of a link speed as sysfs writes it.
|
||||
func genOfSpeed(s string) int {
|
||||
switch {
|
||||
case strings.HasPrefix(s, "2.5 GT/s"):
|
||||
return 1
|
||||
case strings.HasPrefix(s, "5.0 GT/s"), strings.HasPrefix(s, "5 GT/s"):
|
||||
return 2
|
||||
case strings.HasPrefix(s, "8.0 GT/s"), strings.HasPrefix(s, "8 GT/s"):
|
||||
return 3
|
||||
case strings.HasPrefix(s, "16.0 GT/s"), strings.HasPrefix(s, "16 GT/s"):
|
||||
return 4
|
||||
case strings.HasPrefix(s, "32.0 GT/s"), strings.HasPrefix(s, "32 GT/s"):
|
||||
return 5
|
||||
case strings.HasPrefix(s, "64.0 GT/s"), strings.HasPrefix(s, "64 GT/s"):
|
||||
return 6
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func parsePCIe(out string) map[int]PCIeLink {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
@@ -35,3 +37,18 @@ func TestTheLinkIsReadFromNvidiaSMI(t *testing.T) {
|
||||
t.Errorf("%+v", links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCardsOwnLanesComeFromTheKernel(t *testing.T) {
|
||||
// The driver says x8 of x8 for a x16 card in a slot wired x8; the
|
||||
// device's own max_link_width says 16.
|
||||
root := t.TempDir()
|
||||
dev := filepath.Join(root, "0000:3b:00.0")
|
||||
_ = os.MkdirAll(dev, 0o755)
|
||||
for f, v := range map[string]string{"max_link_width": "16", "current_link_width": "8", "max_link_speed": "16.0 GT/s PCIe"} {
|
||||
_ = os.WriteFile(filepath.Join(dev, f), []byte(v+"\n"), 0o644)
|
||||
}
|
||||
l := withSysfs(PCIeLink{Gen: 4, Width: 8, WidthMax: 8}, root, "00000000:3B:00.0")
|
||||
if l.WidthMax != 16 || l.Width != 8 || l.GPUGen != 4 || !strings.Contains(l.Problem(), "x8 of its x16") {
|
||||
t.Errorf("%+v: %q", l, l.Problem())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user