How much of each GPU works, beside whether anything does
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7bfe70c8df
commit
c866b3a8ee
34 files changed
+954
-52
No files matched your search
@@ -187,7 +187,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
logRing: ring,
|
||||
provKick: make(chan struct{}, 1),
|
||||
load: discovery.NewLoadSampler(),
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
sensors: discovery.NewSensorSampler().WithProfile(discovery.NewProfSampler(cfg.Discovery.DCGMExporter)),
|
||||
wear: discovery.NewHealthSampler(),
|
||||
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
|
||||
kv: kvevents.NewMirror(),
|
||||
@@ -798,6 +798,11 @@ func (a *Agent) sampleLoad() *llmgwv1.NodeLoad {
|
||||
MemoryUsedMb: g.MemoryUsedMB, MemoryTotalMb: g.MemoryTotalMB,
|
||||
SmClockMhz: g.SMClockMHz, EccUncorrected: g.ECCUncorrected,
|
||||
}
|
||||
if p := g.Profile; p != nil {
|
||||
gs.HasProfile, gs.ProfileSource = true, p.Source
|
||||
gs.SmActivePercent, gs.SmOccupancyPercent = p.SMActive, p.SMOccupancy
|
||||
gs.TensorActivePercent, gs.DramActivePercent = p.TensorActive, p.DRAMActive
|
||||
}
|
||||
if w, ok := wear[g.Index]; ok {
|
||||
gs.HasWear = true
|
||||
gs.PcieReplays, gs.RetiredPagesSbe, gs.RetiredPagesDbe, gs.RetirementPending =
|
||||
|
||||
@@ -28,6 +28,11 @@ type Options struct {
|
||||
// SkipGPUs omits GPU discovery. A node without nvidia-smi is already
|
||||
// found to have none, so this is for one whose driver is broken.
|
||||
SkipGPUs bool `koanf:"skip_gpus"`
|
||||
// DCGMExporter is the dcgm-exporter metrics URL read for how much of
|
||||
// each card's chip is in use, on cards NVML's GPM does not cover (before
|
||||
// Hopper). Empty is http://127.0.0.1:9400/metrics; "off" reads none. A
|
||||
// node without one costs a refused connection every five minutes.
|
||||
DCGMExporter string `koanf:"dcgm_exporter"`
|
||||
// ModelDirs are scanned for weights. Empty means do not look.
|
||||
ModelDirs []string `koanf:"-"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
//go:build !nvml
|
||||
|
||||
package discovery
|
||||
|
||||
// newGPM has nothing to read GPM with in a build without the nvml tag.
|
||||
func newGPM() gpmSource { return nil }
|
||||
@@ -0,0 +1,124 @@
|
||||
//go:build nvml
|
||||
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/NVIDIA/go-nvml/pkg/nvml"
|
||||
)
|
||||
|
||||
// nvmlGPM reads GPM, NVML's own profiling counters, on the cards that have
|
||||
// them: Hopper and later. Each read takes a sample and works out the metrics
|
||||
// against the one before, so NVML stays initialised between reads.
|
||||
type nvmlGPM struct {
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
failed bool
|
||||
cards map[int]*gpmCard
|
||||
}
|
||||
|
||||
type gpmCard struct {
|
||||
dev nvml.Device
|
||||
prev, cur nvml.GpmSample
|
||||
havePrevious bool
|
||||
}
|
||||
|
||||
func newGPM() gpmSource { return &nvmlGPM{} }
|
||||
|
||||
// The GPM metrics read, in the order they are asked for.
|
||||
var gpmMetrics = []struct {
|
||||
id nvml.GpmMetricId
|
||||
set func(*GPUProfile, float64)
|
||||
}{
|
||||
{nvml.GPM_METRIC_SM_UTIL, func(p *GPUProfile, v float64) { p.SMActive = v }},
|
||||
{nvml.GPM_METRIC_SM_OCCUPANCY, func(p *GPUProfile, v float64) { p.SMOccupancy = v }},
|
||||
{nvml.GPM_METRIC_ANY_TENSOR_UTIL, func(p *GPUProfile, v float64) { p.TensorActive = v }},
|
||||
{nvml.GPM_METRIC_DRAM_BW_UTIL, func(p *GPUProfile, v float64) { p.DRAMActive = v }},
|
||||
}
|
||||
|
||||
func (g *nvmlGPM) read() map[int]GPUProfile {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
if g.failed {
|
||||
return nil
|
||||
}
|
||||
if !g.opened && !g.open() {
|
||||
g.failed = true
|
||||
return nil
|
||||
}
|
||||
out := map[int]GPUProfile{}
|
||||
for idx, c := range g.cards {
|
||||
if c.cur.Get(c.dev) != nvml.SUCCESS {
|
||||
// A card in MIG mode, or one that has gone: no sample, and the
|
||||
// next must not be measured against an old one.
|
||||
c.havePrevious = false
|
||||
continue
|
||||
}
|
||||
if c.havePrevious {
|
||||
if p, ok := gpmBetween(c.prev, c.cur); ok {
|
||||
out[idx] = p
|
||||
}
|
||||
}
|
||||
c.prev, c.cur = c.cur, c.prev
|
||||
c.havePrevious = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// open finds the cards with GPM. False is NVML not starting, or no card
|
||||
// having it, and the reader then never tries again.
|
||||
func (g *nvmlGPM) open() bool {
|
||||
if nvml.Init() != nvml.SUCCESS {
|
||||
return false
|
||||
}
|
||||
g.cards = map[int]*gpmCard{}
|
||||
n, ret := nvml.DeviceGetCount()
|
||||
if ret != nvml.SUCCESS {
|
||||
n = 0
|
||||
}
|
||||
for i := range n {
|
||||
dev, ret := nvml.DeviceGetHandleByIndex(i)
|
||||
if ret != nvml.SUCCESS {
|
||||
continue
|
||||
}
|
||||
sup, ret := dev.GpmQueryDeviceSupport()
|
||||
if ret != nvml.SUCCESS || sup.IsSupportedDevice == 0 {
|
||||
continue
|
||||
}
|
||||
a, ret := nvml.GpmSampleAlloc()
|
||||
if ret != nvml.SUCCESS {
|
||||
continue
|
||||
}
|
||||
b, ret := nvml.GpmSampleAlloc()
|
||||
if ret != nvml.SUCCESS {
|
||||
_ = a.Free()
|
||||
continue
|
||||
}
|
||||
g.cards[i] = &gpmCard{dev: dev, prev: a, cur: b}
|
||||
}
|
||||
if len(g.cards) == 0 {
|
||||
_ = nvml.Shutdown()
|
||||
return false
|
||||
}
|
||||
g.opened = true
|
||||
return true
|
||||
}
|
||||
|
||||
// gpmBetween is the profile over the time between two samples.
|
||||
func gpmBetween(earlier, later nvml.GpmSample) (GPUProfile, bool) {
|
||||
q := nvml.GpmMetricsGetType{NumMetrics: uint32(len(gpmMetrics)), Sample1: earlier, Sample2: later}
|
||||
for i, m := range gpmMetrics {
|
||||
q.Metrics[i].MetricId = uint32(m.id)
|
||||
}
|
||||
if nvml.GpmMetricsGet(&q) != nvml.SUCCESS {
|
||||
return GPUProfile{}, false
|
||||
}
|
||||
p := GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "gpm"}
|
||||
for i, m := range gpmMetrics {
|
||||
if r := q.Metrics[i]; r.NvmlReturn == uint32(nvml.SUCCESS) && r.Value >= 0 {
|
||||
m.set(&p, r.Value)
|
||||
}
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GPUProfile is how much of a card's chip the work on it uses. nvidia-smi's
|
||||
// utilization.gpu cannot say: it is the share of time any kernel ran, and a
|
||||
// kernel keeping one SM of a hundred busy reads 100%. Measured on a Quadro
|
||||
// RTX 6000 under decode, it read 92% while the SMs had work 69% of the time
|
||||
// and issued an instruction on 18% of their cycles.
|
||||
//
|
||||
// These come from the card's profiling counters, which only two things read
|
||||
// without stopping the work: NVML's GPM, on Hopper and later, and DCGM, which
|
||||
// a site may run as dcgm-exporter. Percentages; -1 is not reported.
|
||||
type GPUProfile struct {
|
||||
// SMActive is the share of time the SMs had work, averaged over all of
|
||||
// them; SMOccupancy how full those SMs were with warps.
|
||||
SMActive, SMOccupancy float64
|
||||
// TensorActive is the tensor cores. Decode at batch one hardly uses them
|
||||
// even when nothing is wrong: a quantised matrix-vector product runs on
|
||||
// the integer units. Prefill and batches are where they show.
|
||||
TensorActive float64
|
||||
// DRAMActive is the memory interface, which decode is bound by.
|
||||
DRAMActive float64
|
||||
// Source is "gpm" or "dcgm".
|
||||
Source string
|
||||
}
|
||||
|
||||
// DefaultDCGMExporter is where dcgm-exporter listens unless told otherwise.
|
||||
const DefaultDCGMExporter = "http://127.0.0.1:9400/metrics"
|
||||
|
||||
// dcgmRetry is how long a dcgm-exporter that did not answer is left alone.
|
||||
// Most nodes run none, and asking every ten seconds would be asking nobody.
|
||||
const dcgmRetry = 5 * time.Minute
|
||||
|
||||
// gpmSource reads GPM, where this build and the card have it.
|
||||
type gpmSource interface {
|
||||
read() map[int]GPUProfile
|
||||
}
|
||||
|
||||
// ProfSampler reads each card's GPUProfile: from GPM where the card has it,
|
||||
// otherwise from a dcgm-exporter on the node.
|
||||
type ProfSampler struct {
|
||||
gpm gpmSource
|
||||
dcgm string
|
||||
get func(ctx context.Context, url string) ([]byte, error)
|
||||
now func() time.Time
|
||||
|
||||
dcgmAfter time.Time
|
||||
}
|
||||
|
||||
// NewProfSampler reads this machine. dcgmURL is the dcgm-exporter to ask:
|
||||
// empty is DefaultDCGMExporter, "off" asks none.
|
||||
func NewProfSampler(dcgmURL string) *ProfSampler {
|
||||
switch dcgmURL {
|
||||
case "":
|
||||
dcgmURL = DefaultDCGMExporter
|
||||
case "off":
|
||||
dcgmURL = ""
|
||||
}
|
||||
return &ProfSampler{gpm: newGPM(), dcgm: dcgmURL, get: httpGet, now: time.Now}
|
||||
}
|
||||
|
||||
// Sample is each card's profile by index; a card missing has none. GPM needs
|
||||
// two samples, so its cards appear from the second call.
|
||||
func (p *ProfSampler) Sample(ctx context.Context) map[int]GPUProfile {
|
||||
out := map[int]GPUProfile{}
|
||||
if p.gpm != nil {
|
||||
for i, v := range p.gpm.read() {
|
||||
out[i] = v
|
||||
}
|
||||
}
|
||||
if p.dcgm == "" || p.now().Before(p.dcgmAfter) {
|
||||
return out
|
||||
}
|
||||
body, err := p.get(ctx, p.dcgm)
|
||||
if err != nil {
|
||||
p.dcgmAfter = p.now().Add(dcgmRetry)
|
||||
return out
|
||||
}
|
||||
for i, v := range parseDCGM(body) {
|
||||
if _, ok := out[i]; !ok {
|
||||
out[i] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func httpGet(ctx context.Context, url string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
return io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
}
|
||||
|
||||
// The dcgm-exporter fields read, as ratios of one. Its default field list
|
||||
// has them on every card DCGM can profile.
|
||||
var dcgmFields = map[string]func(*GPUProfile, float64){
|
||||
"DCGM_FI_PROF_SM_ACTIVE": func(p *GPUProfile, v float64) { p.SMActive = v },
|
||||
"DCGM_FI_PROF_SM_OCCUPANCY": func(p *GPUProfile, v float64) { p.SMOccupancy = v },
|
||||
"DCGM_FI_PROF_PIPE_TENSOR_ACTIVE": func(p *GPUProfile, v float64) { p.TensorActive = v },
|
||||
"DCGM_FI_PROF_DRAM_ACTIVE": func(p *GPUProfile, v float64) { p.DRAMActive = v },
|
||||
}
|
||||
|
||||
// parseDCGM reads dcgm-exporter's text exposition: a line per field and
|
||||
// card, the card in the gpu label. A card with none of the fields — DCGM
|
||||
// cannot profile it — is left out. A MIG instance's lines carry a
|
||||
// GPU_I_ID and are left out too: these are whole cards.
|
||||
func parseDCGM(body []byte) map[int]GPUProfile {
|
||||
out := map[int]GPUProfile{}
|
||||
for _, line := range strings.Split(string(body), "\n") {
|
||||
if line == "" || line[0] == '#' {
|
||||
continue
|
||||
}
|
||||
open := strings.IndexByte(line, '{')
|
||||
end := strings.LastIndexByte(line, '}')
|
||||
if open < 0 || end < open {
|
||||
continue
|
||||
}
|
||||
set, ok := dcgmFields[line[:open]]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
labels := line[open+1 : end]
|
||||
if strings.Contains(labels, `GPU_I_ID="`) && !strings.Contains(labels, `GPU_I_ID=""`) {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(labelValue(labels, "gpu"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
val := strings.Fields(line[end+1:])
|
||||
if len(val) == 0 {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseFloat(val[0], 64)
|
||||
if err != nil || v < 0 {
|
||||
continue
|
||||
}
|
||||
p, seen := out[idx]
|
||||
if !seen {
|
||||
p = GPUProfile{SMActive: -1, SMOccupancy: -1, TensorActive: -1, DRAMActive: -1, Source: "dcgm"}
|
||||
}
|
||||
set(&p, 100*v)
|
||||
out[idx] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// labelValue is one label's value from a Prometheus label set.
|
||||
func labelValue(labels, name string) string {
|
||||
for _, kv := range strings.Split(labels, ",") {
|
||||
k, v, ok := strings.Cut(strings.TrimSpace(kv), "=")
|
||||
if ok && k == name {
|
||||
return strings.Trim(v, `"`)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package discovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dcgmBody is dcgm-exporter's output for two cards and a MIG instance:
|
||||
// card 0 profiled, card 1 a model DCGM cannot profile, which has the device
|
||||
// fields and none of the PROF ones.
|
||||
const dcgmBody = `# HELP DCGM_FI_DEV_GPU_UTIL GPU utilization (in %).
|
||||
# TYPE DCGM_FI_DEV_GPU_UTIL gauge
|
||||
DCGM_FI_DEV_GPU_UTIL{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 100
|
||||
DCGM_FI_DEV_GPU_UTIL{gpu="1",UUID="GPU-b",device="nvidia1",modelName="Quadro RTX 6000",Hostname="n1"} 92
|
||||
# HELP DCGM_FI_PROF_SM_ACTIVE The ratio of cycles an SM has at least 1 warp assigned.
|
||||
# TYPE DCGM_FI_PROF_SM_ACTIVE gauge
|
||||
DCGM_FI_PROF_SM_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.007042
|
||||
DCGM_FI_PROF_SM_ACTIVE{gpu="2",UUID="GPU-c",device="nvidia2",modelName="NVIDIA A100",GPU_I_PROFILE="3g.40gb",GPU_I_ID="1",Hostname="n1"} 0.5
|
||||
DCGM_FI_PROF_SM_OCCUPANCY{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0.25
|
||||
DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{gpu="0",UUID="GPU-a",device="nvidia0",modelName="NVIDIA L40S",Hostname="n1"} 0
|
||||
`
|
||||
|
||||
func TestDCGMExporterSaysHowMuchOfEachChipIsInUse(t *testing.T) {
|
||||
got := parseDCGM([]byte(dcgmBody))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want card 0 alone (card 1 unprofiled, card 2 a MIG instance): %+v", got)
|
||||
}
|
||||
p := got[0]
|
||||
// The LinkedIn L40S: 100% busy, one SM of 142 at work.
|
||||
if p.SMActive < 0.70 || p.SMActive > 0.71 || p.SMOccupancy != 25 || p.TensorActive != 0 || p.Source != "dcgm" {
|
||||
t.Errorf("card 0 %+v", p)
|
||||
}
|
||||
// A field the exporter does not send is unknown, not zero.
|
||||
if p.DRAMActive != -1 {
|
||||
t.Errorf("DRAM active not sent, read as %v", p.DRAMActive)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeGPM map[int]GPUProfile
|
||||
|
||||
func (f fakeGPM) read() map[int]GPUProfile { return f }
|
||||
|
||||
func TestGPMIsPreferredAndDCGMFillsTheCardsItDoesNotCover(t *testing.T) {
|
||||
calls := 0
|
||||
p := &ProfSampler{
|
||||
gpm: fakeGPM{1: {SMActive: 40, Source: "gpm"}},
|
||||
dcgm: DefaultDCGMExporter,
|
||||
get: func(context.Context, string) ([]byte, error) {
|
||||
calls++
|
||||
return []byte(`DCGM_FI_PROF_SM_ACTIVE{gpu="0"} 0.5` + "\n" + `DCGM_FI_PROF_SM_ACTIVE{gpu="1"} 0.9` + "\n"), nil
|
||||
},
|
||||
now: time.Now,
|
||||
}
|
||||
got := p.Sample(t.Context())
|
||||
if got[0].Source != "dcgm" || got[0].SMActive != 50 {
|
||||
t.Errorf("card 0 %+v", got[0])
|
||||
}
|
||||
if got[1].Source != "gpm" || got[1].SMActive != 40 {
|
||||
t.Errorf("GPM's card was taken from DCGM: %+v", got[1])
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("%d calls", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWithoutDCGMIsNotAskedEverySample(t *testing.T) {
|
||||
now := time.Now()
|
||||
calls := 0
|
||||
p := &ProfSampler{dcgm: DefaultDCGMExporter, now: func() time.Time { return now },
|
||||
get: func(context.Context, string) ([]byte, error) { calls++; return nil, errors.New("connection refused") }}
|
||||
for range 5 {
|
||||
if got := p.Sample(t.Context()); len(got) != 0 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
now = now.Add(10 * time.Second)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("asked %d times in 50 s", calls)
|
||||
}
|
||||
now = now.Add(dcgmRetry)
|
||||
p.Sample(t.Context())
|
||||
if calls != 2 {
|
||||
t.Errorf("not asked again after %v", dcgmRetry)
|
||||
}
|
||||
if off := NewProfSampler("off"); off.dcgm != "" {
|
||||
t.Errorf("off still asks %q", off.dcgm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachSampleCarriesTheProfileOfTheCardsThatHaveOne(t *testing.T) {
|
||||
smi := func(context.Context, string) ([]byte, error) {
|
||||
return []byte("0, 71, 239, 260, 0x0, 92, 30, 23000, 24576, 1878, 0, 0\n" +
|
||||
"1, 30, 20, 260, 0x0, 0, 0, 512, 24576, 300, 0, 0\n"), nil
|
||||
}
|
||||
now := time.Now()
|
||||
s := sampler(t.TempDir(), smi, &now).WithProfile(&ProfSampler{
|
||||
gpm: fakeGPM{0: {SMActive: 69, SMOccupancy: 45, TensorActive: 1, DRAMActive: 33, Source: "gpm"}}, now: time.Now})
|
||||
got := s.Sample(t.Context()).GPUs
|
||||
if got[0].Profile == nil || got[0].Profile.SMActive != 69 || got[0].UtilPercent != 92 {
|
||||
t.Errorf("card 0 %+v", got[0])
|
||||
}
|
||||
if got[1].Profile != nil {
|
||||
t.Errorf("card 1 has a profile nobody read: %+v", got[1].Profile)
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,9 @@ type GPUSensor struct {
|
||||
MemoryUsedMB, MemoryTotalMB float64
|
||||
SMClockMHz float64
|
||||
ECCUncorrected float64
|
||||
// Profile is how much of the chip the work uses, where something on the
|
||||
// node can say: nil on most cards. UtilPercent is not that.
|
||||
Profile *GPUProfile
|
||||
}
|
||||
|
||||
// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
|
||||
@@ -61,6 +64,8 @@ type SensorSampler struct {
|
||||
sys string // /sys, or a test's copy of it
|
||||
smi func(ctx context.Context, fields string) ([]byte, error)
|
||||
now func() time.Time
|
||||
// prof reads the cards' profiling counters; nil reads none.
|
||||
prof *ProfSampler
|
||||
|
||||
mu sync.Mutex
|
||||
last Sensors
|
||||
@@ -74,6 +79,12 @@ func NewSensorSampler() *SensorSampler {
|
||||
return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
|
||||
}
|
||||
|
||||
// WithProfile has each sample carry the cards' profiles as well.
|
||||
func (s *SensorSampler) WithProfile(p *ProfSampler) *SensorSampler {
|
||||
s.prof = p
|
||||
return s
|
||||
}
|
||||
|
||||
// SampleGPUs reads every card's heat, power and how busy it is now, without
|
||||
// a sampler's ten seconds of memory: for a screen that asks every few seconds
|
||||
// and wants what it asked for. Nil on a machine without nvidia-smi or a card.
|
||||
@@ -101,6 +112,14 @@ func (s *SensorSampler) Sample(ctx context.Context) Sensors {
|
||||
out.CPUTempC, out.InletTempC = s.hwmon()
|
||||
out.CPUPowerW = s.rapl(now)
|
||||
out.GPUs = s.gpus(ctx)
|
||||
if s.prof != nil {
|
||||
profs := s.prof.Sample(ctx)
|
||||
for i := range out.GPUs {
|
||||
if p, ok := profs[out.GPUs[i].Index]; ok {
|
||||
out.GPUs[i].Profile = &p
|
||||
}
|
||||
}
|
||||
}
|
||||
s.last, s.lastAt = out, now
|
||||
return out
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user