Ilker ManapandClaude Opus 5.5
c866b3a8ee
CI / test (push) Failing after 6m42s
The busy figure is nvidia-smi's utilization.gpu: the share of time any kernel ran, which a kernel on one SM fills. On cuda01 decode read 92% busy while the SMs had work 69% of the time and issued on 18% of their cycles. Busy is now said to be that, and beside it: each card's draw against its power limit (efficiency table, fleet and node charts, the console's GPU work chart), and where the profiling counters can be read, SMs active, SM occupancy, tensor cores and DRAM (llmgw_gpu_*_active_percent), from NVML's GPM on Hopper and later or a dcgm-exporter on the node (discovery.dcgm_exporter). The GPU map colours by work by default and keeps busy as a mode. A MIG instance is never published with its card's profile. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
42 lines
1.8 KiB
Go
42 lines
1.8 KiB
Go
package fleetmetrics
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func TestAMIGInstanceHasItsOwnMemoryItsCardsHeatAndNoUtilisation(t *testing.T) {
|
|
// Card 0 in MIG mode: the sensor sample is the card's, 60 GB used across
|
|
// both instances and a utilisation the driver does not give per instance.
|
|
inst := discovery.GPU{Index: 1001, Name: "NVIDIA A100-SXM4-80GB MIG 3g.40gb", MemoryTotalMB: 40192, MemoryUsedMB: 21000,
|
|
MIG: &discovery.MIGDevice{ParentIndex: 0, ParentMemoryTotalMB: 81920, Device: 1, Profile: "3g.40gb"}}
|
|
n := inventory.Node{ID: "g1", State: inventory.Ready, GPUs: []discovery.GPU{inst},
|
|
Load: &inventory.Load{At: time.Now(), GPUs: []inventory.GPUSensor{{Index: 0, TempC: 61, PowerW: 250,
|
|
UtilPercent: 0, MemoryUsedMB: 60000, MemoryTotalMB: 81920,
|
|
Profile: &inventory.GPUProfile{SMActive: 50, SMOccupancy: 30, TensorActive: 10, DRAMActive: 20}}}}}
|
|
out := gather(t, fakeSource{nodes: []inventory.Node{n}})
|
|
labels := "class=a100-80g-mig-3g.40gb gpu=1001 node=g1"
|
|
for _, want := range []string{
|
|
"llmgw_gpu_memory_used_bytes " + labels + " -> 22020096000\n",
|
|
"llmgw_gpu_memory_total_bytes " + labels + " -> 42144366592\n",
|
|
"llmgw_gpu_temperature_celsius " + labels + " -> 61\n",
|
|
"llmgw_gpu_power_watts " + labels + " -> 250\n",
|
|
} {
|
|
if !strings.Contains(out, want) {
|
|
t.Errorf("missing %s in:\n%s", want, out)
|
|
}
|
|
}
|
|
// Not the card's zero, not anything: absent.
|
|
if strings.Contains(out, "llmgw_gpu_utilization_percent "+labels) {
|
|
t.Errorf("a utilisation was published for a MIG instance:\n%s", out)
|
|
}
|
|
// The card's profile is the whole card's, not this instance's.
|
|
if strings.Contains(out, "llmgw_gpu_sm_active_percent "+labels) {
|
|
t.Errorf("the card's SM activity was published for a MIG instance:\n%s", out)
|
|
}
|
|
}
|