Ilker ManapandClaude Opus 5.5
82a63cfc6e
Every minute the agent reads nvidia-smi's full report, its NVLink error counters and the kernel's XID lines. They are published per card, shown on the node page and the GPU map, charted in Grafana, and five alerts watch them. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
86 lines
3.3 KiB
Go
86 lines
3.3 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
|
)
|
|
|
|
// mapFleet is a node of four cards: chat holds two and is idle, code holds
|
|
// one and is busy, one is free.
|
|
type mapFleet struct{ fakeSource }
|
|
|
|
func (mapFleet) Nodes() []adminapi.NodeView {
|
|
busy := func(i int, util, temp float64) inventory.GPUSensor {
|
|
return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300}
|
|
}
|
|
return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
|
|
Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{busy(0, 1, 35), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
|
|
}
|
|
|
|
func worn(s inventory.GPUSensor) inventory.GPUSensor {
|
|
s.Wear = &inventory.GPUWear{RemapPending: true, XIDs: 1, LastXID: 79, LastXIDText: "GPU has fallen off the bus."}
|
|
return s
|
|
}
|
|
|
|
func (mapFleet) Replicas(string) []adminapi.ReplicaView {
|
|
return []adminapi.ReplicaView{
|
|
{ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
|
|
{ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
|
|
}
|
|
}
|
|
|
|
func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
|
|
_, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
|
if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
|
|
t.Errorf("%d cards on the map, want 4", n)
|
|
}
|
|
for _, want := range []string{
|
|
"gpu07 GPU 2 · code (code-1) · 93% busy",
|
|
"gpu07 GPU 3 · free",
|
|
"memory rows remapped, waiting for a reset · XID 79 (GPU has fallen off the bus.)",
|
|
"border:2px solid #c0392b",
|
|
"border-style:dashed",
|
|
"Cards held and doing nothing",
|
|
"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("the page lacks %q", want)
|
|
}
|
|
}
|
|
if strings.Contains(body, "<code>code-1</code></td><td><code>gpu07</code>") {
|
|
t.Error("a busy replica was listed as idle")
|
|
}
|
|
_, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
|
|
if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
|
|
t.Error("the map coloured by temperature does not say so, or show it")
|
|
}
|
|
_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
|
if strings.Contains(cpu, "gpu-map") {
|
|
t.Error("a fleet without GPUs has a GPU map")
|
|
}
|
|
}
|
|
|
|
func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
|
|
h := &fakeHistory{byExpr: map[string][]promq.Series{}}
|
|
for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}} {
|
|
h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])}
|
|
}
|
|
got := loadEfficiency(context.Background(), h)
|
|
if len(got) != 2 || got[0].Model != "chat" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
c := got[0]
|
|
if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 {
|
|
t.Errorf("chat: %+v", c)
|
|
}
|
|
if got[1].PerGPU != 15 {
|
|
t.Errorf("code: %+v", got[1])
|
|
}
|
|
}
|