llmgw/internal/web/gpumap_test.go

package web

import (
	"context"
	"strings"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)

// mapFleet is a node of four cards: chat holds two and is idle, code holds
// one and is busy, one is free.
type mapFleet struct{ fakeSource }

func (mapFleet) Nodes() []adminapi.NodeView {
	busy := func(i int, util, temp float64) inventory.GPUSensor {
		return inventory.GPUSensor{Index: i, UtilPercent: util, TempC: temp, MemoryUsedMB: 40000, MemoryTotalMB: 81559, PowerW: 300, PowerLimitW: 700}
	}
	profiled := func(s inventory.GPUSensor) inventory.GPUSensor {
		s.Profile = &inventory.GPUProfile{SMActive: 0, SMOccupancy: 0, TensorActive: 0, DRAMActive: -1, Source: "dcgm"}
		return s
	}
	return []adminapi.NodeView{{ID: "gpu07", State: "ready", GPUTotal: 4, GPUFree: 1, Classes: []string{"h100-80g"},
		Load: &adminapi.LoadView{GPUs: []inventory.GPUSensor{profiled(busy(0, 1, 35)), busy(1, 0, 34), busy(2, 93, 81), worn(busy(3, 0, 30))}}}}
}

func worn(s inventory.GPUSensor) inventory.GPUSensor {
	s.Wear = &inventory.GPUWear{RemapPending: true, XIDs: 1, LastXID: 79, LastXIDText: "GPU has fallen off the bus."}
	return s
}

func (mapFleet) Replicas(string) []adminapi.ReplicaView {
	return []adminapi.ReplicaView{
		{ID: "chat-1", Model: "chat", State: "ready", IdleSec: 1800, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{0, 1}}}},
		{ID: "code-1", Model: "code", State: "ready", IdleSec: 2, Shards: []adminapi.ShardView{{Node: "gpu07", GPUs: []int{2}}}},
	}
}

func TestTheClusterPageMapsEveryCardAndNamesTheIdleOnes(t *testing.T) {
	_, body := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
	if n := strings.Count(body, `class="gpu-cell"`); n != 4 {
		t.Errorf("%d cards on the map, want 4", n)
	}
	for _, want := range []string{
		"gpu07 GPU 2 · code (code-1) · 93% busy",
		"gpu07 GPU 3 · free",
		"memory rows remapped, waiting for a reset · XID 79 (GPU has fallen off the bus.)",
		"border:2px solid #c0392b",
		"border-style:dashed",
		"Cards held and doing nothing",
		"<code>chat-1</code></td><td><code>gpu07</code>", "30m0s ago",
	} {
		if !strings.Contains(body, want) {
			t.Errorf("the page lacks %q", want)
		}
	}
	if strings.Contains(body, "<code>code-1</code></td><td><code>gpu07</code>") {
		t.Error("a busy replica was listed as idle")
	}
	_, temp := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=temp", nil)
	if !strings.Contains(temp, "<strong>temperature</strong>") || !strings.Contains(temp, ">81</span>") {
		t.Error("the map coloured by temperature does not say so, or show it")
	}
	// The default colours by how much of the chip works: card 2's draw of
	// 300 W of 700, where nothing reads its SMs; card 0's SMs where
	// something does. Busy stays a mode of its own.
	if !strings.Contains(body, "<strong>work</strong>") || !strings.Contains(body, ">43</span>") ||
		!strings.Contains(body, "300 of 700 W") {
		t.Error("the map does not colour by work, or say the draw against the limit")
	}
	if !strings.Contains(body, "gpu07 GPU 0 · chat (chat-1) · 1% busy · SMs 0% active · tensor cores 0%") {
		t.Error("a profiled card does not say how much of its chip is at work")
	}
	_, util := get(t, Dashboard(mapFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/?gpumap=util", nil)
	if !strings.Contains(util, "<strong>busy</strong>") || !strings.Contains(util, ">93</span>") {
		t.Error("the map coloured by busy does not say so, or show it")
	}
	_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
	if strings.Contains(cpu, "gpu-map") {
		t.Error("a fleet without GPUs has a GPU map")
	}
}

func TestEachModelSaysWhatItGetsFromItsCards(t *testing.T) {
	h := &fakeHistory{byExpr: map[string][]promq.Series{}}
	for i, vals := range [][2]float64{{120, 30}, {2, 2}, {1200, 400}, {80, 40}, {0.6, 0.2}, {55, 0}} {
		h.byExpr[efficiencyQueries[i].expr] = []promq.Series{ts("chat", vals[0], vals[0]), ts("code", vals[1], vals[1])}
	}
	got := loadEfficiency(context.Background(), h)
	if len(got) != 2 || got[0].Model != "chat" {
		t.Fatalf("%+v", got)
	}
	c := got[0]
	if c.PerGPU != 60 || c.PerWh != 120*3600/1200.0 || c.Util != 80 || c.PowerLoad != 60 || c.SMActive != 55 {
		t.Errorf("chat: %+v", c)
	}
	if got[1].PerGPU != 15 {
		t.Errorf("code: %+v", got[1])
	}
}