llmgw/internal/web/node_page_test.go

package web

import (
	"net/http"
	"strings"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/topology"
	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)

// nodeMux mounts the page where it lives, so the path carries the node.
func nodeMux(src Source, h History) http.Handler {
	mux := http.NewServeMux()
	if h == nil {
		mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil))
	} else {
		mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil))
	}
	return mux
}

// busySource is fakeSource with a node that has been measured.
type busySource struct{ fakeSource }

func (busySource) Nodes() []adminapi.NodeView {
	n := fakeSource{}.Nodes()[0]
	n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
		MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
	n.IBPorts = []adminapi.IBPortView{
		{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, Active: true,
			Fabric: "ib-a", TxBytes: 4 << 30},
		{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", Errors: map[string]uint64{"link_downed": 7}},
	}
	return []adminapi.NodeView{n}
}

func (busySource) Replicas(string) []adminapi.ReplicaView {
	r := fakeSource{}.Replicas("")
	r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30
	return r
}

func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) {
	h := &fakeHistory{byExpr: map[string][]promq.Series{
		`llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)},
	}}
	code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
	if code != http.StatusOK {
		t.Fatalf("status %d", code)
	}
	for _, want := range []string{
		"62%", "CPU · 8 cores", "RAM in use", "3.25",
		// The engine and its share: four of eight cores is half the machine.
		"<td>served</td>", "4.0 cores", "width: 50%",
		`hx-get="/admin/nodes/llmgw-node01"`,
	} {
		if !strings.Contains(body, want) {
			t.Errorf("missing %q", want)
		}
	}
	// Every chart is about this node, and nothing else.
	for _, q := range h.asked {
		if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) {
			t.Errorf("a query not scoped to the node: %s", q)
		}
	}
}

func TestAnUnknownNodeIsNotFound(t *testing.T) {
	h := &fakeHistory{}
	code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil)
	if code != http.StatusNotFound {
		t.Errorf("status %d", code)
	}
	// Nothing was asked of the metrics store for a name nobody knows.
	if len(h.asked) != 0 {
		t.Errorf("queried for an unknown node: %v", h.asked)
	}
}

func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
	_, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
	if !strings.Contains(body, "not measured lately") {
		t.Error("an unmeasured node should say so")
	}
	if !strings.Contains(body, "No metrics store is configured") {
		t.Error("without a metrics store the page should say where history comes from")
	}
}

func TestANodesPageShowsItsFabricPorts(t *testing.T) {
	_, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
	for _, want := range []string{
		"<h2>Fabric</h2>", "On ib-a ·", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
		`<td class="bad">DOWN <span class="note">Polling</span></td>`, "link_downed 7",
	} {
		if !strings.Contains(body, want) {
			t.Errorf("missing %q", want)
		}
	}
}

// topoSource is busySource with a mapped fabric.
type topoSource struct{ busySource }

func (topoSource) Topologies() []adminapi.TopologyView {
	return []adminapi.TopologyView{{ID: "sm-0x01", Key: "sm:0x01", Label: "fabric of subnet manager 0x01", Declared: []string{"ib-a"}}}
}

func (s topoSource) Topology(id string) (adminapi.TopologyView, bool) {
	if id != "sm-0x01" {
		return adminapi.TopologyView{}, false
	}
	t := s.Topologies()[0]
	t.Map = &topology.View{Tiers: 2, Switches: []topology.SwitchView{{GUID: "0xs1", Name: "leaf-01</script><script>alert(1)", Tier: 1}}}
	return t, true
}

func TestAFabricsPageCarriesItsMapSafely(t *testing.T) {
	mux := http.NewServeMux()
	mux.HandleFunc("GET /admin/fabrics", FabricsPage(topoSource{}, false, "v1"))
	mux.HandleFunc("GET /admin/fabrics/{id}", FabricsPage(topoSource{}, false, "v1"))
	mux.HandleFunc("GET /admin/fabrics/{id}/topology.json", FabricTopologyJSON(topoSource{}))

	code, body := get(t, mux, "http://x/admin/fabrics", nil)
	if code != 200 {
		t.Fatalf("status %d", code)
	}
	for _, want := range []string{`id="fabric-map"`, `data-src="/admin/fabrics/sm-0x01/topology.json"`, "fabric.js?v=", `id="fabric-data"`} {
		if !strings.Contains(body, want) {
			t.Errorf("missing %q", want)
		}
	}
	// A switch named to break out of the script it is carried in cannot.
	if strings.Contains(body, "</script><script>alert(1)") {
		t.Error("a switch description closed the data script")
	}
	if code, _ := get(t, mux, "http://x/admin/fabrics/nope", nil); code != 404 {
		t.Errorf("unknown fabric: %d", code)
	}
	if code, body := get(t, mux, "http://x/admin/fabrics/sm-0x01/topology.json", nil); code != 200 || !strings.Contains(body, `"tiers":2`) {
		t.Errorf("json: %d %s", code, body)
	}
}

type nvlSource struct{ busySource }

func (nvlSource) NVLink() nvlink.Report {
	return nvlink.Report{
		Domains: []nvlink.Domain{{Key: "76b1ad76-7c24/7", ClusterUUID: "76b1ad76-7c24", CliqueID: "7", GPUs: 8, Joined: 7, Usable: 1,
			Members: []nvlink.Member{
				{Node: "t1", Usable: true, IMEX: "active", IMEXChannels: 1, GPUs: []nvlink.GPU{{Index: 0, Joined: true, State: "Completed"}}},
				{Node: "t2", IMEX: "inactive", GPUs: []nvlink.GPU{{Index: 3, State: "In Progress"}}},
			}}},
		Problems: []nvlink.Problem{{Level: nvlink.Fail, Domain: "76b1ad76-7c24/7", Message: "t2: nvidia-imex is inactive"}},
	}
}

func TestTheNVLinkPageDrawsEachDomainAsARack(t *testing.T) {
	_, body := get(t, NVLinkPage(nvlSource{}, false, "v1"), "http://x/admin/nvlink", nil)
	for _, want := range []string{`class="rack troubled"`, "76b1ad76·7", "1/2</b><span>trays usable", `class="gpu-chip joined"`,
		`class="gpu-chip lost"`, `class="tray out"`, "t2: nvidia-imex is inactive", `class="imex up"`} {
		if !strings.Contains(body, want) {
			t.Errorf("missing %q", want)
		}
	}
	// And the cluster page points at it.
	_, body = get(t, Dashboard(nvlSource{}, fakeUsage{}, "v1", time.Second, false, NavExtras{}, nil), "http://x/admin", nil)
	if !strings.Contains(body, "see the racks") {
		t.Error("the cluster page does not lead to the racks")
	}
}

// hotSource is a GPU node whose sensors report, one card held back.
type hotSource struct{ fakeSource }

func (hotSource) Nodes() []adminapi.NodeView {
	n := fakeSource{}.Nodes()[0]
	n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
		CPUTempC: 58, CPUPowerW: 312, InletTempC: 24.5,
		GPUs: []inventory.GPUSensor{
			{Index: 0, TempC: 71, MemoryTempC: 84, PowerW: 612, PowerLimitW: 700},
			{Index: 1, TempC: 88, PowerW: 698, PowerLimitW: 700, Throttle: []string{"sw_power_cap", "sw_thermal"}},
		}}
	return []adminapi.NodeView{n}
}

func TestANodesSensorsAreOnItsPageAndNothingItCannotMeasure(t *testing.T) {
	_, body := get(t, nodeMux(hotSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
	for _, want := range []string{"58 °C", "312 W", "24.5 °C", "inlet air", "84 °C", "612 W", "of 700",
		`<span class="bad">sw_power_cap, sw_thermal</span>`} {
		if !strings.Contains(body, want) {
			t.Errorf("the page lacks %q", want)
		}
	}
	_, vm := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
	for _, absent := range []string{"°C", "<h2>GPUs</h2>", "inlet air"} {
		if strings.Contains(vm, absent) {
			t.Errorf("a node with no sensors shows %q", absent)
		}
	}
}

// gpuSource is a node with two GPUs, one busy and one idle.
type gpuSource struct{ fakeSource }

func (gpuSource) Nodes() []adminapi.NodeView {
	n := fakeSource{}.Nodes()[0]
	n.GPUTotal = 2
	n.Load = &adminapi.LoadView{CPUPercent: 40, CPUs: 64, MemTotal: 1 << 40, MemUsed: 1 << 39,
		GPUs: []inventory.GPUSensor{
			{Index: 0, TempC: 71, PowerW: 612, PowerLimitW: 700, UtilPercent: 97, MemUtilPercent: 64,
				MemoryUsedMB: 72909, MemoryTotalMB: 81559, SMClockMHz: 1980, ECCUncorrected: 0},
			{Index: 1, TempC: 30, PowerW: 70, PowerLimitW: 700, UtilPercent: 0, MemUtilPercent: 0,
				MemoryUsedMB: 512, MemoryTotalMB: 81559, SMClockMHz: 345, ECCUncorrected: 3},
		}}
	return []adminapi.NodeView{n}
}

func TestAGPUNodesPageShowsHowBusyEachCardIsAndOverTime(t *testing.T) {
	q := func(expr string) string { return strings.Replace(expr, "%s", `"llmgw-node01"`, 1) }
	h := &fakeHistory{byExpr: map[string][]promq.Series{
		q(`llmgw_gpu_utilization_percent{node=%s} / 100`): {ts("0", 0.9, 0.97), ts("1", 0, 0)},
		q(`llmgw_gpu_memory_used_bytes{node=%s}`):         {ts("0", 7e10, 7.6e10)},
		q(`llmgw_gpu_temperature_celsius{node=%s}`):       {ts("0", 65, 71)},
		q(`llmgw_gpu_power_watts{node=%s}`):               {ts("0", 500, 612)},
	}}
	_, body := get(t, nodeMux(gpuSource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
	for _, want := range []string{
		"97%", "memory 64%", "71.2 <span class=\"note\">of 80 GB", "1980 MHz",
		"3 uncorrected ECC errors",
		"GPU busy, by card", "GPU memory used, by card", "GPU temperature, °C", "GPU power, W", "GPU 0",
	} {
		if !strings.Contains(body, want) {
			t.Errorf("the page lacks %q", want)
		}
	}
	// A node without GPUs is not asked about them.
	h2 := &fakeHistory{}
	get(t, nodeMux(cpuFleet{}, h2), "http://x/admin/nodes/llmgw-node01", nil)
	for _, e := range h2.asked {
		if strings.Contains(e, "llmgw_gpu_") {
			t.Errorf("a node with no GPU was asked for %s", e)
		}
	}
}

func TestTheClusterPageDrawsItsGPUsByClass(t *testing.T) {
	h := &fakeHistory{byExpr: map[string][]promq.Series{
		`avg by (class) (llmgw_gpu_utilization_percent) / 100`:                                         {ts("h100-80g", 0.2, 0.6)},
		`sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`: {ts("h100-80g", 0.8, 0.9)},
	}}
	_, body := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{History: h, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil)
	if !strings.Contains(body, "GPUs busy, by class") || !strings.Contains(body, "GPU memory in use, by class") {
		t.Error("a fleet with GPUs has no GPU charts")
	}
	h2 := &fakeHistory{}
	_, cpu := get(t, Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{History: h2, Window: time.Hour}, nil), "http://127.0.0.1:9090/", nil)
	if strings.Contains(cpu, "GPUs busy") {
		t.Error("a fleet without GPUs was drawn GPU charts")
	}
	for _, e := range h2.asked {
		if strings.Contains(e, "llmgw_gpu_") {
			t.Errorf("a fleet without GPUs was asked for %s", e)
		}
	}
}

// versionSource is hotSource with an agent a release behind, a model on its
// second GPU, and engines with their versions.
type versionSource struct{ hotSource }

func (versionSource) Nodes() []adminapi.NodeView {
	n := hotSource{}.Nodes()[0]
	n.AgentVersion = "v0"
	n.EngineVersions = map[string]string{"llamacpp": "b10991", "comfyui": "0.39.2"}
	return []adminapi.NodeView{n}
}

func (versionSource) Replicas(string) []adminapi.ReplicaView {
	r := fakeSource{}.Replicas("")
	r[0].Shards[0].GPUs = []int{1}
	return r
}

func TestANodesPageSaysWhatItRunsAndWhichModelIsOnEachGPU(t *testing.T) {
	_, body := get(t, nodeMux(versionSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
	model := versionSource{}.Replicas("")[0].Model
	for _, want := range []string{
		// The page's control plane runs v1: an agent on v0 is marked.
		`class="bad" title="the control plane runs v1">llmgw v0</span>`,
		"Engines: comfyui 0.39.2, llamacpp b10991",
	} {
		if !strings.Contains(body, want) {
			t.Errorf("the page lacks %q", want)
		}
	}
	// GPU 1 holds the model, GPU 0 nothing.
	rows := strings.Split(body, "<tr>")
	var gpu0, gpu1 string
	for _, row := range rows {
		switch {
		case strings.HasPrefix(row, "\n  <td class=\"num\">0</td>"):
			gpu0 = row
		case strings.HasPrefix(row, "\n  <td class=\"num\">1</td>"):
			gpu1 = row
		}
	}
	if !strings.Contains(gpu1, "<td>"+model+"</td>") || strings.Contains(gpu0, model) {
		t.Errorf("GPU rows:\n0: %.300s\n1: %.300s", gpu0, gpu1)
	}
}