Ilker ManapandClaude Opus 5.5
daed835d88
The nodes are CPU machines, and nothing measured how busy one was: the heartbeat carried GPU memory and disk, and llmgw_node_memory_bytes is the total. The agent now reads /proc between heartbeats - CPU across all cores, load averages, memory used and available, and each engine's process group's cores and resident memory - and sends it with the heartbeat. Nothing new runs on the nodes and Prometheus needs no new target: the control plane exports it with the rest of the fleet, as llmgw_node_cpu_utilization_percent, llmgw_node_cpus, llmgw_node_load, llmgw_node_memory_used_bytes, llmgw_node_memory_available_bytes, llmgw_shard_cpu_cores and llmgw_shard_memory_bytes. A measurement more than a minute old is left out rather than repeated. The cluster page's node table shows CPU and RAM as bars, and each node links to a page of its own: current CPU, RAM, load and disk, the engines on it with their cores and memory, and an hour of CPU, RAM, load, per-model cores and memory and queue depth from Prometheus. It refreshes in place. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
87 lines
2.7 KiB
Go
87 lines
2.7 KiB
Go
package web
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
|
)
|
|
|
|
// nodeMux mounts the page where it lives, so the path carries the node.
|
|
func nodeMux(src Source, h History) http.Handler {
|
|
mux := http.NewServeMux()
|
|
if h == nil {
|
|
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil))
|
|
} else {
|
|
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil))
|
|
}
|
|
return mux
|
|
}
|
|
|
|
// busySource is fakeSource with a node that has been measured.
|
|
type busySource struct{ fakeSource }
|
|
|
|
func (busySource) Nodes() []adminapi.NodeView {
|
|
n := fakeSource{}.Nodes()[0]
|
|
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
|
|
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
|
|
return []adminapi.NodeView{n}
|
|
}
|
|
|
|
func (busySource) Replicas(string) []adminapi.ReplicaView {
|
|
r := fakeSource{}.Replicas("")
|
|
r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30
|
|
return r
|
|
}
|
|
|
|
func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) {
|
|
h := &fakeHistory{byExpr: map[string][]promq.Series{
|
|
`llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)},
|
|
}}
|
|
code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
for _, want := range []string{
|
|
"62%", "CPU · 8 cores", "RAM in use", "3.25",
|
|
// The engine and its share: four of eight cores is half the machine.
|
|
"<td>served</td>", "4.0 cores", "width: 50%",
|
|
`hx-get="/admin/nodes/llmgw-node01"`,
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
// Every chart is about this node, and nothing else.
|
|
for _, q := range h.asked {
|
|
if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) {
|
|
t.Errorf("a query not scoped to the node: %s", q)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownNodeIsNotFound(t *testing.T) {
|
|
h := &fakeHistory{}
|
|
code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil)
|
|
if code != http.StatusNotFound {
|
|
t.Errorf("status %d", code)
|
|
}
|
|
// Nothing was asked of the metrics store for a name nobody knows.
|
|
if len(h.asked) != 0 {
|
|
t.Errorf("queried for an unknown node: %v", h.asked)
|
|
}
|
|
}
|
|
|
|
func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
|
|
_, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
|
if !strings.Contains(body, "not measured lately") {
|
|
t.Error("an unmeasured node should say so")
|
|
}
|
|
if !strings.Contains(body, "No metrics store is configured") {
|
|
t.Error("without a metrics store the page should say where history comes from")
|
|
}
|
|
}
|