A btop-like monitor in the console, from Prometheus; node exporters discovered and installed

Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk
as Braille charts over 5m/1h/24h/7d, then per node, card and model
sparklines; node and model pages have their own. History is read from
monitoring.prometheus through POST /api/v1/metrics/range, or without it
from what the console saw. The control plane serves the cluster's machines
for Prometheus http_sd at /prometheus/targets on its metrics port, and node
add installs prometheus-node-exporter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:51:05 +02:00
1 parent 2e55487eca
commit a1b1663e9d
16 files changed
+1217 -4

No files matched your search

+99
View File
@@ -17,6 +17,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
@@ -492,3 +493,101 @@ func TestJobsAreFollowedFromTheirOwnView(t *testing.T) {
t.Errorf("steps not shown:\n%s", v)
}
}
func TestTheMonitorDrawsFromPrometheusAndWithoutIt(t *testing.T) {
pts := func(vals ...float64) []promq.Series {
s := promq.Series{Name: "x"}
for _, v := range vals {
s.Points = append(s.Points, promq.Point{Value: v})
}
return []promq.Series{s}
}
var asked adminapi.MetricsRequest
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/metrics/range" {
_, _ = w.Write([]byte("[]"))
return
}
_ = json.NewDecoder(r.Body).Decode(&asked)
res := adminapi.MetricsResult{Configured: true, Series: map[string][]promq.Series{
"cpu": pts(10, 40, 80), "mem": pts(50, 55), "rps": pts(1, 2, 3), "rpsby": pts(1, 2, 3), "tokout": pts(100, 200),
"netin": pts(1e6, 2e6),
}}
res.Series["rpsby"][0].Name = "qwen"
_ = json.NewEncoder(w).Encode(res)
}))
defer admin.Close()
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel, m.width, m.height = true, cMonitor, true, 160, 60
var tm tea.Model = m
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("2")})
tm, _ = tm.Update(cmd())
if asked.Window != time.Hour || len(asked.Queries) < 10 {
t.Fatalf("asked %+v", asked)
}
v := tm.View()
for _, want := range []string{"[1h]", "CPU", "80%", "Requests", "qwen", "from Prometheus", "Network", "1.9 MB/s"} {
if !strings.Contains(v, want) {
t.Errorf("no %q:\n%s", want, v)
}
}
if !strings.ContainsAny(v, "⣀⣤⣶⣿") {
t.Errorf("no braille chart:\n%s", v)
}
// No Prometheus: what the console saw, and it says so.
m2, _ := consoleWith(t, cMonitor)
m2.width, m2.height = 160, 60
for _, c := range []float64{20, 60} {
m2.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready", Load: &adminapi.LoadView{CPUPercent: c, MemPercent: 30,
MemUsed: 3, MemTotal: 10}}}
m2.mon.sample(m2.data)
}
if v := m2.View(); !strings.Contains(v, "since it opened") || !strings.Contains(v, "60%") {
t.Errorf("local:\n%s", v)
}
}
func TestBrailleChartsFillToTheirHeight(t *testing.T) {
rows := brailleChart([]chartSeries{{values: []float64{0, 50, 100, 100}}}, 2, 2, 100)
if len(rows) != 2 {
t.Fatal(rows)
}
// The right cell is full top to bottom; the left holds a low point and a middle one.
if !strings.Contains(rows[0], "⣿") || !strings.Contains(rows[1], "⣿") {
t.Errorf("%q", rows)
}
if s := sparkline([]float64{0, 1, 2, 3}, 4, 3); !strings.Contains(s, "▁") || !strings.Contains(s, "█") {
t.Errorf("%q", s)
}
}
func TestANodesPageHasItsCharts(t *testing.T) {
var asked adminapi.MetricsRequest
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/metrics/range" {
_, _ = w.Write([]byte("[]"))
return
}
_ = json.NewDecoder(r.Body).Decode(&asked)
_ = json.NewEncoder(w).Encode(adminapi.MetricsResult{Configured: true, Series: map[string][]promq.Series{
"cpu": {{Name: "n1", Points: []promq.Point{{Value: 12}, {Value: 34}}}}}})
}))
defer admin.Close()
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel, m.width, m.height = true, cNodes, true, 160, 60
m.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready"}}
var tm tea.Model = m
tm = press(tm, "enter") // the menu; Details first
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
if cmd == nil {
t.Fatal("opening the page read nothing")
}
tm, _ = tm.Update(cmd())
if len(asked.Queries) == 0 || !strings.Contains(asked.Queries[0].Expr, `node="n1"`) {
t.Fatalf("asked %+v", asked.Queries)
}
if v := tm.View(); !strings.Contains(v, "Over the last") || !strings.Contains(v, "34%") {
t.Errorf("%s", v)
}
}