A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -17,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
@@ -492,3 +493,101 @@ func TestJobsAreFollowedFromTheirOwnView(t *testing.T) {
|
||||
t.Errorf("steps not shown:\n%s", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMonitorDrawsFromPrometheusAndWithoutIt(t *testing.T) {
|
||||
pts := func(vals ...float64) []promq.Series {
|
||||
s := promq.Series{Name: "x"}
|
||||
for _, v := range vals {
|
||||
s.Points = append(s.Points, promq.Point{Value: v})
|
||||
}
|
||||
return []promq.Series{s}
|
||||
}
|
||||
var asked adminapi.MetricsRequest
|
||||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/metrics/range" {
|
||||
_, _ = w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&asked)
|
||||
res := adminapi.MetricsResult{Configured: true, Series: map[string][]promq.Series{
|
||||
"cpu": pts(10, 40, 80), "mem": pts(50, 55), "rps": pts(1, 2, 3), "rpsby": pts(1, 2, 3), "tokout": pts(100, 200),
|
||||
"netin": pts(1e6, 2e6),
|
||||
}}
|
||||
res.Series["rpsby"][0].Name = "qwen"
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
}))
|
||||
defer admin.Close()
|
||||
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
|
||||
m.loaded, m.menu, m.inPanel, m.width, m.height = true, cMonitor, true, 160, 60
|
||||
var tm tea.Model = m
|
||||
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("2")})
|
||||
tm, _ = tm.Update(cmd())
|
||||
if asked.Window != time.Hour || len(asked.Queries) < 10 {
|
||||
t.Fatalf("asked %+v", asked)
|
||||
}
|
||||
v := tm.View()
|
||||
for _, want := range []string{"[1h]", "CPU", "80%", "Requests", "qwen", "from Prometheus", "Network", "1.9 MB/s"} {
|
||||
if !strings.Contains(v, want) {
|
||||
t.Errorf("no %q:\n%s", want, v)
|
||||
}
|
||||
}
|
||||
if !strings.ContainsAny(v, "⣀⣤⣶⣿") {
|
||||
t.Errorf("no braille chart:\n%s", v)
|
||||
}
|
||||
|
||||
// No Prometheus: what the console saw, and it says so.
|
||||
m2, _ := consoleWith(t, cMonitor)
|
||||
m2.width, m2.height = 160, 60
|
||||
for _, c := range []float64{20, 60} {
|
||||
m2.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready", Load: &adminapi.LoadView{CPUPercent: c, MemPercent: 30,
|
||||
MemUsed: 3, MemTotal: 10}}}
|
||||
m2.mon.sample(m2.data)
|
||||
}
|
||||
if v := m2.View(); !strings.Contains(v, "since it opened") || !strings.Contains(v, "60%") {
|
||||
t.Errorf("local:\n%s", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrailleChartsFillToTheirHeight(t *testing.T) {
|
||||
rows := brailleChart([]chartSeries{{values: []float64{0, 50, 100, 100}}}, 2, 2, 100)
|
||||
if len(rows) != 2 {
|
||||
t.Fatal(rows)
|
||||
}
|
||||
// The right cell is full top to bottom; the left holds a low point and a middle one.
|
||||
if !strings.Contains(rows[0], "⣿") || !strings.Contains(rows[1], "⣿") {
|
||||
t.Errorf("%q", rows)
|
||||
}
|
||||
if s := sparkline([]float64{0, 1, 2, 3}, 4, 3); !strings.Contains(s, "▁") || !strings.Contains(s, "█") {
|
||||
t.Errorf("%q", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodesPageHasItsCharts(t *testing.T) {
|
||||
var asked adminapi.MetricsRequest
|
||||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/metrics/range" {
|
||||
_, _ = w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&asked)
|
||||
_ = json.NewEncoder(w).Encode(adminapi.MetricsResult{Configured: true, Series: map[string][]promq.Series{
|
||||
"cpu": {{Name: "n1", Points: []promq.Point{{Value: 12}, {Value: 34}}}}}})
|
||||
}))
|
||||
defer admin.Close()
|
||||
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
|
||||
m.loaded, m.menu, m.inPanel, m.width, m.height = true, cNodes, true, 160, 60
|
||||
m.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready"}}
|
||||
var tm tea.Model = m
|
||||
tm = press(tm, "enter") // the menu; Details first
|
||||
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
if cmd == nil {
|
||||
t.Fatal("opening the page read nothing")
|
||||
}
|
||||
tm, _ = tm.Update(cmd())
|
||||
if len(asked.Queries) == 0 || !strings.Contains(asked.Queries[0].Expr, `node="n1"`) {
|
||||
t.Fatalf("asked %+v", asked.Queries)
|
||||
}
|
||||
if v := tm.View(); !strings.Contains(v, "Over the last") || !strings.Contains(v, "34%") {
|
||||
t.Errorf("%s", v)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user