A btop-like monitor in the console, from Prometheus; node exporters discovered and installed

Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk
as Braille charts over 5m/1h/24h/7d, then per node, card and model
sparklines; node and model pages have their own. History is read from
monitoring.prometheus through POST /api/v1/metrics/range, or without it
from what the console saw. The control plane serves the cluster's machines
for Prometheus http_sd at /prometheus/targets on its metrics port, and node
add installs prometheus-node-exporter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:51:05 +02:00
1 parent 2e55487eca
commit a1b1663e9d
16 files changed
+1217 -4

No files matched your search

+534
View File
@@ -0,0 +1,534 @@
package cli
import (
"context"
"fmt"
"math"
"net/http"
"sort"
"strconv"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)
// The monitor: the cluster drawn the way btop draws a machine. History comes
// from the Prometheus that scrapes the control plane (monitoring.prometheus in
// server.yaml), read through the admin API; without one, from what this
// console has seen since it opened.
var monRanges = []struct {
label string
window time.Duration
refresh time.Duration
}{
{"5m", 5 * time.Minute, 5 * time.Second},
{"1h", time.Hour, 15 * time.Second},
{"24h", 24 * time.Hour, time.Minute},
{"7d", 7 * 24 * time.Hour, 5 * time.Minute},
}
type monitorState struct {
rng int
res adminapi.MetricsResult
err error
at time.Time
key string
// local is what this console sampled itself, every refresh, by series:
// the charts when there is no Prometheus.
local map[string][]float64
localAt []time.Time
}
type monitorMsg struct {
key string
res adminapi.MetricsResult
err error
}
const localKeep = 1200 // an hour of three-second samples
// sample keeps what the cluster looks like now, for the charts without a
// Prometheus.
func (s *monitorState) sample(d consoleData) {
if s.local == nil {
s.local = map[string][]float64{}
}
add := func(k string, v float64) {
s.local[k] = append(s.local[k], v)
if len(s.local[k]) > localKeep {
s.local[k] = s.local[k][len(s.local[k])-localKeep:]
}
}
var cpu, used, total float64
n := 0
for _, nd := range d.nodes {
if nd.Load == nil {
continue
}
l := nd.Load
cpu += l.CPUPercent
n++
used, total = used+float64(l.MemUsed), total+float64(l.MemTotal)
add("nodecpu/"+nd.ID, l.CPUPercent)
add("nodemem/"+nd.ID, l.MemPercent)
for _, g := range l.GPUs {
card := nd.ID + ":" + strconv.Itoa(g.Index)
add("gpu/"+card, g.UtilPercent)
if g.MemoryTotalMB > 0 {
add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB)
}
add("gputemp/"+card, g.TempC)
add("gpupow/"+card, g.PowerW)
}
}
if n > 0 {
add("cpu/", cpu/float64(n))
}
if total > 0 {
add("mem/", 100*used/total)
}
}
// localSeries is the series sampled under an id, as charts take them.
func (s monitorState) localSeries(id string) []promq.Series {
var out []promq.Series
for k, vals := range s.local {
pre, name, ok := strings.Cut(k, "/")
if !ok || pre != id {
continue
}
ser := promq.Series{Name: name}
for _, v := range vals {
ser.Points = append(ser.Points, promq.Point{Value: v})
}
out = append(out, ser)
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
// netDevs and diskDevs leave out what is not the machine's own traffic:
// loopback, container bridges, device-mapper over the disks it is made of.
const (
netDevs = `job="nodes",device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|cali.*|virbr.*|tap.*"`
diskDevs = `job="nodes",device!~"loop.*|ram.*|dm-.*|sr.*"`
)
// bps is bytes a second as a person reads it.
func bps(v float64) string { return bytesShort(int64(v)) + "/s" }
// promDur is a duration as PromQL writes one.
func promDur(d time.Duration) string {
if d%time.Minute == 0 {
return strconv.Itoa(int(d/time.Minute)) + "m"
}
return strconv.Itoa(int(d/time.Second)) + "s"
}
// monitorQueries are what the monitor draws, for a window drawn at points
// points wide.
func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) {
step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second))
// A rate over at least four scrapes, or it is noise.
rw := promDur(max(time.Minute, 4*step).Round(time.Second))
card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` }
return step, []adminapi.MetricsQuery{
{ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`},
{ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`},
{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"},
{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"},
{ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"},
{ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"},
// Totals fall back to zero: a counter not yet incremented since the
// control plane started is no series at all, and a quiet hour should
// read as none, not as missing.
{ID: "rps", Expr: `sum(rate(llmgw_requests_total[` + rw + `])) or vector(0)`},
{ID: "rpsby", Expr: `sum by (model) (rate(llmgw_requests_total[` + rw + `]))`, By: "model"},
{ID: "errs", Expr: `sum(rate(llmgw_requests_total{status=~"5..|429"}[` + rw + `])) or vector(0)`},
{ID: "tokin", Expr: `sum(rate(llmgw_tokens_total{direction="prompt"}[` + rw + `])) or vector(0)`},
{ID: "tokout", Expr: `sum(rate(llmgw_tokens_total{direction="completion"}[` + rw + `])) or vector(0)`},
// The machines' own, from their node exporters.
{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]))`},
{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`},
{ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `}[` + rw + `]))`},
{ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `}[` + rw + `]))`},
{ID: "nodenet", Expr: `sum by (node) (rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]) + rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`, By: "node"},
{ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`},
{ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`},
{ID: "inflight", Expr: `sum(llmgw_endpoint_inflight)`},
{ID: "queue", Expr: `sum(llmgw_admission_queue_depth)`},
{ID: "nodecpu", Expr: `llmgw_node_cpu_utilization_percent`, By: "node"},
{ID: "nodemem", Expr: `100 * llmgw_node_memory_used_bytes / llmgw_node_memory_bytes`, By: "node"},
}
}
func (m consoleModel) monitorWidth() int { return max(40, m.width-navWidth-8) }
// monitorKey says what is drawn: the range and the width it is drawn at.
func (m consoleModel) monitorKey() string {
return fmt.Sprintf("%d/%d", m.mon.rng, m.monitorWidth())
}
// monitorDue is whether the charts should be read again.
func (m consoleModel) monitorDue() bool {
return m.mon.key != m.monitorKey() || time.Since(m.mon.at) >= monRanges[m.mon.rng].refresh
}
func (m *consoleModel) fetchMonitor() tea.Cmd {
r := monRanges[m.mon.rng]
step, qs := monitorQueries(r.window, (m.monitorWidth()/2-2)*2)
key, c := m.monitorKey(), m.client
m.mon.at, m.mon.key = time.Now(), key
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
var res adminapi.MetricsResult
err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range",
adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res)
return monitorMsg{key: key, res: res, err: err}
}
}
// series is what is drawn under an id: from Prometheus, or sampled here.
func (m consoleModel) series(id string) []chartSeries {
src := m.mon.res.Series[id]
if !m.mon.res.Configured {
src = m.mon.localSeries(id)
}
out := make([]chartSeries, 0, len(src))
for _, s := range src {
cs := chartSeries{name: s.Name}
for _, p := range s.Points {
cs.values = append(cs.values, p.Value)
}
out = append(out, cs)
}
return out
}
func lastOf(s []chartSeries) float64 {
t := 0.0
for _, x := range s {
for i := len(x.values) - 1; i >= 0; i-- {
if !math.IsNaN(x.values[i]) {
t += x.values[i]
break
}
}
}
return t
}
func rate(v float64) string {
switch {
case v >= 1000:
return fmt.Sprintf("%.1fk/s", v/1000)
case v >= 10:
return fmt.Sprintf("%.0f/s", v)
}
return fmt.Sprintf("%.2f/s", v)
}
// side puts two blocks next to each other, each w wide.
func side(w int, a, b string) string {
st := lipgloss.NewStyle().Width(w)
return lipgloss.JoinHorizontal(lipgloss.Top, st.Render(a), " ", st.Render(b))
}
func (m consoleModel) monitorPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "1–4 or R range · PgUp/PgDn scroll · ← back · Q leave"
}
r := monRanges[m.mon.rng]
var head strings.Builder
head.WriteString("\n ")
for i, x := range monRanges {
if i == m.mon.rng {
head.WriteString(fSelected.Render("["+x.label+"]") + " ")
} else {
head.WriteString(sDim.Render(" "+x.label+" ") + " ")
}
}
src := "from Prometheus"
switch {
case m.mon.err != nil:
src = sBad.Render("the control plane: " + m.mon.err.Error())
case !m.mon.res.Configured:
src = sWarn.Render("from what this console has seen since it opened") +
sDim.Render(" — set monitoring.prometheus in server.yaml for history")
case len(m.mon.res.Errors) > 0:
for _, e := range m.mon.res.Errors {
src = sWarn.Render("Prometheus: " + truncate(e, 80))
break
}
}
head.WriteString(" " + src + "\n\n")
w := (pw - 6) / 2
ch := 5
span := r.label
if !m.mon.res.Configured {
span = "since opened"
}
pct := func(id string) string { return fmt.Sprintf("%.0f%%", lastOf(m.series(id))) }
box := func(title, now string, s []chartSeries, top float64) string {
if len(s) == 0 {
empty := "no samples yet"
if m.mon.res.Configured {
empty = "no requests in the last " + span
}
return mdBold.Render(title) + "\n\n " + sDim.Render(empty)
}
return chartBox(title, now, s, w, ch, top, span)
}
netBox := func(title string, s []chartSeries) string {
if len(s) == 0 {
return mdBold.Render(title) + "\n\n " + sDim.Render("no node exporter answering: llmgw node add installs one")
}
return box(title, bps(lastOf(s)), s, 0)
}
var b strings.Builder
b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100),
box("Memory", pct("mem"), m.series("mem"), 100)), " ") + "\n\n")
if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 {
b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), " ") + "\n\n")
}
if m.mon.res.Configured {
rps := m.series("rps")
tok := append(named(m.series("tokin"), "prompt"), named(m.series("tokout"), "completion")...)
b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0),
box("Tokens", rate(lastOf(tok)), tok, 0)), " ") + "\n\n")
ttft := append(named(m.series("ttft50"), "p50"), named(m.series("ttft99"), "p99")...)
load := append(named(m.series("inflight"), "in flight"), named(m.series("queue"), "queued")...)
load = append(load, named(m.series("errs"), "errors/s")...)
if len(ttft) == 0 {
ttft = nil
}
b.WriteString(indent(side(w, box("Time to first token (s)", "", ttft, 0),
box("In flight · queued · errors", "", load, 0)), " ") + "\n\n")
net := append(named(m.series("netin"), "in"), named(m.series("netout"), "out")...)
disk := append(named(m.series("diskr"), "read"), named(m.series("diskw"), "written")...)
b.WriteString(indent(side(w, netBox("Network", net), netBox("Disk", disk)), " ") + "\n\n")
} else {
b.WriteString(" " + sDim.Render("Requests, tokens, latency, network and disk are drawn from Prometheus.") + "\n\n")
}
// Each node: its CPU and memory now, and how they went.
cpuBy, memBy, netBy := byName(m.series("nodecpu")), byName(m.series("nodemem")), byName(m.series("nodenet"))
sw := max(10, (pw-50)/2)
b.WriteString(" " + mdBold.Render("Nodes") + sDim.Render(fmt.Sprintf("%*s", 25+sw, "CPU")) +
sDim.Render(fmt.Sprintf("%*s", 18+sw, "memory")) + "\n")
for _, nd := range m.data.nodes {
c, mm := 0.0, 0.0
if nd.Load != nil {
c, mm = nd.Load.CPUPercent, nd.Load.MemPercent
}
netCol := ""
if v, ok := netBy[nd.ID]; ok {
netCol = fmt.Sprintf(" net %s %9s", sparkline(v, 12, 0), bps(last(v)))
}
fmt.Fprintf(&b, " %-16s %s %4.0f%% %s %s %4.0f%% %s%s\n", truncate(nd.ID, 16),
meter(c/100, 8), c, sparkline(cpuBy[nd.ID], sw, 100), meter(mm/100, 8), mm, sparkline(memBy[nd.ID], sw, 100), netCol)
}
// Each card: busy, memory, heat and power.
if gpus := m.series("gpu"); len(gpus) > 0 {
vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow"))
b.WriteString("\n " + mdBold.Render("GPUs") + "\n")
for _, g := range gpus {
fmt.Fprintf(&b, " %-16s busy %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16),
sparkline(g.values, sw/2, 100), last(g.values), sparkline(vram[g.name], sw/2, 100), last(vram[g.name]),
last(temp[g.name]), sparkline(temp[g.name], 8, 100), last(pow[g.name]))
}
}
// Each model's requests.
if rps := m.series("rpsby"); len(rps) > 0 {
b.WriteString("\n " + mdBold.Render("Models") + sDim.Render(" requests a second") + "\n")
for _, s := range rps {
fmt.Fprintf(&b, " %-30s %s %s\n", truncate(s.name, 30), sparkline(s.values, max(20, pw-50), 0), rate(last(s.values)))
}
}
return "Monitor", head.String() + b.String(), keys
}
func named(s []chartSeries, name string) []chartSeries {
for i := range s {
s[i].name = name
}
return s
}
func byName(s []chartSeries) map[string][]float64 {
out := map[string][]float64{}
for _, x := range s {
out[x.name] = x.values
}
return out
}
func last(v []float64) float64 {
for i := len(v) - 1; i >= 0; i-- {
if !math.IsNaN(v[i]) {
return v[i]
}
}
return 0
}
// Charts on a node's and a model's own page: the same reading, kept apart so
// it does not disturb the monitor's.
func detailQueries(kind, id string, window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) {
step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second))
rw := promDur(max(time.Minute, 4*step).Round(time.Second))
q := strconv.Quote(id)
card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` }
if kind == "node" {
return step, []adminapi.MetricsQuery{
{ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`},
{ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`},
{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"},
{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"},
{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
{ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`},
{ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`},
}
}
return step, []adminapi.MetricsQuery{
{ID: "rps", Expr: `sum(rate(llmgw_requests_total{model=` + q + `}[` + rw + `]))`},
{ID: "errs", Expr: `sum(rate(llmgw_requests_total{model=` + q + `,status=~"5..|429"}[` + rw + `]))`},
{ID: "tok", Expr: `sum by (direction) (rate(llmgw_tokens_total{model=` + q + `,direction!="cached"}[` + rw + `]))`, By: "direction"},
{ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`},
{ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`},
}
}
// detailTarget is the node or model whose page is open, if any.
func (m consoleModel) detailTarget() (kind, id string) {
if !m.detail {
return "", ""
}
switch {
case m.menu == cNodes && m.row < len(m.data.nodes):
return "node", m.data.nodes[m.row].ID
case m.menu == cModels && m.row < len(m.data.models):
return "model", m.data.models[m.row].Name
}
return "", ""
}
func (m consoleModel) detailKey() string {
kind, id := m.detailTarget()
return fmt.Sprintf("%s/%s/%d/%d", kind, id, m.mon.rng, m.monitorWidth())
}
func (m consoleModel) detailDue() bool {
if k, _ := m.detailTarget(); k == "" {
return false
}
return m.det.key != m.detailKey() || time.Since(m.det.at) >= monRanges[m.mon.rng].refresh
}
type detailMsg monitorMsg
func (m *consoleModel) fetchDetail() tea.Cmd {
kind, id := m.detailTarget()
if kind == "" {
return nil
}
r := monRanges[m.mon.rng]
step, qs := detailQueries(kind, id, r.window, (m.monitorWidth()/2-2)*2)
key, c := m.detailKey(), m.client
m.det.at = time.Now()
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
var res adminapi.MetricsResult
err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range",
adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res)
return detailMsg{key: key, res: res, err: err}
}
}
// detailCharts are the charts at the foot of a node's or model's page.
func (m consoleModel) detailCharts(pw int) string {
kind, id := m.detailTarget()
if kind == "" {
return ""
}
r := monRanges[m.mon.rng]
head := "\n " + mdBold.Render("Over the last "+r.label) + sDim.Render(" (1–4 or R another range)") + "\n\n"
if m.det.err != nil {
return head + " " + sBad.Render(m.det.err.Error())
}
get := func(qid string) []chartSeries {
var out []chartSeries
for _, s := range m.det.res.Series[qid] {
cs := chartSeries{name: s.Name}
for _, p := range s.Points {
cs.values = append(cs.values, p.Value)
}
out = append(out, cs)
}
return out
}
if !m.det.res.Configured {
// Without Prometheus, a node's own samples from this console.
if kind != "node" {
return head + " " + sDim.Render("A model's traffic is drawn from Prometheus: set monitoring.prometheus in server.yaml.")
}
get = func(qid string) []chartSeries {
var out []chartSeries
for _, s := range m.mon.localSeries("node" + qid) {
if s.Name == id {
cs := chartSeries{name: s.Name}
for _, p := range s.Points {
cs.values = append(cs.values, p.Value)
}
out = append(out, cs)
}
}
return out
}
}
w := (pw - 6) / 2
box := func(title, now string, s []chartSeries, top float64) string {
if len(s) == 0 {
return mdBold.Render(title) + "\n\n " + sDim.Render("no samples")
}
return chartBox(title, now, s, w, 5, top, r.label)
}
var b strings.Builder
b.WriteString(head)
if kind == "node" {
cpu, mem := get("cpu"), get("mem")
b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100),
box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), " "))
if g := get("gpu"); len(g) > 0 {
b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), " "))
}
if m.det.res.Configured {
net := append(named(get("netin"), "in"), named(get("netout"), "out")...)
disk := append(named(get("diskr"), "read"), named(get("diskw"), "written")...)
if len(net)+len(disk) > 0 {
b.WriteString("\n\n" + indent(side(w, box("Network", bps(lastOf(net)), net, 0), box("Disk", bps(lastOf(disk)), disk, 0)), " "))
} else {
b.WriteString("\n\n " + sDim.Render("Network and disk come from the node's exporter, which is not answering: apt-get install prometheus-node-exporter"))
}
}
return b.String()
}
rps, tok := get("rps"), get("tok")
b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0), box("Tokens", rate(lastOf(tok)), tok, 0)), " "))
ttft := append(named(get("ttft50"), "p50"), named(get("ttft99"), "p99")...)
b.WriteString("\n\n" + indent(side(w, box("Time to first token (s)", "", ttft, 0),
box("Errors", rate(lastOf(get("errs"))), get("errs"), 0)), " "))
return b.String()
}