A btop-like monitor in the console, from Prometheus; node exporters discovered and installed

Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk
as Braille charts over 5m/1h/24h/7d, then per node, card and model
sparklines; node and model pages have their own. History is read from
monitoring.prometheus through POST /api/v1/metrics/range, or without it
from what the console saw. The control plane serves the cluster's machines
for Prometheus http_sd at /prometheus/targets on its metrics port, and node
add installs prometheus-node-exporter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:51:05 +02:00
1 parent 2e55487eca
commit a1b1663e9d
16 files changed
+1217 -4

No files matched your search

+39 -3
View File
@@ -33,6 +33,7 @@ import (
// Sidebar entries, in order. Headings are not entries.
const (
cOverview = iota
cMonitor
cNodes
cGPUs
cModels
@@ -57,7 +58,7 @@ const (
cCount
)
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual",
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual",
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
@@ -159,6 +160,8 @@ type consoleModel struct {
mirror consoleMirrorMsg
hf hfState
docs docViewer
mon monitorState
det monitorState
}
var usageGroupings = []string{"team", "project", "org"}
@@ -275,6 +278,12 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
if m.menu == cMirror {
cmds = append(cmds, m.fetchMirror())
}
if m.menu == cMonitor && m.monitorDue() {
cmds = append(cmds, m.fetchMonitor())
}
if m.detailDue() {
cmds = append(cmds, m.fetchDetail())
}
if m.peopleView() {
cmds = append(cmds, m.fetchPeople())
}
@@ -288,6 +297,16 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.data.err = msg.err
} else {
m.data = consoleData(msg)
m.mon.sample(m.data)
}
case detailMsg:
if msg.key == m.detailKey() {
m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key
}
case monitorMsg:
// Kept if it is still what the screen wants: the range and width.
if msg.key == m.monitorKey() {
m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key
}
case consoleUsageMsg:
m.usage, m.usageErr = msg.rows, msg.err
@@ -467,6 +486,18 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.docs.show(m.row)
m.detail = true
}
case "1", "2", "3", "4", "r":
if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" {
if k.String() == "r" {
m.mon.rng = (m.mon.rng + 1) % len(monRanges)
} else {
m.mon.rng = int(k.String()[0] - '1')
}
if kind != "" {
return m, m.fetchDetail()
}
return m, m.fetchMonitor()
}
case "/":
if m.menu == cServe {
m.hfSearchDialog()
@@ -524,6 +555,8 @@ func (m consoleModel) onEnter() tea.Cmd {
return m.fetchQuotas()
case cMirror:
return m.fetchMirror()
case cMonitor:
return m.fetchMonitor()
case cServe:
if m.hf.models == nil {
return m.fetchHF()
@@ -961,7 +994,7 @@ func (m consoleModel) panel() (title, body, keys string) {
if m.menu == cLogs {
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
}
return "Node · " + n.ID, m.nodeDetail(n), "← back · PgUp/PgDn · Q leave"
return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave"
}
var rows [][]string
for _, n := range m.data.nodes {
@@ -993,7 +1026,8 @@ func (m consoleModel) panel() (title, body, keys string) {
m.gpuRows(), m.row, pw, m.inPanel), keys
case cModels:
if m.detail && m.row < len(m.data.models) {
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]), "← back · PgUp/PgDn · Q leave"
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw),
"← back · 1–4 or R range · PgUp/PgDn · Q leave"
}
var rows [][]string
for _, md := range m.data.models {
@@ -1074,6 +1108,8 @@ func (m consoleModel) panel() (title, body, keys string) {
return m.mirrorPanel(pw)
case cJobs:
return m.jobsPanel(pw)
case cMonitor:
return m.monitorPanel(pw)
case cServe:
return m.hfPanel(pw)
case cSettings: