A btop-like monitor in the console, from Prometheus; node exporters discovered and installed
Monitor draws CPU, memory, GPUs, requests, tokens, latency, network and disk as Braille charts over 5m/1h/24h/7d, then per node, card and model sparklines; node and model pages have their own. History is read from monitoring.prometheus through POST /api/v1/metrics/range, or without it from what the console saw. The control plane serves the cluster's machines for Prometheus http_sd at /prometheus/targets on its metrics port, and node add installs prometheus-node-exporter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
2e55487eca
commit
a1b1663e9d
16 files changed
+1217
-4
No files matched your search
@@ -33,6 +33,7 @@ import (
|
||||
// Sidebar entries, in order. Headings are not entries.
|
||||
const (
|
||||
cOverview = iota
|
||||
cMonitor
|
||||
cNodes
|
||||
cGPUs
|
||||
cModels
|
||||
@@ -57,7 +58,7 @@ const (
|
||||
cCount
|
||||
)
|
||||
|
||||
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual",
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual",
|
||||
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
|
||||
"Restart all replicas", "Back up now", "Leave"}
|
||||
|
||||
@@ -159,6 +160,8 @@ type consoleModel struct {
|
||||
mirror consoleMirrorMsg
|
||||
hf hfState
|
||||
docs docViewer
|
||||
mon monitorState
|
||||
det monitorState
|
||||
}
|
||||
|
||||
var usageGroupings = []string{"team", "project", "org"}
|
||||
@@ -275,6 +278,12 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
if m.menu == cMirror {
|
||||
cmds = append(cmds, m.fetchMirror())
|
||||
}
|
||||
if m.menu == cMonitor && m.monitorDue() {
|
||||
cmds = append(cmds, m.fetchMonitor())
|
||||
}
|
||||
if m.detailDue() {
|
||||
cmds = append(cmds, m.fetchDetail())
|
||||
}
|
||||
if m.peopleView() {
|
||||
cmds = append(cmds, m.fetchPeople())
|
||||
}
|
||||
@@ -288,6 +297,16 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.data.err = msg.err
|
||||
} else {
|
||||
m.data = consoleData(msg)
|
||||
m.mon.sample(m.data)
|
||||
}
|
||||
case detailMsg:
|
||||
if msg.key == m.detailKey() {
|
||||
m.det.res, m.det.err, m.det.key = msg.res, msg.err, msg.key
|
||||
}
|
||||
case monitorMsg:
|
||||
// Kept if it is still what the screen wants: the range and width.
|
||||
if msg.key == m.monitorKey() {
|
||||
m.mon.res, m.mon.err, m.mon.key = msg.res, msg.err, msg.key
|
||||
}
|
||||
case consoleUsageMsg:
|
||||
m.usage, m.usageErr = msg.rows, msg.err
|
||||
@@ -467,6 +486,18 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.docs.show(m.row)
|
||||
m.detail = true
|
||||
}
|
||||
case "1", "2", "3", "4", "r":
|
||||
if kind, _ := m.detailTarget(); m.menu == cMonitor || kind != "" {
|
||||
if k.String() == "r" {
|
||||
m.mon.rng = (m.mon.rng + 1) % len(monRanges)
|
||||
} else {
|
||||
m.mon.rng = int(k.String()[0] - '1')
|
||||
}
|
||||
if kind != "" {
|
||||
return m, m.fetchDetail()
|
||||
}
|
||||
return m, m.fetchMonitor()
|
||||
}
|
||||
case "/":
|
||||
if m.menu == cServe {
|
||||
m.hfSearchDialog()
|
||||
@@ -524,6 +555,8 @@ func (m consoleModel) onEnter() tea.Cmd {
|
||||
return m.fetchQuotas()
|
||||
case cMirror:
|
||||
return m.fetchMirror()
|
||||
case cMonitor:
|
||||
return m.fetchMonitor()
|
||||
case cServe:
|
||||
if m.hf.models == nil {
|
||||
return m.fetchHF()
|
||||
@@ -961,7 +994,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
if m.menu == cLogs {
|
||||
return "Agent log · " + n.ID, m.logView(), "← back · PgUp/PgDn · refreshed every three seconds"
|
||||
}
|
||||
return "Node · " + n.ID, m.nodeDetail(n), "← back · PgUp/PgDn · Q leave"
|
||||
return "Node · " + n.ID, m.nodeDetail(n) + m.detailCharts(pw), "← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||||
}
|
||||
var rows [][]string
|
||||
for _, n := range m.data.nodes {
|
||||
@@ -993,7 +1026,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
m.gpuRows(), m.row, pw, m.inPanel), keys
|
||||
case cModels:
|
||||
if m.detail && m.row < len(m.data.models) {
|
||||
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]), "← back · PgUp/PgDn · Q leave"
|
||||
return "Model · " + m.data.models[m.row].Name, m.modelDetail(m.data.models[m.row]) + m.detailCharts(pw),
|
||||
"← back · 1–4 or R range · PgUp/PgDn · Q leave"
|
||||
}
|
||||
var rows [][]string
|
||||
for _, md := range m.data.models {
|
||||
@@ -1074,6 +1108,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
return m.mirrorPanel(pw)
|
||||
case cJobs:
|
||||
return m.jobsPanel(pw)
|
||||
case cMonitor:
|
||||
return m.monitorPanel(pw)
|
||||
case cServe:
|
||||
return m.hfPanel(pw)
|
||||
case cSettings:
|
||||
|
||||
Reference in new issue
Block a user