package cli import ( "context" "fmt" "math" "net/http" "sort" "strconv" "strings" "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) // The monitor: the cluster drawn the way btop draws a machine. History comes // from the Prometheus that scrapes the control plane (monitoring.prometheus in // server.yaml), read through the admin API; without one, from what this // console has seen since it opened. var monRanges = []struct { label string window time.Duration refresh time.Duration }{ {"5m", 5 * time.Minute, 5 * time.Second}, {"1h", time.Hour, 15 * time.Second}, {"24h", 24 * time.Hour, time.Minute}, {"7d", 7 * 24 * time.Hour, 5 * time.Minute}, } type monitorState struct { rng int res adminapi.MetricsResult err error at time.Time key string // local is what this console sampled itself, every refresh, by series: // the charts when there is no Prometheus. local map[string][]float64 localAt []time.Time } type monitorMsg struct { key string res adminapi.MetricsResult err error } const localKeep = 1200 // an hour of three-second samples // sample keeps what the cluster looks like now, for the charts without a // Prometheus. func (s *monitorState) sample(d consoleData) { if s.local == nil { s.local = map[string][]float64{} } add := func(k string, v float64) { s.local[k] = append(s.local[k], v) if len(s.local[k]) > localKeep { s.local[k] = s.local[k][len(s.local[k])-localKeep:] } } var cpu, used, total float64 n := 0 for _, nd := range d.nodes { if nd.Load == nil { continue } l := nd.Load cpu += l.CPUPercent n++ used, total = used+float64(l.MemUsed), total+float64(l.MemTotal) add("nodecpu/"+nd.ID, l.CPUPercent) add("nodemem/"+nd.ID, l.MemPercent) for _, g := range l.GPUs { card := nd.ID + ":" + strconv.Itoa(g.Index) add("gpu/"+card, g.UtilPercent) if g.MemoryTotalMB > 0 { add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB) } add("gputemp/"+card, g.TempC) add("gpupow/"+card, g.PowerW) } } if n > 0 { add("cpu/", cpu/float64(n)) } if total > 0 { add("mem/", 100*used/total) } } // localSeries is the series sampled under an id, as charts take them. func (s monitorState) localSeries(id string) []promq.Series { var out []promq.Series for k, vals := range s.local { pre, name, ok := strings.Cut(k, "/") if !ok || pre != id { continue } ser := promq.Series{Name: name} for _, v := range vals { ser.Points = append(ser.Points, promq.Point{Value: v}) } out = append(out, ser) } sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out } // netDevs and diskDevs leave out what is not the machine's own traffic: // loopback, container bridges, device-mapper over the disks it is made of. const ( netDevs = `job="nodes",device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|cali.*|virbr.*|tap.*"` diskDevs = `job="nodes",device!~"loop.*|ram.*|dm-.*|sr.*"` ) // bps is bytes a second as a person reads it. func bps(v float64) string { return bytesShort(int64(v)) + "/s" } // promDur is a duration as PromQL writes one. func promDur(d time.Duration) string { if d%time.Minute == 0 { return strconv.Itoa(int(d/time.Minute)) + "m" } return strconv.Itoa(int(d/time.Second)) + "s" } // monitorQueries are what the monitor draws, for a window drawn at points // points wide. func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) { step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second)) // A rate over at least four scrapes, or it is noise. rw := promDur(max(time.Minute, 4*step).Round(time.Second)) card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` } return step, []adminapi.MetricsQuery{ {ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`}, {ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`}, {ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"}, {ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"}, {ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"}, {ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"}, // Totals fall back to zero: a counter not yet incremented since the // control plane started is no series at all, and a quiet hour should // read as none, not as missing. {ID: "rps", Expr: `sum(rate(llmgw_requests_total[` + rw + `])) or vector(0)`}, {ID: "rpsby", Expr: `sum by (model) (rate(llmgw_requests_total[` + rw + `]))`, By: "model"}, {ID: "errs", Expr: `sum(rate(llmgw_requests_total{status=~"5..|429"}[` + rw + `])) or vector(0)`}, {ID: "tokin", Expr: `sum(rate(llmgw_tokens_total{direction="prompt"}[` + rw + `])) or vector(0)`}, {ID: "tokout", Expr: `sum(rate(llmgw_tokens_total{direction="completion"}[` + rw + `])) or vector(0)`}, // The machines' own, from their node exporters. {ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]))`}, {ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`}, {ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `}[` + rw + `]))`}, {ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `}[` + rw + `]))`}, {ID: "nodenet", Expr: `sum by (node) (rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]) + rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`, By: "node"}, {ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`}, {ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`}, {ID: "inflight", Expr: `sum(llmgw_endpoint_inflight)`}, {ID: "queue", Expr: `sum(llmgw_admission_queue_depth)`}, {ID: "nodecpu", Expr: `llmgw_node_cpu_utilization_percent`, By: "node"}, {ID: "nodemem", Expr: `100 * llmgw_node_memory_used_bytes / llmgw_node_memory_bytes`, By: "node"}, } } func (m consoleModel) monitorWidth() int { return max(40, m.width-navWidth-8) } // monitorKey says what is drawn: the range and the width it is drawn at. func (m consoleModel) monitorKey() string { return fmt.Sprintf("%d/%d", m.mon.rng, m.monitorWidth()) } // monitorDue is whether the charts should be read again. func (m consoleModel) monitorDue() bool { return m.mon.key != m.monitorKey() || time.Since(m.mon.at) >= monRanges[m.mon.rng].refresh } func (m *consoleModel) fetchMonitor() tea.Cmd { r := monRanges[m.mon.rng] step, qs := monitorQueries(r.window, (m.monitorWidth()/2-2)*2) key, c := m.monitorKey(), m.client m.mon.at, m.mon.key = time.Now(), key return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() var res adminapi.MetricsResult err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range", adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res) return monitorMsg{key: key, res: res, err: err} } } // series is what is drawn under an id: from Prometheus, or sampled here. func (m consoleModel) series(id string) []chartSeries { src := m.mon.res.Series[id] if !m.mon.res.Configured { src = m.mon.localSeries(id) } return toChart(src, m.mon.res.Configured, m.mon.at, monRanges[m.mon.rng].window) } // toChart turns what Prometheus gave into chart series placed by time over // the window that ended when it was read; sampled here, they are spread // over what was seen. func toChart(src []promq.Series, timed bool, readAt time.Time, window time.Duration) []chartSeries { out := make([]chartSeries, 0, len(src)) for _, s := range src { cs := chartSeries{name: s.Name} stamped := timed for _, p := range s.Points { cs.values = append(cs.values, p.Value) cs.at = append(cs.at, p.At) stamped = stamped && !p.At.IsZero() } if !stamped { cs.at = nil } if stamped && !readAt.IsZero() { cs.from, cs.to = readAt.Add(-window), readAt } out = append(out, cs) } return out } func lastOf(s []chartSeries) float64 { t := 0.0 for _, x := range s { for i := len(x.values) - 1; i >= 0; i-- { if !math.IsNaN(x.values[i]) { t += x.values[i] break } } } return t } func rate(v float64) string { switch { case v >= 1000: return fmt.Sprintf("%.1fk/s", v/1000) case v >= 10: return fmt.Sprintf("%.0f/s", v) } return fmt.Sprintf("%.2f/s", v) } // side puts two blocks next to each other, each w wide. func side(w int, a, b string) string { st := lipgloss.NewStyle().Width(w) return lipgloss.JoinHorizontal(lipgloss.Top, st.Render(a), " ", st.Render(b)) } func (m consoleModel) monitorPanel(pw int) (string, string, string) { keys := "→ or Enter open · ↑↓ choose · N test the webhooks · Q leave" if m.inPanel { keys = "1–4 or R range · PgUp/PgDn scroll · ← back · Q leave" } r := monRanges[m.mon.rng] var head strings.Builder head.WriteString("\n ") for i, x := range monRanges { if i == m.mon.rng { head.WriteString(fSelected.Render("["+x.label+"]") + " ") } else { head.WriteString(sDim.Render(" "+x.label+" ") + " ") } } src := "from Prometheus" switch { case m.mon.err != nil: src = sBad.Render("the control plane: " + m.mon.err.Error()) case !m.mon.res.Configured: src = sWarn.Render("from what this console has seen since it opened") + sDim.Render(" — set monitoring.prometheus in server.yaml for history") case len(m.mon.res.Errors) > 0: for _, e := range m.mon.res.Errors { src = sWarn.Render("Prometheus: " + truncate(e, 80)) break } } head.WriteString(" " + src + "\n\n") w := (pw - 6) / 2 ch := 5 span := r.label if !m.mon.res.Configured { span = "since opened" } pct := func(id string) string { return fmt.Sprintf("%.0f%%", lastOf(m.series(id))) } box := func(title, now string, s []chartSeries, top float64) string { if len(s) == 0 { empty := "no samples yet" if m.mon.res.Configured { empty = "no requests in the last " + span } return mdBold.Render(title) + "\n\n " + sDim.Render(empty) } return chartBox(title, now, s, w, ch, top, span) } netBox := func(title string, s []chartSeries) string { if len(s) == 0 { return mdBold.Render(title) + "\n\n " + sDim.Render("no node exporter answering: llmgw node add installs one") } return box(title, bps(lastOf(s)), s, 0) } var b strings.Builder b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100), box("Memory", pct("mem"), m.series("mem"), 100)), " ") + "\n\n") if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 { b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), " ") + "\n\n") } if m.mon.res.Configured { rps := m.series("rps") tok := append(named(m.series("tokin"), "prompt"), named(m.series("tokout"), "completion")...) b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0), box("Tokens", rate(lastOf(tok)), tok, 0)), " ") + "\n\n") ttft := append(named(m.series("ttft50"), "p50"), named(m.series("ttft99"), "p99")...) load := append(named(m.series("inflight"), "in flight"), named(m.series("queue"), "queued")...) load = append(load, named(m.series("errs"), "errors/s")...) if len(ttft) == 0 { ttft = nil } b.WriteString(indent(side(w, box("Time to first token (s)", "", ttft, 0), box("In flight · queued · errors", "", load, 0)), " ") + "\n\n") net := append(named(m.series("netin"), "in"), named(m.series("netout"), "out")...) disk := append(named(m.series("diskr"), "read"), named(m.series("diskw"), "written")...) b.WriteString(indent(side(w, netBox("Network", net), netBox("Disk", disk)), " ") + "\n\n") } else { b.WriteString(" " + sDim.Render("Requests, tokens, latency, network and disk are drawn from Prometheus.") + "\n\n") } // Each node: its CPU and memory now, and how they went. cpuBy, memBy, netBy := byName(m.series("nodecpu")), byName(m.series("nodemem")), byName(m.series("nodenet")) sw := max(10, (pw-50)/2) b.WriteString(" " + mdBold.Render("Nodes") + sDim.Render(fmt.Sprintf("%*s", 25+sw, "CPU")) + sDim.Render(fmt.Sprintf("%*s", 18+sw, "memory")) + "\n") for _, nd := range m.data.nodes { c, mm := 0.0, 0.0 if nd.Load != nil { c, mm = nd.Load.CPUPercent, nd.Load.MemPercent } netCol := "" if v, ok := netBy[nd.ID]; ok { netCol = fmt.Sprintf(" net %s %9s", sparkSeries(v, 12, 0), bps(last(v.values))) } fmt.Fprintf(&b, " %-16s %s %4.0f%% %s %s %4.0f%% %s%s\n", truncate(nd.ID, 16), meter(c/100, 8), c, sparkSeries(cpuBy[nd.ID], sw, 100), meter(mm/100, 8), mm, sparkSeries(memBy[nd.ID], sw, 100), netCol) } // Each card: busy, memory, heat and power. if gpus := m.series("gpu"); len(gpus) > 0 { vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow")) b.WriteString("\n " + mdBold.Render("GPUs") + "\n") for _, g := range gpus { fmt.Fprintf(&b, " %-16s busy %s %4.0f%% memory %s %4.0f%% %3.0f°C %s %4.0f W\n", truncate(g.name, 16), sparkSeries(g, sw/2, 100), last(g.values), sparkSeries(vram[g.name], sw/2, 100), last(vram[g.name].values), last(temp[g.name].values), sparkSeries(temp[g.name], 8, 100), last(pow[g.name].values)) } } // Each model's requests. if rps := m.series("rpsby"); len(rps) > 0 { b.WriteString("\n " + mdBold.Render("Models") + sDim.Render(" requests a second") + "\n") for _, s := range rps { fmt.Fprintf(&b, " %-30s %s %s\n", truncate(s.name, 30), sparkSeries(s, max(20, pw-50), 0), rate(last(s.values))) } } return "Monitor", head.String() + b.String(), keys } func named(s []chartSeries, name string) []chartSeries { for i := range s { s[i].name = name } return s } func byName(s []chartSeries) map[string]chartSeries { out := map[string]chartSeries{} for _, x := range s { out[x.name] = x } return out } func last(v []float64) float64 { for i := len(v) - 1; i >= 0; i-- { if !math.IsNaN(v[i]) { return v[i] } } return 0 } // Charts on a node's and a model's own page: the same reading, kept apart so // it does not disturb the monitor's. func detailQueries(kind, id string, window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) { step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second)) rw := promDur(max(time.Minute, 4*step).Round(time.Second)) q := strconv.Quote(id) card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` } if kind == "node" { return step, []adminapi.MetricsQuery{ {ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`}, {ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`}, {ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"}, {ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"}, {ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`}, {ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`}, {ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`}, {ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`}, } } return step, []adminapi.MetricsQuery{ {ID: "rps", Expr: `sum(rate(llmgw_requests_total{model=` + q + `}[` + rw + `]))`}, {ID: "errs", Expr: `sum(rate(llmgw_requests_total{model=` + q + `,status=~"5..|429"}[` + rw + `]))`}, {ID: "tok", Expr: `sum by (direction) (rate(llmgw_tokens_total{model=` + q + `,direction!="cached"}[` + rw + `]))`, By: "direction"}, {ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`}, {ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`}, } } // detailTarget is the node or model whose page is open, if any. func (m consoleModel) detailTarget() (kind, id string) { if !m.detail { return "", "" } switch { case m.menu == cNodes && m.row < len(m.data.nodes): return "node", m.data.nodes[m.row].ID case m.menu == cModels && m.row < len(m.data.models): return "model", m.data.models[m.row].Name } return "", "" } func (m consoleModel) detailKey() string { kind, id := m.detailTarget() return fmt.Sprintf("%s/%s/%d/%d", kind, id, m.mon.rng, m.monitorWidth()) } func (m consoleModel) detailDue() bool { if k, _ := m.detailTarget(); k == "" { return false } return m.det.key != m.detailKey() || time.Since(m.det.at) >= monRanges[m.mon.rng].refresh } type detailMsg monitorMsg func (m *consoleModel) fetchDetail() tea.Cmd { kind, id := m.detailTarget() if kind == "" { return nil } r := monRanges[m.mon.rng] step, qs := detailQueries(kind, id, r.window, (m.monitorWidth()/2-2)*2) key, c := m.detailKey(), m.client m.det.at = time.Now() return func() tea.Msg { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() var res adminapi.MetricsResult err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range", adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res) return detailMsg{key: key, res: res, err: err} } } // detailCharts are the charts at the foot of a node's or model's page. func (m consoleModel) detailCharts(pw int) string { kind, id := m.detailTarget() if kind == "" { return "" } r := monRanges[m.mon.rng] head := "\n " + mdBold.Render("Over the last "+r.label) + sDim.Render(" (1–4 or R another range)") + "\n\n" if m.det.err != nil { return head + " " + sBad.Render(m.det.err.Error()) } get := func(qid string) []chartSeries { return toChart(m.det.res.Series[qid], true, m.det.at, r.window) } if !m.det.res.Configured { // Without Prometheus, a node's own samples from this console. if kind != "node" { return head + " " + sDim.Render("A model's traffic is drawn from Prometheus: set monitoring.prometheus in server.yaml.") } get = func(qid string) []chartSeries { var out []chartSeries for _, s := range m.mon.localSeries("node" + qid) { if s.Name == id { cs := chartSeries{name: s.Name} for _, p := range s.Points { cs.values = append(cs.values, p.Value) } out = append(out, cs) } } return out } } w := (pw - 6) / 2 box := func(title, now string, s []chartSeries, top float64) string { if len(s) == 0 { return mdBold.Render(title) + "\n\n " + sDim.Render("no samples") } return chartBox(title, now, s, w, 5, top, r.label) } var b strings.Builder b.WriteString(head) if kind == "node" { cpu, mem := get("cpu"), get("mem") b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100), box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), " ")) if g := get("gpu"); len(g) > 0 { b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), " ")) } if m.det.res.Configured { net := append(named(get("netin"), "in"), named(get("netout"), "out")...) disk := append(named(get("diskr"), "read"), named(get("diskw"), "written")...) if len(net)+len(disk) > 0 { b.WriteString("\n\n" + indent(side(w, box("Network", bps(lastOf(net)), net, 0), box("Disk", bps(lastOf(disk)), disk, 0)), " ")) } else { b.WriteString("\n\n " + sDim.Render("Network and disk come from the node's exporter, which is not answering: apt-get install prometheus-node-exporter")) } } return b.String() } rps, tok := get("rps"), get("tok") b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0), box("Tokens", rate(lastOf(tok)), tok, 0)), " ")) ttft := append(named(get("ttft50"), "p50"), named(get("ttft99"), "p99")...) b.WriteString("\n\n" + indent(side(w, box("Time to first token (s)", "", ttft, 0), box("Errors", rate(lastOf(get("errs"))), get("errs"), 0)), " ")) return b.String() }