llmgw/internal/cli/tui_monitor.go

package cli

import (
	"context"
	"fmt"
	"math"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	tea "github.com/charmbracelet/bubbletea"
	"github.com/charmbracelet/lipgloss"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
)

// The monitor: the cluster drawn the way btop draws a machine. History comes
// from the Prometheus that scrapes the control plane (monitoring.prometheus in
// server.yaml), read through the admin API; without one, from what this
// console has seen since it opened.

var monRanges = []struct {
	label   string
	window  time.Duration
	refresh time.Duration
}{
	{"5m", 5 * time.Minute, 5 * time.Second},
	{"1h", time.Hour, 15 * time.Second},
	{"24h", 24 * time.Hour, time.Minute},
	{"7d", 7 * 24 * time.Hour, 5 * time.Minute},
}

type monitorState struct {
	rng int
	res adminapi.MetricsResult
	err error
	at  time.Time
	key string
	// local is what this console sampled itself, every refresh, by series:
	// the charts when there is no Prometheus.
	local   map[string][]float64
	localAt []time.Time
}

type monitorMsg struct {
	key string
	res adminapi.MetricsResult
	err error
}

const localKeep = 1200 // an hour of three-second samples

// sample keeps what the cluster looks like now, for the charts without a
// Prometheus.
func (s *monitorState) sample(d consoleData) {
	if s.local == nil {
		s.local = map[string][]float64{}
	}
	add := func(k string, v float64) {
		s.local[k] = append(s.local[k], v)
		if len(s.local[k]) > localKeep {
			s.local[k] = s.local[k][len(s.local[k])-localKeep:]
		}
	}
	var cpu, used, total float64
	n := 0
	for _, nd := range d.nodes {
		if nd.Load == nil {
			continue
		}
		l := nd.Load
		cpu += l.CPUPercent
		n++
		used, total = used+float64(l.MemUsed), total+float64(l.MemTotal)
		add("nodecpu/"+nd.ID, l.CPUPercent)
		add("nodemem/"+nd.ID, l.MemPercent)
		for _, g := range l.GPUs {
			card := nd.ID + ":" + strconv.Itoa(g.Index)
			add("gpu/"+card, g.UtilPercent)
			if g.MemoryTotalMB > 0 {
				add("vram/"+card, 100*g.MemoryUsedMB/g.MemoryTotalMB)
			}
			add("gputemp/"+card, g.TempC)
			add("gpupow/"+card, g.PowerW)
		}
	}
	if n > 0 {
		add("cpu/", cpu/float64(n))
	}
	if total > 0 {
		add("mem/", 100*used/total)
	}
}

// localSeries is the series sampled under an id, as charts take them.
func (s monitorState) localSeries(id string) []promq.Series {
	var out []promq.Series
	for k, vals := range s.local {
		pre, name, ok := strings.Cut(k, "/")
		if !ok || pre != id {
			continue
		}
		ser := promq.Series{Name: name}
		for _, v := range vals {
			ser.Points = append(ser.Points, promq.Point{Value: v})
		}
		out = append(out, ser)
	}
	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
	return out
}

// netDevs and diskDevs leave out what is not the machine's own traffic:
// loopback, container bridges, device-mapper over the disks it is made of.
const (
	netDevs  = `job="nodes",device!~"lo|veth.*|docker.*|br-.*|cni.*|flannel.*|cali.*|virbr.*|tap.*"`
	diskDevs = `job="nodes",device!~"loop.*|ram.*|dm-.*|sr.*"`
)

// bps is bytes a second as a person reads it.
func bps(v float64) string { return bytesShort(int64(v)) + "/s" }

// promDur is a duration as PromQL writes one.
func promDur(d time.Duration) string {
	if d%time.Minute == 0 {
		return strconv.Itoa(int(d/time.Minute)) + "m"
	}
	return strconv.Itoa(int(d/time.Second)) + "s"
}

// monitorQueries are what the monitor draws, for a window drawn at points
// points wide.
func monitorQueries(window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) {
	step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second))
	// A rate over at least four scrapes, or it is noise.
	rw := promDur(max(time.Minute, 4*step).Round(time.Second))
	card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` }
	return step, []adminapi.MetricsQuery{
		{ID: "cpu", Expr: `avg(llmgw_node_cpu_utilization_percent)`},
		{ID: "mem", Expr: `100 * sum(llmgw_node_memory_used_bytes) / sum(llmgw_node_memory_bytes)`},
		{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent`), By: "card"},
		{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes`), By: "card"},
		{ID: "gputemp", Expr: card(`llmgw_gpu_temperature_celsius`), By: "card"},
		{ID: "gpupow", Expr: card(`llmgw_gpu_power_watts`), By: "card"},
		// Totals fall back to zero: a counter not yet incremented since the
		// control plane started is no series at all, and a quiet hour should
		// read as none, not as missing.
		{ID: "rps", Expr: `sum(rate(llmgw_requests_total[` + rw + `])) or vector(0)`},
		{ID: "rpsby", Expr: `sum by (model) (rate(llmgw_requests_total[` + rw + `]))`, By: "model"},
		{ID: "errs", Expr: `sum(rate(llmgw_requests_total{status=~"5..|429"}[` + rw + `])) or vector(0)`},
		{ID: "tokin", Expr: `sum(rate(llmgw_tokens_total{direction="prompt"}[` + rw + `])) or vector(0)`},
		{ID: "tokout", Expr: `sum(rate(llmgw_tokens_total{direction="completion"}[` + rw + `])) or vector(0)`},
		// The machines' own, from their node exporters.
		{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]))`},
		{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`},
		{ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `}[` + rw + `]))`},
		{ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `}[` + rw + `]))`},
		{ID: "nodenet", Expr: `sum by (node) (rate(node_network_receive_bytes_total{` + netDevs + `}[` + rw + `]) + rate(node_network_transmit_bytes_total{` + netDevs + `}[` + rw + `]))`, By: "node"},
		{ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`},
		{ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket[` + rw + `])))`},
		{ID: "inflight", Expr: `sum(llmgw_endpoint_inflight)`},
		{ID: "queue", Expr: `sum(llmgw_admission_queue_depth)`},
		{ID: "nodecpu", Expr: `llmgw_node_cpu_utilization_percent`, By: "node"},
		{ID: "nodemem", Expr: `100 * llmgw_node_memory_used_bytes / llmgw_node_memory_bytes`, By: "node"},
	}
}

func (m consoleModel) monitorWidth() int { return max(40, m.width-navWidth-8) }

// monitorKey says what is drawn: the range and the width it is drawn at.
func (m consoleModel) monitorKey() string {
	return fmt.Sprintf("%d/%d", m.mon.rng, m.monitorWidth())
}

// monitorDue is whether the charts should be read again.
func (m consoleModel) monitorDue() bool {
	return m.mon.key != m.monitorKey() || time.Since(m.mon.at) >= monRanges[m.mon.rng].refresh
}

func (m *consoleModel) fetchMonitor() tea.Cmd {
	r := monRanges[m.mon.rng]
	step, qs := monitorQueries(r.window, (m.monitorWidth()/2-2)*2)
	key, c := m.monitorKey(), m.client
	m.mon.at, m.mon.key = time.Now(), key
	return func() tea.Msg {
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
		defer cancel()
		var res adminapi.MetricsResult
		err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range",
			adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res)
		return monitorMsg{key: key, res: res, err: err}
	}
}

// series is what is drawn under an id: from Prometheus, or sampled here.
func (m consoleModel) series(id string) []chartSeries {
	src := m.mon.res.Series[id]
	if !m.mon.res.Configured {
		src = m.mon.localSeries(id)
	}
	return toChart(src, m.mon.res.Configured, m.mon.at, monRanges[m.mon.rng].window)
}

// toChart turns what Prometheus gave into chart series placed by time over
// the window that ended when it was read; sampled here, they are spread
// over what was seen.
func toChart(src []promq.Series, timed bool, readAt time.Time, window time.Duration) []chartSeries {
	out := make([]chartSeries, 0, len(src))
	for _, s := range src {
		cs := chartSeries{name: s.Name}
		stamped := timed
		for _, p := range s.Points {
			cs.values = append(cs.values, p.Value)
			cs.at = append(cs.at, p.At)
			stamped = stamped && !p.At.IsZero()
		}
		if !stamped {
			cs.at = nil
		}
		if stamped && !readAt.IsZero() {
			cs.from, cs.to = readAt.Add(-window), readAt
		}
		out = append(out, cs)
	}
	return out
}

func lastOf(s []chartSeries) float64 {
	t := 0.0
	for _, x := range s {
		for i := len(x.values) - 1; i >= 0; i-- {
			if !math.IsNaN(x.values[i]) {
				t += x.values[i]
				break
			}
		}
	}
	return t
}

func rate(v float64) string {
	switch {
	case v >= 1000:
		return fmt.Sprintf("%.1fk/s", v/1000)
	case v >= 10:
		return fmt.Sprintf("%.0f/s", v)
	}
	return fmt.Sprintf("%.2f/s", v)
}

// side puts two blocks next to each other, each w wide.
func side(w int, a, b string) string {
	st := lipgloss.NewStyle().Width(w)
	return lipgloss.JoinHorizontal(lipgloss.Top, st.Render(a), "  ", st.Render(b))
}

func (m consoleModel) monitorPanel(pw int) (string, string, string) {
	keys := "→ or Enter open · ↑↓ choose · N test the webhooks · Q leave"
	if m.inPanel {
		keys = "1–4 or R range · PgUp/PgDn scroll · ← back · Q leave"
	}
	r := monRanges[m.mon.rng]
	var head strings.Builder
	head.WriteString("\n  ")
	for i, x := range monRanges {
		if i == m.mon.rng {
			head.WriteString(fSelected.Render("["+x.label+"]") + " ")
		} else {
			head.WriteString(sDim.Render(" "+x.label+" ") + " ")
		}
	}
	src := "from Prometheus"
	switch {
	case m.mon.err != nil:
		src = sBad.Render("the control plane: " + m.mon.err.Error())
	case !m.mon.res.Configured:
		src = sWarn.Render("from what this console has seen since it opened") +
			sDim.Render(" — set monitoring.prometheus in server.yaml for history")
	case len(m.mon.res.Errors) > 0:
		for _, e := range m.mon.res.Errors {
			src = sWarn.Render("Prometheus: " + truncate(e, 80))
			break
		}
	}
	head.WriteString("  " + src + "\n\n")

	w := (pw - 6) / 2
	ch := 5
	span := r.label
	if !m.mon.res.Configured {
		span = "since opened"
	}
	pct := func(id string) string { return fmt.Sprintf("%.0f%%", lastOf(m.series(id))) }
	box := func(title, now string, s []chartSeries, top float64) string {
		if len(s) == 0 {
			empty := "no samples yet"
			if m.mon.res.Configured {
				empty = "no requests in the last " + span
			}
			return mdBold.Render(title) + "\n\n  " + sDim.Render(empty)
		}
		return chartBox(title, now, s, w, ch, top, span)
	}
	netBox := func(title string, s []chartSeries) string {
		if len(s) == 0 {
			return mdBold.Render(title) + "\n\n  " + sDim.Render("no node exporter answering: llmgw node add installs one")
		}
		return box(title, bps(lastOf(s)), s, 0)
	}
	var b strings.Builder
	b.WriteString(indent(side(w, box("CPU", pct("cpu"), m.series("cpu"), 100),
		box("Memory", pct("mem"), m.series("mem"), 100)), "  ") + "\n\n")
	if g := m.series("gpu"); len(g) > 0 || len(m.data.sum.GPUs) > 0 {
		b.WriteString(indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", m.series("vram"), 100)), "  ") + "\n\n")
	}
	if m.mon.res.Configured {
		rps := m.series("rps")
		tok := append(named(m.series("tokin"), "prompt"), named(m.series("tokout"), "completion")...)
		b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0),
			box("Tokens", rate(lastOf(tok)), tok, 0)), "  ") + "\n\n")
		ttft := append(named(m.series("ttft50"), "p50"), named(m.series("ttft99"), "p99")...)
		load := append(named(m.series("inflight"), "in flight"), named(m.series("queue"), "queued")...)
		load = append(load, named(m.series("errs"), "errors/s")...)
		if len(ttft) == 0 {
			ttft = nil
		}
		b.WriteString(indent(side(w, box("Time to first token (s)", "", ttft, 0),
			box("In flight · queued · errors", "", load, 0)), "  ") + "\n\n")
		net := append(named(m.series("netin"), "in"), named(m.series("netout"), "out")...)
		disk := append(named(m.series("diskr"), "read"), named(m.series("diskw"), "written")...)
		b.WriteString(indent(side(w, netBox("Network", net), netBox("Disk", disk)), "  ") + "\n\n")
	} else {
		b.WriteString("  " + sDim.Render("Requests, tokens, latency, network and disk are drawn from Prometheus.") + "\n\n")
	}

	// Each node: its CPU and memory now, and how they went.
	cpuBy, memBy, netBy := byName(m.series("nodecpu")), byName(m.series("nodemem")), byName(m.series("nodenet"))
	sw := max(10, (pw-50)/2)
	b.WriteString("  " + mdBold.Render("Nodes") + sDim.Render(fmt.Sprintf("%*s", 25+sw, "CPU")) +
		sDim.Render(fmt.Sprintf("%*s", 18+sw, "memory")) + "\n")
	for _, nd := range m.data.nodes {
		c, mm := 0.0, 0.0
		if nd.Load != nil {
			c, mm = nd.Load.CPUPercent, nd.Load.MemPercent
		}
		netCol := ""
		if v, ok := netBy[nd.ID]; ok {
			netCol = fmt.Sprintf("  net %s %9s", sparkSeries(v, 12, 0), bps(last(v.values)))
		}
		fmt.Fprintf(&b, "  %-16s %s %4.0f%% %s  %s %4.0f%% %s%s\n", truncate(nd.ID, 16),
			meter(c/100, 8), c, sparkSeries(cpuBy[nd.ID], sw, 100), meter(mm/100, 8), mm, sparkSeries(memBy[nd.ID], sw, 100), netCol)
	}
	// Each card: busy, memory, heat and power.
	if gpus := m.series("gpu"); len(gpus) > 0 {
		vram, temp, pow := byName(m.series("vram")), byName(m.series("gputemp")), byName(m.series("gpupow"))
		b.WriteString("\n  " + mdBold.Render("GPUs") + "\n")
		for _, g := range gpus {
			fmt.Fprintf(&b, "  %-16s busy %s %4.0f%%  memory %s %4.0f%%  %3.0f°C %s  %4.0f W\n", truncate(g.name, 16),
				sparkSeries(g, sw/2, 100), last(g.values), sparkSeries(vram[g.name], sw/2, 100), last(vram[g.name].values),
				last(temp[g.name].values), sparkSeries(temp[g.name], 8, 100), last(pow[g.name].values))
		}
	}
	// Each model's requests.
	if rps := m.series("rpsby"); len(rps) > 0 {
		b.WriteString("\n  " + mdBold.Render("Models") + sDim.Render("  requests a second") + "\n")
		for _, s := range rps {
			fmt.Fprintf(&b, "  %-30s %s %s\n", truncate(s.name, 30), sparkSeries(s, max(20, pw-50), 0), rate(last(s.values)))
		}
	}
	return "Monitor", head.String() + b.String(), keys
}

func named(s []chartSeries, name string) []chartSeries {
	for i := range s {
		s[i].name = name
	}
	return s
}

func byName(s []chartSeries) map[string]chartSeries {
	out := map[string]chartSeries{}
	for _, x := range s {
		out[x.name] = x
	}
	return out
}

func last(v []float64) float64 {
	for i := len(v) - 1; i >= 0; i-- {
		if !math.IsNaN(v[i]) {
			return v[i]
		}
	}
	return 0
}

// Charts on a node's and a model's own page: the same reading, kept apart so
// it does not disturb the monitor's.

func detailQueries(kind, id string, window time.Duration, points int) (time.Duration, []adminapi.MetricsQuery) {
	step := max(5*time.Second, (window / time.Duration(max(points, 1))).Round(time.Second))
	rw := promDur(max(time.Minute, 4*step).Round(time.Second))
	q := strconv.Quote(id)
	card := func(expr string) string { return `label_join(` + expr + `, "card", ":", "node", "gpu")` }
	if kind == "node" {
		return step, []adminapi.MetricsQuery{
			{ID: "cpu", Expr: `llmgw_node_cpu_utilization_percent{node=` + q + `}`},
			{ID: "mem", Expr: `100 * llmgw_node_memory_used_bytes{node=` + q + `} / llmgw_node_memory_bytes{node=` + q + `}`},
			{ID: "gpu", Expr: card(`llmgw_gpu_utilization_percent{node=` + q + `}`), By: "card"},
			{ID: "vram", Expr: card(`100 * llmgw_gpu_memory_used_bytes{node=` + q + `} / llmgw_gpu_memory_total_bytes{node=` + q + `}`), By: "card"},
			{ID: "netin", Expr: `sum(rate(node_network_receive_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
			{ID: "netout", Expr: `sum(rate(node_network_transmit_bytes_total{` + netDevs + `,node=` + q + `}[` + rw + `]))`},
			{ID: "diskr", Expr: `sum(rate(node_disk_read_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`},
			{ID: "diskw", Expr: `sum(rate(node_disk_written_bytes_total{` + diskDevs + `,node=` + q + `}[` + rw + `]))`},
		}
	}
	return step, []adminapi.MetricsQuery{
		{ID: "rps", Expr: `sum(rate(llmgw_requests_total{model=` + q + `}[` + rw + `]))`},
		{ID: "errs", Expr: `sum(rate(llmgw_requests_total{model=` + q + `,status=~"5..|429"}[` + rw + `]))`},
		{ID: "tok", Expr: `sum by (direction) (rate(llmgw_tokens_total{model=` + q + `,direction!="cached"}[` + rw + `]))`, By: "direction"},
		{ID: "ttft50", Expr: `histogram_quantile(0.5, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`},
		{ID: "ttft99", Expr: `histogram_quantile(0.99, sum by (le) (rate(llmgw_ttft_seconds_bucket{model=` + q + `}[` + rw + `])))`},
	}
}

// detailTarget is the node or model whose page is open, if any.
func (m consoleModel) detailTarget() (kind, id string) {
	if !m.detail {
		return "", ""
	}
	switch {
	case m.menu == cNodes && m.row < len(m.data.nodes):
		return "node", m.data.nodes[m.row].ID
	case m.menu == cModels && m.row < len(m.data.models):
		return "model", m.data.models[m.row].Name
	}
	return "", ""
}

func (m consoleModel) detailKey() string {
	kind, id := m.detailTarget()
	return fmt.Sprintf("%s/%s/%d/%d", kind, id, m.mon.rng, m.monitorWidth())
}

func (m consoleModel) detailDue() bool {
	if k, _ := m.detailTarget(); k == "" {
		return false
	}
	return m.det.key != m.detailKey() || time.Since(m.det.at) >= monRanges[m.mon.rng].refresh
}

type detailMsg monitorMsg

func (m *consoleModel) fetchDetail() tea.Cmd {
	kind, id := m.detailTarget()
	if kind == "" {
		return nil
	}
	r := monRanges[m.mon.rng]
	step, qs := detailQueries(kind, id, r.window, (m.monitorWidth()/2-2)*2)
	key, c := m.detailKey(), m.client
	m.det.at = time.Now()
	return func() tea.Msg {
		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
		defer cancel()
		var res adminapi.MetricsResult
		err := c.do(ctx, http.MethodPost, "/api/v1/metrics/range",
			adminapi.MetricsRequest{Window: r.window, Step: step, Queries: qs}, &res)
		return detailMsg{key: key, res: res, err: err}
	}
}

// detailCharts are the charts at the foot of a node's or model's page.
func (m consoleModel) detailCharts(pw int) string {
	kind, id := m.detailTarget()
	if kind == "" {
		return ""
	}
	r := monRanges[m.mon.rng]
	head := "\n  " + mdBold.Render("Over the last "+r.label) + sDim.Render("  (1–4 or R another range)") + "\n\n"
	if m.det.err != nil {
		return head + "  " + sBad.Render(m.det.err.Error())
	}
	get := func(qid string) []chartSeries {
		return toChart(m.det.res.Series[qid], true, m.det.at, r.window)
	}
	if !m.det.res.Configured {
		// Without Prometheus, a node's own samples from this console.
		if kind != "node" {
			return head + "  " + sDim.Render("A model's traffic is drawn from Prometheus: set monitoring.prometheus in server.yaml.")
		}
		get = func(qid string) []chartSeries {
			var out []chartSeries
			for _, s := range m.mon.localSeries("node" + qid) {
				if s.Name == id {
					cs := chartSeries{name: s.Name}
					for _, p := range s.Points {
						cs.values = append(cs.values, p.Value)
					}
					out = append(out, cs)
				}
			}
			return out
		}
	}
	w := (pw - 6) / 2
	box := func(title, now string, s []chartSeries, top float64) string {
		if len(s) == 0 {
			return mdBold.Render(title) + "\n\n  " + sDim.Render("no samples")
		}
		return chartBox(title, now, s, w, 5, top, r.label)
	}
	var b strings.Builder
	b.WriteString(head)
	if kind == "node" {
		cpu, mem := get("cpu"), get("mem")
		b.WriteString(indent(side(w, box("CPU", fmt.Sprintf("%.0f%%", lastOf(cpu)), cpu, 100),
			box("Memory", fmt.Sprintf("%.0f%%", lastOf(mem)), mem, 100)), "  "))
		if g := get("gpu"); len(g) > 0 {
			b.WriteString("\n\n" + indent(side(w, box("GPU busy", "", g, 100), box("GPU memory", "", get("vram"), 100)), "  "))
		}
		if m.det.res.Configured {
			net := append(named(get("netin"), "in"), named(get("netout"), "out")...)
			disk := append(named(get("diskr"), "read"), named(get("diskw"), "written")...)
			if len(net)+len(disk) > 0 {
				b.WriteString("\n\n" + indent(side(w, box("Network", bps(lastOf(net)), net, 0), box("Disk", bps(lastOf(disk)), disk, 0)), "  "))
			} else {
				b.WriteString("\n\n  " + sDim.Render("Network and disk come from the node's exporter, which is not answering: apt-get install prometheus-node-exporter"))
			}
		}
		return b.String()
	}
	rps, tok := get("rps"), get("tok")
	b.WriteString(indent(side(w, box("Requests", rate(lastOf(rps)), rps, 0), box("Tokens", rate(lastOf(tok)), tok, 0)), "  "))
	ttft := append(named(get("ttft50"), "p50"), named(get("ttft99"), "p99")...)
	b.WriteString("\n\n" + indent(side(w, box("Time to first token (s)", "", ttft, 0),
		box("Errors", rate(lastOf(get("errs"))), get("errs"), 0)), "  "))
	return b.String()
}