package web
import (
"context"
"fmt"
"html/template"
"log/slog"
"net/http"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// nodePageData is one node: what it is, how busy it is now, and how busy it
// has been.
type nodePageData struct {
Nav Nav
OnAdminListener bool
Node adminapi.NodeView
Engines []nodeEngine
Charts *nodeCharts
HasHistory bool
Back, Logs string
// Self is this page's address, for the in-place refresh.
Self string
Version, Now string
RefreshSec int
// RefreshMeta is the reload for a browser without script.
RefreshMeta template.HTML
}
// nodeEngine is one engine running on the node, and its share of it.
type nodeEngine struct {
Replica, Model, Shard, State string
CPUCores float64
MemoryBytes int64
// CPUShare is its cores as a percentage of the machine, for the bar.
CPUShare float64
}
// nodeCharts are the node's recent history, from the metrics store.
type nodeCharts struct {
CPU, Memory, Load, EngineCPU, EngineMemory, Queue TimeSeries
// The GPUs', one line per card; empty on a node without them.
GPUUtil, GPUMemory, GPUTemp, GPUPower TimeSeries
Window string
Err string
}
// nodePanels are the charts, each a query scoped to this node. %s is the
// node's ID, quoted.
var nodePanels = []struct {
expr, nameBy, unit string
into func(*nodeCharts, TimeSeries)
}{
// Divided to a fraction because that is what the "%" unit draws.
{`llmgw_node_cpu_utilization_percent{node=%s} / 100`, "node", "%",
func(c *nodeCharts, t TimeSeries) { c.CPU = t }},
{`llmgw_node_memory_used_bytes{node=%s}`, "node", "bytes",
func(c *nodeCharts, t TimeSeries) { c.Memory = t }},
{`llmgw_node_load{node=%s}`, "window", "",
func(c *nodeCharts, t TimeSeries) { c.Load = t }},
{`sum by (model) (llmgw_shard_cpu_cores{node=%s})`, "model", "",
func(c *nodeCharts, t TimeSeries) { c.EngineCPU = t }},
{`sum by (model) (llmgw_shard_memory_bytes{node=%s})`, "model", "bytes",
func(c *nodeCharts, t TimeSeries) { c.EngineMemory = t }},
}
// gpuPanels are the GPUs' charts, one line per card, for a node that has
// them. Read from the heartbeat's samples, not from what the node said when
// it registered.
var gpuPanels = []struct {
expr, unit string
into func(*nodeCharts, TimeSeries)
}{
{`llmgw_gpu_utilization_percent{node=%s} / 100`, "%",
func(c *nodeCharts, t TimeSeries) { c.GPUUtil = t }},
{`llmgw_gpu_memory_used_bytes{node=%s}`, "bytes",
func(c *nodeCharts, t TimeSeries) { c.GPUMemory = t }},
{`llmgw_gpu_temperature_celsius{node=%s}`, "",
func(c *nodeCharts, t TimeSeries) { c.GPUTemp = t }},
{`llmgw_gpu_power_watts{node=%s}`, "",
func(c *nodeCharts, t TimeSeries) { c.GPUPower = t }},
}
// NodePage shows one node. Its charts come from the metrics store when one
// is configured; the current figures come from the node's last heartbeat
// either way.
func NodePage(src Source, history History, window time.Duration, onAdminListener bool,
version string, log *slog.Logger,
) http.HandlerFunc {
if window <= 0 {
window = time.Hour
}
return func(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
d := nodePageData{
Nav: navFrom(r, "cluster"), OnAdminListener: onAdminListener,
Back: "/admin", Logs: "/admin/nodes/" + id + "/logs", Self: "/admin/nodes/" + id,
HasHistory: history != nil, Version: version, RefreshSec: 15, RefreshMeta: refreshMeta(15),
Now: time.Now().UTC().Format(time.RFC3339),
}
l := langOf(r)
if onAdminListener {
d.Back, d.Logs, d.Self = "/", "/nodes/"+id+"/logs", "/nodes/"+id
}
found := false
for _, n := range src.Nodes() {
if n.ID == id {
d.Node, found = n, true
d.Node.LastSeenAgo = l.clockText(n.LastSeenAgo)
break
}
}
// Only a node the control plane knows reaches a query below, which
// is what keeps the ID a name rather than something typed into PromQL.
if !found {
http.NotFound(w, r)
return
}
var replicas []string
for _, rep := range src.Replicas("") {
for _, sh := range rep.Shards {
if sh.Node != id {
continue
}
e := nodeEngine{Replica: rep.ID, Model: rep.Model, Shard: sh.ID, State: sh.State,
CPUCores: sh.CPUCores, MemoryBytes: sh.MemoryBytes}
if l := d.Node.Load; l != nil && l.CPUs > 0 {
e.CPUShare = sh.CPUCores * 100 / float64(l.CPUs)
}
d.Engines = append(d.Engines, e)
replicas = append(replicas, rep.ID)
}
}
sort.Slice(d.Engines, func(i, j int) bool { return d.Engines[i].CPUCores > d.Engines[j].CPUCores })
if history != nil {
d.Charts = loadNodeCharts(r.Context(), l, history, id, replicas, d.Node.GPUTotal, window, log)
}
noStore(w)
render(w, r, "node", d)
}
}
func loadNodeCharts(ctx context.Context, l *Lang, h History, node string, replicas []string, gpus int,
window time.Duration, log *slog.Logger,
) *nodeCharts {
step := window / 60
out := &nodeCharts{Window: l.windowWord(window)}
quoted := fmt.Sprintf("%q", node)
for _, p := range nodePanels {
series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), p.nameBy, window, step)
if err != nil {
if log != nil {
log.Warn("could not read a node's history", "node", node, "err", err)
}
out.Err = err.Error()
return out
}
p.into(out, l.LineChart(series, p.unit, 4))
}
// Every card its own line, up to the eight a node commonly has.
if gpus > 0 {
for _, p := range gpuPanels {
series, err := h.RangeQuery(ctx, fmt.Sprintf(p.expr, quoted), "gpu", window, step)
if err != nil {
continue
}
for i := range series {
series[i].Name = "GPU " + series[i].Name
}
p.into(out, l.LineChart(series, p.unit, max(gpus, 8)))
}
}
// The engines' queues, for the replicas that are here now.
if len(replicas) > 0 {
re := make([]string, len(replicas))
for i, r := range replicas {
re[i] = regexpQuote(r)
}
expr := fmt.Sprintf(`llmgw_endpoint_waiting{replica=~%q}`, strings.Join(re, "|"))
if series, err := h.RangeQuery(ctx, expr, "replica", window, step); err == nil {
out.Queue = l.LineChart(series, "", 4)
}
}
return out
}
// regexpQuote escapes what a replica ID could carry into a PromQL regex.
// IDs are letters, digits and dashes; this is for the day one is not.
func regexpQuote(s string) string {
var b strings.Builder
for _, r := range s {
if strings.ContainsRune(`\.+*?()|[]{}^$`, r) {
b.WriteRune('\\')
}
b.WriteRune(r)
}
return b.String()
}