package web import ( "context" "log/slog" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" ) // History reads the cluster's past out of a metrics store. type History interface { RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error) } // FleetCharts are the operational panels: what the cluster is doing, rather // than what it has been billed for. // // The usage charts already on this page come from llmgw's own store and answer // a different question — who spent what. These answer whether the thing is // healthy, and they need history llmgw does not keep. type FleetCharts struct { Saturation TimeSeries Queue TimeSeries KV TimeSeries Ready TimeSeries // GPUBusy and GPUMemory are the fleet's cards by class, for a fleet that // has them. GPUBusy, GPUMemory TimeSeries Window string // Err is set when the metrics store could not be read. Shown as a line of // text where the charts would be: an empty panel reads as a healthy cluster // with nothing happening, which is the opposite of not knowing. The // error is the metrics store's own, in English. Err string // lang is the page's language, which the charts added later are drawn in. lang *Lang } // fleetPanel is one chart and the query behind it. var fleetPanels = []struct { name string expr string nameBy string unit string into func(*FleetCharts, TimeSeries) }{ { "saturation", "llmgw_pool_saturation", "model", "", func(f *FleetCharts, t TimeSeries) { f.Saturation = t }, }, { "queue depth", "llmgw_endpoint_waiting", "replica", "", func(f *FleetCharts, t TimeSeries) { f.Queue = t }, }, { "kv cache", "llmgw_endpoint_kv_cache_usage", "replica", "%", func(f *FleetCharts, t TimeSeries) { f.KV = t }, }, { "replicas ready", "llmgw_model_replicas_ready", "model", "", func(f *FleetCharts, t TimeSeries) { f.Ready = t }, }, } // LoadFleetCharts queries each panel and renders it. // // One failure fails them all, deliberately: four panels where three are drawn // and one is blank invites reading the blank one as "nothing is happening // there". Either the metrics store answered or it did not. func LoadFleetCharts(ctx context.Context, h History, window time.Duration, log *slog.Logger, ) *FleetCharts { return english.loadFleetCharts(ctx, h, window, log) } // loadFleetCharts is LoadFleetCharts drawn in this language. func (l *Lang) loadFleetCharts(ctx context.Context, h History, window time.Duration, log *slog.Logger, ) *FleetCharts { if h == nil { return nil } if window <= 0 { window = time.Hour } // Sixty points across the window. More is detail nobody can see at this // size and a bigger response to parse on every page load. step := window / 60 out := &FleetCharts{Window: l.windowWord(window), lang: l} for _, p := range fleetPanels { series, err := h.RangeQuery(ctx, p.expr, p.nameBy, window, step) if err != nil { if log != nil { log.Warn("could not read history for the dashboard", "panel", p.name, "err", err) } out.Err = err.Error() return out } p.into(out, l.LineChart(series, p.unit, 4)) } return out } // windowWord names a chart window the way a sentence in this language // would: "last hour", "son 6 saat". A window that is not whole minutes is // written as Go writes it, which is rare enough not to be worth words. func (l *Lang) windowWord(d time.Duration) string { switch { case d == time.Hour: return l.T("chart.window.hour") case d%time.Hour == 0: return l.N("chart.window.hours", int64(d.Hours())) case d%time.Minute == 0: return l.N("chart.window.minutes", int64(d.Minutes())) } return d.String() } // LoadGPU adds the fleet's GPUs, by class: how busy on average, and how full // their memory. Asked only of a fleet with GPUs. A failure leaves these two // empty and the rest of the page as it was; they are not what the other // panels' all-or-nothing is about. func (f *FleetCharts) LoadGPU(ctx context.Context, h History, window time.Duration) { if f == nil || h == nil || f.Err != "" { return } if window <= 0 { window = time.Hour } step := window / 60 l := f.lang if l == nil { l = english } if s, err := h.RangeQuery(ctx, `avg by (class) (llmgw_gpu_utilization_percent) / 100`, "class", window, step); err == nil { f.GPUBusy = l.LineChart(s, "%", 6) } if s, err := h.RangeQuery(ctx, `sum by (class) (llmgw_gpu_memory_used_bytes) / sum by (class) (llmgw_gpu_memory_total_bytes)`, "class", window, step); err == nil { f.GPUMemory = l.LineChart(s, "%", 6) } }