The dashboard charted what the cluster had been billed for — requests, tokens, spend, all from llmgw's own store — and nothing about whether it was healthy. Saturation, queue depth, cache occupancy and replica counts were numbers in a table at the instant the page loaded, with no way to see that one of them had been climbing for an hour. Four line charts now, read back from wherever the metrics were scraped to. Read back rather than kept twice. llmgw publishes these already; what a gauge cannot do is remember, and a ring buffer in this process would be a rival copy of the same numbers with its own retention, lost on every restart and disagreeing with the one an operator has on screen elsewhere. monitoring.prometheus is empty by default and the panels are then absent rather than empty. One axis per chart, always. Two measures of different scale belong in two charts: a second y-axis lets any two lines be made to cross wherever the author likes and the reader cannot tell it happened. Four colours, handed out in fixed order, never cycled. A fifth series is summed into "other" and drawn in grey — dropping it would make a chart that does not add up, and inventing a hue for it would put two lines on the page that nobody can tell apart. The fold is by timestamp, not by position: the tail's series do not share sample times, and adding them by index slides points sideways onto each other's clock. A legend for every chart with more than one line, because colour alone is not an encoding a reader can rely on. A flat zero is drawn along the bottom rather than left blank: nothing happening is a real answer, and an empty frame reads as a broken chart. A single sample is not a line and says so instead of putting a dot in the corner. One failed query fails all four panels. Three drawn and one blank invites reading the blank one as "nothing is happening there". NaN is dropped rather than plotted. Prometheus sends it as a string for a sample it has no value for, Go's ParseFloat accepts it happily, and one NaN coordinate is an SVG path that does not draw at all — taking the rest of the line with it. Axis maxima are rounded to a number a person would pick. An axis topping out at 0.8347 makes every gridline unreadable, and the shape is what a chart is for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b921fa0d6b
commit
c6df0659af
14 files changed
+975
-1
No files matched your search
@@ -58,6 +58,9 @@ type DashboardData struct {
|
||||
// the other into a 404 — or, on the public port, into "no api key was
|
||||
// provided", which is a far more confusing way to say the same thing.
|
||||
OnAdminListener bool
|
||||
// Fleet is the operational charts, nil when no metrics store is configured.
|
||||
Fleet *FleetCharts
|
||||
|
||||
// Nav says which optional admin pages are mounted. Asked rather than
|
||||
// assumed: a link to a page that was never registered lands on whatever the
|
||||
// catch-all route serves, which on the client-facing port is the landing
|
||||
@@ -118,6 +121,11 @@ type UsageSource interface {
|
||||
type NavExtras struct {
|
||||
// Mirror is true when /admin/mirror is mounted.
|
||||
Mirror bool
|
||||
// History reads the cluster's past for the operational charts. Nil leaves
|
||||
// them off the page rather than drawing empty frames.
|
||||
History History
|
||||
// Window is how far back those charts look.
|
||||
Window time.Duration
|
||||
}
|
||||
|
||||
func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration,
|
||||
@@ -133,6 +141,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
d := DashboardData{
|
||||
OnAdminListener: onAdminListener,
|
||||
Nav: nav,
|
||||
Fleet: LoadFleetCharts(r.Context(), nav.History, nav.Window, log),
|
||||
RefreshSec: int(refresh.Seconds()),
|
||||
Version: version,
|
||||
Now: time.Now().UTC().Format(time.RFC3339),
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
// History reads the cluster's past out of a metrics store.
|
||||
type History interface {
|
||||
RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error)
|
||||
}
|
||||
|
||||
// FleetCharts are the operational panels: what the cluster is doing, rather
|
||||
// than what it has been billed for.
|
||||
//
|
||||
// The usage charts already on this page come from llmgw's own store and answer
|
||||
// a different question — who spent what. These answer whether the thing is
|
||||
// healthy, and they need history llmgw does not keep.
|
||||
type FleetCharts struct {
|
||||
Saturation TimeSeries
|
||||
Queue TimeSeries
|
||||
KV TimeSeries
|
||||
Ready TimeSeries
|
||||
Window string
|
||||
// Err is set when the metrics store could not be read. Shown as a line of
|
||||
// text where the charts would be: an empty panel reads as a healthy cluster
|
||||
// with nothing happening, which is the opposite of not knowing.
|
||||
Err string
|
||||
}
|
||||
|
||||
// fleetPanel is one chart and the query behind it.
|
||||
var fleetPanels = []struct {
|
||||
name string
|
||||
expr string
|
||||
nameBy string
|
||||
unit string
|
||||
into func(*FleetCharts, TimeSeries)
|
||||
}{
|
||||
{
|
||||
"saturation", "llmgw_pool_saturation", "model", "",
|
||||
func(f *FleetCharts, t TimeSeries) { f.Saturation = t },
|
||||
},
|
||||
{
|
||||
"queue depth", "llmgw_endpoint_waiting", "replica", "",
|
||||
func(f *FleetCharts, t TimeSeries) { f.Queue = t },
|
||||
},
|
||||
{
|
||||
"kv cache", "llmgw_endpoint_kv_cache_usage", "replica", "%",
|
||||
func(f *FleetCharts, t TimeSeries) { f.KV = t },
|
||||
},
|
||||
{
|
||||
"replicas ready", "llmgw_model_replicas_ready", "model", "",
|
||||
func(f *FleetCharts, t TimeSeries) { f.Ready = t },
|
||||
},
|
||||
}
|
||||
|
||||
// LoadFleetCharts queries each panel and renders it.
|
||||
//
|
||||
// One failure fails them all, deliberately: four panels where three are drawn
|
||||
// and one is blank invites reading the blank one as "nothing is happening
|
||||
// there". Either the metrics store answered or it did not.
|
||||
func LoadFleetCharts(ctx context.Context, h History, window time.Duration,
|
||||
log *slog.Logger,
|
||||
) *FleetCharts {
|
||||
if h == nil {
|
||||
return nil
|
||||
}
|
||||
if window <= 0 {
|
||||
window = time.Hour
|
||||
}
|
||||
// Sixty points across the window. More is detail nobody can see at this
|
||||
// size and a bigger response to parse on every page load.
|
||||
step := window / 60
|
||||
|
||||
out := &FleetCharts{Window: window.String()}
|
||||
for _, p := range fleetPanels {
|
||||
series, err := h.RangeQuery(ctx, p.expr, p.nameBy, window, step)
|
||||
if err != nil {
|
||||
if log != nil {
|
||||
log.Warn("could not read history for the dashboard",
|
||||
"panel", p.name, "err", err)
|
||||
}
|
||||
out.Err = err.Error()
|
||||
return out
|
||||
}
|
||||
p.into(out, LineChart(series, p.unit, 4))
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -282,3 +282,17 @@ tr:has(+ .url-row) td, tr:has(+ .error-row) td { border-bottom: none; }
|
||||
inside a table cell breaks the ruling of every column beside it. */
|
||||
tr.editing > td { background: color-mix(in srgb, var(--accent) 8%, var(--card)); }
|
||||
tr.editing form { margin: .4rem 0; }
|
||||
|
||||
/* Multi-line time series. Thin marks, recessive grid: the data is the darkest
|
||||
thing on the chart, which is the only rule that makes a chart readable. */
|
||||
.chart-lines { height: 150px; }
|
||||
.chart-lines .line { stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; }
|
||||
.chart-lines .axis { stroke: var(--viz-grid); stroke-width: 1; }
|
||||
.chart-lines .tick { fill: var(--muted); font-size: 9px; text-anchor: end; }
|
||||
/* A legend is always present for two lines or more: colour alone is not an
|
||||
encoding a reader can rely on. */
|
||||
.legend { display: flex; flex-wrap: wrap; gap: .1rem 1rem; margin: .2rem 0 1rem; font-size: .8rem; }
|
||||
.legend span { display: inline-flex; align-items: center; gap: .35rem; color: var(--muted); }
|
||||
.legend i { width: 10px; height: 2px; border-radius: 1px; display: inline-block; }
|
||||
.legend b { font-weight: 600; color: var(--fg); font-variant-numeric: tabular-nums; }
|
||||
.chart-title { font-size: .9rem; font-weight: 600; margin: 0 0 .3rem; }
|
||||
@@ -120,6 +120,28 @@ requested whenever a fallback served it.</p>
|
||||
{{else}}<p class="empty">No agents have registered.</p>{{end}}
|
||||
|
||||
<h2>Models</h2>
|
||||
{{with .Fleet}}
|
||||
<h2>What the cluster is doing <span class="opt">last {{.Window}}</span></h2>
|
||||
{{if .Err}}
|
||||
{{/* Said outright. An empty panel reads as a healthy cluster with nothing
|
||||
happening, which is the opposite of not knowing. */}}
|
||||
<p class="note error-row">No history: {{.Err}}</p>
|
||||
{{else}}
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Saturation "title" "Pool saturation, by model"
|
||||
"note" "The worse of queue depth and cache pressure. 1 is the limit measured as healthy, not the point of failure."}}
|
||||
{{template "tschart" dict "t" .Ready "title" "Replicas ready, by model"
|
||||
"note" "A model deliberately stopped has no line at all, which is why an on-demand catalogue looks quiet here."}}
|
||||
</div>
|
||||
<div class="chart-row">
|
||||
{{template "tschart" dict "t" .Queue "title" "Engine queue depth, by replica"
|
||||
"note" "Requests accepted and not yet started. The first thing to move when a replica is falling behind."}}
|
||||
{{template "tschart" dict "t" .KV "title" "KV cache occupancy, by replica"
|
||||
"note" "How full the attention cache is. Long contexts fill it with an empty queue, which is why saturation reads both."}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if .Models}}
|
||||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||||
{{range .Models}}<tr>
|
||||
@@ -205,3 +227,25 @@ exposed:
|
||||
<span>{{.Now}}</span>
|
||||
</footer>
|
||||
</div></body></html>{{end}}
|
||||
|
||||
{{/* One chart: the SVG, its legend, and the sentence that says how to read it.
|
||||
A legend is always present for two lines or more — colour alone is not an
|
||||
encoding a reader can rely on — and a single line needs none, because the
|
||||
title already names it. */}}
|
||||
{{define "tschart"}}
|
||||
<div class="chart-wrap">
|
||||
<h3 class="chart-title">{{.title}}</h3>
|
||||
{{if .t.Empty}}
|
||||
<p class="empty">Nothing recorded yet.</p>
|
||||
{{else}}
|
||||
{{.t.SVG}}
|
||||
<span class="chart-peak">peak {{.t.Peak}}</span>
|
||||
{{if gt (len .t.Legend) 1}}
|
||||
<div class="legend">
|
||||
{{range .t.Legend}}<span><i style="background:var({{cssvar .Colour}})"></i>{{.Name}} <b>{{.Last}}</b></span>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
<p class="note">{{.note}}</p>
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -0,0 +1,232 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
// vizColours is the categorical palette, in the order it is handed out.
|
||||
//
|
||||
// Fixed order, never cycled. A fifth line does not get a generated hue — it is
|
||||
// folded into "other" by foldSeries below, because a chart whose colours repeat
|
||||
// is one where two lines mean the same thing and neither can be read.
|
||||
var vizColours = []string{"--viz-1", "--viz-2", "--viz-3", "--viz-4"}
|
||||
|
||||
// otherColour is the muted grey the tail is drawn in, so it reads as a total
|
||||
// rather than as a fifth thing with a name.
|
||||
const otherColour = "--muted"
|
||||
|
||||
const (
|
||||
tsW = 720.0
|
||||
tsH = 150.0
|
||||
tsPadL = 44.0
|
||||
tsPadB = 18.0
|
||||
tsPadT = 8.0
|
||||
tsGridRows = 3
|
||||
)
|
||||
|
||||
// TimeSeries is a chart of several lines over one clock.
|
||||
type TimeSeries struct {
|
||||
SVG template.HTML
|
||||
Legend []LegendEntry
|
||||
// Empty is true when there was nothing to draw, so the page can say why
|
||||
// rather than showing a frame with no lines in it.
|
||||
Empty bool
|
||||
Peak string
|
||||
}
|
||||
|
||||
// LegendEntry names one line and the colour it was drawn in.
|
||||
//
|
||||
// A legend is always present for two lines or more: colour alone is not an
|
||||
// encoding anybody can rely on, and a reader who cannot tell two lines apart
|
||||
// has a picture rather than a chart.
|
||||
type LegendEntry struct {
|
||||
Name string
|
||||
Colour string
|
||||
Last string
|
||||
}
|
||||
|
||||
// LineChart draws several series against one axis.
|
||||
//
|
||||
// One axis, always. Two measures of different scale belong in two charts: a
|
||||
// second y-axis lets any two lines be made to cross wherever the author likes,
|
||||
// and the reader has no way to know it happened.
|
||||
func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries {
|
||||
if maxSeries <= 0 || maxSeries > len(vizColours) {
|
||||
maxSeries = len(vizColours)
|
||||
}
|
||||
series = foldSeries(series, maxSeries)
|
||||
if len(series) == 0 {
|
||||
return TimeSeries{Empty: true}
|
||||
}
|
||||
|
||||
minT, maxT, maxV := bounds(series)
|
||||
if maxT.Equal(minT) {
|
||||
// One instant is not a line. Reported as empty rather than drawn as a
|
||||
// dot in the corner, which reads as a broken chart.
|
||||
return TimeSeries{Empty: true}
|
||||
}
|
||||
// A flat zero is a real answer — nothing is happening — and drawing it
|
||||
// along the bottom says so, where an autoscaled empty frame would not.
|
||||
if maxV <= 0 {
|
||||
maxV = 1
|
||||
}
|
||||
maxV = niceCeiling(maxV)
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `<svg class="chart chart-lines" viewBox="0 0 %.0f %.0f" role="img">`, tsW, tsH)
|
||||
writeGrid(&b, maxV, unit)
|
||||
|
||||
out := TimeSeries{Peak: formatValue(maxV, unit)}
|
||||
span := maxT.Sub(minT).Seconds()
|
||||
for i, s := range series {
|
||||
colour := otherColour
|
||||
if s.Name != otherLabel {
|
||||
colour = vizColours[i%len(vizColours)]
|
||||
}
|
||||
writeLine(&b, s, minT, span, maxV, colour)
|
||||
out.Legend = append(out.Legend, LegendEntry{
|
||||
Name: s.Name, Colour: colour, Last: formatValue(s.Last(), unit),
|
||||
})
|
||||
}
|
||||
b.WriteString(`</svg>`)
|
||||
out.SVG = template.HTML(b.String()) //nolint:gosec // numbers and CSS vars only
|
||||
return out
|
||||
}
|
||||
|
||||
const otherLabel = "other"
|
||||
|
||||
// foldSeries keeps the busiest lines and sums the rest into one.
|
||||
//
|
||||
// Rather than dropping the tail, which would make a chart that does not add up,
|
||||
// or colouring it, which needs hues the palette does not have. The fold is
|
||||
// labelled so nobody reads it as a single model.
|
||||
func foldSeries(in []promq.Series, keep int) []promq.Series {
|
||||
clean := make([]promq.Series, 0, len(in))
|
||||
for _, s := range in {
|
||||
if len(s.Points) > 0 {
|
||||
clean = append(clean, s)
|
||||
}
|
||||
}
|
||||
if len(clean) <= keep {
|
||||
return clean
|
||||
}
|
||||
sort.SliceStable(clean, func(i, j int) bool { return clean[i].Max() > clean[j].Max() })
|
||||
|
||||
rest := clean[keep-1:]
|
||||
folded := promq.Series{Name: otherLabel}
|
||||
// Summed by timestamp, because the tail's lines do not necessarily share
|
||||
// sample times and adding them by index would shift points sideways.
|
||||
at := map[int64]float64{}
|
||||
for _, s := range rest {
|
||||
for _, p := range s.Points {
|
||||
at[p.At.Unix()] += p.Value
|
||||
}
|
||||
}
|
||||
keys := make([]int64, 0, len(at))
|
||||
for k := range at {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
|
||||
for _, k := range keys {
|
||||
folded.Points = append(folded.Points, promq.Point{At: time.Unix(k, 0), Value: at[k]})
|
||||
}
|
||||
return append(append([]promq.Series{}, clean[:keep-1]...), folded)
|
||||
}
|
||||
|
||||
func bounds(series []promq.Series) (minT, maxT time.Time, maxV float64) {
|
||||
for _, s := range series {
|
||||
for _, p := range s.Points {
|
||||
if minT.IsZero() || p.At.Before(minT) {
|
||||
minT = p.At
|
||||
}
|
||||
if p.At.After(maxT) {
|
||||
maxT = p.At
|
||||
}
|
||||
if p.Value > maxV {
|
||||
maxV = p.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
return minT, maxT, maxV
|
||||
}
|
||||
|
||||
// writeGrid draws the horizontal rules and their labels.
|
||||
//
|
||||
// Recessive on purpose: the grid is there to be read off, not looked at, and a
|
||||
// grid as dark as the data competes with it.
|
||||
func writeGrid(b *strings.Builder, maxV float64, unit string) {
|
||||
plotH := tsH - tsPadB - tsPadT
|
||||
for i := 0; i <= tsGridRows; i++ {
|
||||
y := tsPadT + plotH*float64(i)/tsGridRows
|
||||
v := maxV * float64(tsGridRows-i) / tsGridRows
|
||||
fmt.Fprintf(b, `<line x1="%.0f" y1="%.1f" x2="%.0f" y2="%.1f" class="axis"/>`,
|
||||
tsPadL, y, tsW, y)
|
||||
fmt.Fprintf(b, `<text x="%.0f" y="%.1f" class="tick">%s</text>`,
|
||||
tsPadL-6, y+3, escapeText(formatValue(v, unit)))
|
||||
}
|
||||
}
|
||||
|
||||
func writeLine(b *strings.Builder, s promq.Series, minT time.Time,
|
||||
span, maxV float64, colour string,
|
||||
) {
|
||||
plotW := tsW - tsPadL
|
||||
plotH := tsH - tsPadB - tsPadT
|
||||
|
||||
fmt.Fprintf(b, `<polyline class="line" fill="none" stroke="var(%s)" points="`, cssVar(colour))
|
||||
for _, p := range s.Points {
|
||||
x := tsPadL + plotW*(p.At.Sub(minT).Seconds()/span)
|
||||
y := tsPadT + plotH*(1-clamp01(p.Value/maxV))
|
||||
fmt.Fprintf(b, "%.1f,%.1f ", x, y)
|
||||
}
|
||||
b.WriteString(`"/>`)
|
||||
}
|
||||
|
||||
func clamp01(v float64) float64 {
|
||||
if math.IsNaN(v) || v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// niceCeiling rounds an axis maximum up to something a person would choose.
|
||||
//
|
||||
// An axis topping out at 0.8347 makes every gridline a number nobody can hold
|
||||
// in their head, and the shape of the data is what the chart is for.
|
||||
func niceCeiling(v float64) float64 {
|
||||
if v <= 0 {
|
||||
return 1
|
||||
}
|
||||
mag := math.Pow(10, math.Floor(math.Log10(v)))
|
||||
for _, step := range []float64{1, 2, 2.5, 5, 10} {
|
||||
if v <= step*mag {
|
||||
return step * mag
|
||||
}
|
||||
}
|
||||
return 10 * mag
|
||||
}
|
||||
|
||||
func formatValue(v float64, unit string) string {
|
||||
switch unit {
|
||||
case "%":
|
||||
return fmt.Sprintf("%.0f%%", v*100)
|
||||
case "s":
|
||||
if v < 1 {
|
||||
return fmt.Sprintf("%.0fms", v*1000)
|
||||
}
|
||||
return fmt.Sprintf("%.1fs", v)
|
||||
case "bytes":
|
||||
return compact(v) + "B"
|
||||
default:
|
||||
return compact(v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
||||
)
|
||||
|
||||
func ts(name string, vals ...float64) promq.Series {
|
||||
s := promq.Series{Name: name}
|
||||
base := time.Unix(1700000000, 0)
|
||||
for i, v := range vals {
|
||||
s.Points = append(s.Points, promq.Point{At: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestEveryLineGetsItsOwnColourAndALegend(t *testing.T) {
|
||||
c := LineChart([]promq.Series{ts("a", 1, 2), ts("b", 2, 1), ts("c", 3, 3)}, "", 4)
|
||||
|
||||
if len(c.Legend) != 3 {
|
||||
t.Fatalf("expected a legend entry per line, got %+v", c.Legend)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, e := range c.Legend {
|
||||
if seen[e.Colour] {
|
||||
// Two lines in one colour is two lines nobody can tell apart.
|
||||
t.Errorf("colour %s used twice", e.Colour)
|
||||
}
|
||||
seen[e.Colour] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestATailIsFoldedRatherThanGivenNewColours(t *testing.T) {
|
||||
in := []promq.Series{
|
||||
ts("big", 10, 10), ts("m2", 5, 5), ts("m3", 4, 4),
|
||||
ts("m4", 1, 1), ts("m5", 2, 2), ts("m6", 3, 3),
|
||||
}
|
||||
c := LineChart(in, "", 4)
|
||||
|
||||
// The palette has four colours and a fifth line does not get a generated
|
||||
// hue. The tail is summed so the chart still adds up.
|
||||
if len(c.Legend) != 4 {
|
||||
t.Fatalf("expected four lines, got %d: %+v", len(c.Legend), c.Legend)
|
||||
}
|
||||
last := c.Legend[3]
|
||||
if last.Name != "other" {
|
||||
t.Errorf("the tail should be named, got %q", last.Name)
|
||||
}
|
||||
// m4+m5+m6 at the same instant is 6, not any one of them.
|
||||
if last.Last != "6" {
|
||||
t.Errorf("the tail should be summed, got %q", last.Last)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTailIsSummedByTimestampNotByPosition(t *testing.T) {
|
||||
// Two tail series sampled at different times. Added by index their points
|
||||
// would slide sideways onto each other's clock.
|
||||
a := promq.Series{Name: "a", Points: []promq.Point{{At: time.Unix(100, 0), Value: 1}}}
|
||||
b := promq.Series{Name: "b", Points: []promq.Point{{At: time.Unix(200, 0), Value: 2}}}
|
||||
big := ts("big", 9, 9, 9)
|
||||
|
||||
c := LineChart([]promq.Series{big, ts("x", 8, 8), ts("y", 7, 7), a, b}, "", 4)
|
||||
if c.Legend[3].Name != "other" || c.Legend[3].Last != "2" {
|
||||
t.Errorf("expected the latest tail point alone, got %+v", c.Legend[3])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFlatZeroIsDrawnRatherThanLeftBlank(t *testing.T) {
|
||||
c := LineChart([]promq.Series{ts("idle", 0, 0, 0)}, "", 4)
|
||||
|
||||
// Nothing happening is a real answer, and a line along the bottom says so
|
||||
// where an empty frame reads as a broken chart.
|
||||
if c.Empty {
|
||||
t.Fatal("a quiet series is not an absent one")
|
||||
}
|
||||
if !strings.Contains(string(c.SVG), "polyline") {
|
||||
t.Error("the line should be drawn")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingToDrawSaysSoInsteadOfDrawingAFrame(t *testing.T) {
|
||||
if got := LineChart(nil, "", 4); !got.Empty {
|
||||
t.Error("no series is empty")
|
||||
}
|
||||
// One instant is not a line: drawn, it is a dot in the corner.
|
||||
one := promq.Series{Name: "a", Points: []promq.Point{{At: time.Unix(1, 0), Value: 5}}}
|
||||
if got := LineChart([]promq.Series{one}, "", 4); !got.Empty {
|
||||
t.Error("a single sample is not a line")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAxisTopsOutAtANumberAPersonWouldChoose(t *testing.T) {
|
||||
c := LineChart([]promq.Series{ts("a", 0.1, 0.8347)}, "", 4)
|
||||
// 0.8347 as the top makes every gridline unreadable, and the shape of the
|
||||
// data is what the chart is for.
|
||||
if c.Peak != "1" {
|
||||
t.Errorf("expected a rounded ceiling, got %q", c.Peak)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPercentagesAndSecondsReadAsThemselves(t *testing.T) {
|
||||
if got := formatValue(0.42, "%"); got != "42%" {
|
||||
t.Errorf("percent: %q", got)
|
||||
}
|
||||
if got := formatValue(0.035, "s"); got != "35ms" {
|
||||
t.Errorf("sub-second latency should read in ms, got %q", got)
|
||||
}
|
||||
if got := formatValue(12.5, "s"); got != "12.5s" {
|
||||
t.Errorf("seconds: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAValueAboveTheCeilingIsClampedNotDrawnOffTheChart(t *testing.T) {
|
||||
// Only reachable if a series changes between the bounds pass and drawing,
|
||||
// but an SVG point outside the viewBox is a line that leaves the frame.
|
||||
if got := clamp01(1.4); got != 1 {
|
||||
t.Errorf("expected clamping, got %v", got)
|
||||
}
|
||||
if got := clamp01(-2); got != 0 {
|
||||
t.Errorf("expected clamping, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeHistory stands in for a metrics store.
|
||||
type fakeHistory struct {
|
||||
byExpr map[string][]promq.Series
|
||||
err error
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (f *fakeHistory) RangeQuery(_ context.Context, expr, _ string,
|
||||
_, _ time.Duration,
|
||||
) ([]promq.Series, error) {
|
||||
f.asked = append(f.asked, expr)
|
||||
return f.byExpr[expr], f.err
|
||||
}
|
||||
|
||||
func TestTheFleetPanelsAreEachQueried(t *testing.T) {
|
||||
h := &fakeHistory{byExpr: map[string][]promq.Series{
|
||||
"llmgw_pool_saturation": {ts("qwen", 0.1, 0.4)},
|
||||
"llmgw_endpoint_waiting": {ts("r1", 0, 2)},
|
||||
"llmgw_endpoint_kv_cache_usage": {ts("r1", 0.1, 0.2)},
|
||||
"llmgw_model_replicas_ready": {ts("qwen", 3, 3)},
|
||||
}}
|
||||
got := LoadFleetCharts(context.Background(), h, time.Hour, nil)
|
||||
|
||||
if len(h.asked) != 4 {
|
||||
t.Fatalf("expected one query per panel, got %v", h.asked)
|
||||
}
|
||||
for name, c := range map[string]TimeSeries{
|
||||
"saturation": got.Saturation, "queue": got.Queue,
|
||||
"kv": got.KV, "ready": got.Ready,
|
||||
} {
|
||||
if c.Empty {
|
||||
t.Errorf("%s came back empty", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneFailedQueryFailsThemAll(t *testing.T) {
|
||||
h := &fakeHistory{err: errors.New("reaching Prometheus: refused")}
|
||||
got := LoadFleetCharts(context.Background(), h, time.Hour, nil)
|
||||
|
||||
// Three charts drawn and one blank invites reading the blank one as
|
||||
// "nothing is happening there". Either the store answered or it did not.
|
||||
if got.Err == "" {
|
||||
t.Fatal("the failure should be reported")
|
||||
}
|
||||
if !strings.Contains(got.Err, "refused") {
|
||||
t.Errorf("the reason should survive: %q", got.Err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNoMetricsStoreThereAreNoPanels(t *testing.T) {
|
||||
// A deployment with no Prometheus leaves them off rather than showing
|
||||
// empty frames where charts should be.
|
||||
if got := LoadFleetCharts(context.Background(), nil, time.Hour, nil); got != nil {
|
||||
t.Errorf("expected nothing, got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,21 @@ var funcs = template.FuncMap{
|
||||
}
|
||||
return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
|
||||
},
|
||||
// dict lets one chart template be called with named arguments. Go
|
||||
// templates pass a single value, and four charts differing in three
|
||||
// strings each is four copies of the same markup otherwise.
|
||||
"dict": func(kv ...any) map[string]any {
|
||||
m := map[string]any{}
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
k, _ := kv[i].(string)
|
||||
m[k] = kv[i+1]
|
||||
}
|
||||
return m
|
||||
},
|
||||
// cssvar turns a palette name into the custom property it refers to, so a
|
||||
// legend swatch follows the page's light and dark themes like the line it
|
||||
// stands for.
|
||||
"cssvar": cssVar,
|
||||
"date": func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "—"
|
||||
|
||||
Reference in new issue
Block a user