The dashboard charted what the cluster had been billed for — requests, tokens, spend, all from llmgw's own store — and nothing about whether it was healthy. Saturation, queue depth, cache occupancy and replica counts were numbers in a table at the instant the page loaded, with no way to see that one of them had been climbing for an hour. Four line charts now, read back from wherever the metrics were scraped to. Read back rather than kept twice. llmgw publishes these already; what a gauge cannot do is remember, and a ring buffer in this process would be a rival copy of the same numbers with its own retention, lost on every restart and disagreeing with the one an operator has on screen elsewhere. monitoring.prometheus is empty by default and the panels are then absent rather than empty. One axis per chart, always. Two measures of different scale belong in two charts: a second y-axis lets any two lines be made to cross wherever the author likes and the reader cannot tell it happened. Four colours, handed out in fixed order, never cycled. A fifth series is summed into "other" and drawn in grey — dropping it would make a chart that does not add up, and inventing a hue for it would put two lines on the page that nobody can tell apart. The fold is by timestamp, not by position: the tail's series do not share sample times, and adding them by index slides points sideways onto each other's clock. A legend for every chart with more than one line, because colour alone is not an encoding a reader can rely on. A flat zero is drawn along the bottom rather than left blank: nothing happening is a real answer, and an empty frame reads as a broken chart. A single sample is not a line and says so instead of putting a dot in the corner. One failed query fails all four panels. Three drawn and one blank invites reading the blank one as "nothing is happening there". NaN is dropped rather than plotted. Prometheus sends it as a string for a sample it has no value for, Go's ParseFloat accepts it happily, and one NaN coordinate is an SVG path that does not draw at all — taking the rest of the line with it. Axis maxima are rounded to a number a person would pick. An axis topping out at 0.8347 makes every gridline unreadable, and the shape is what a chart is for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
186 lines
5.8 KiB
Go
186 lines
5.8 KiB
Go
package web
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
|
)
|
|
|
|
func ts(name string, vals ...float64) promq.Series {
|
|
s := promq.Series{Name: name}
|
|
base := time.Unix(1700000000, 0)
|
|
for i, v := range vals {
|
|
s.Points = append(s.Points, promq.Point{At: base.Add(time.Duration(i) * time.Minute), Value: v})
|
|
}
|
|
return s
|
|
}
|
|
|
|
func TestEveryLineGetsItsOwnColourAndALegend(t *testing.T) {
|
|
c := LineChart([]promq.Series{ts("a", 1, 2), ts("b", 2, 1), ts("c", 3, 3)}, "", 4)
|
|
|
|
if len(c.Legend) != 3 {
|
|
t.Fatalf("expected a legend entry per line, got %+v", c.Legend)
|
|
}
|
|
seen := map[string]bool{}
|
|
for _, e := range c.Legend {
|
|
if seen[e.Colour] {
|
|
// Two lines in one colour is two lines nobody can tell apart.
|
|
t.Errorf("colour %s used twice", e.Colour)
|
|
}
|
|
seen[e.Colour] = true
|
|
}
|
|
}
|
|
|
|
func TestATailIsFoldedRatherThanGivenNewColours(t *testing.T) {
|
|
in := []promq.Series{
|
|
ts("big", 10, 10), ts("m2", 5, 5), ts("m3", 4, 4),
|
|
ts("m4", 1, 1), ts("m5", 2, 2), ts("m6", 3, 3),
|
|
}
|
|
c := LineChart(in, "", 4)
|
|
|
|
// The palette has four colours and a fifth line does not get a generated
|
|
// hue. The tail is summed so the chart still adds up.
|
|
if len(c.Legend) != 4 {
|
|
t.Fatalf("expected four lines, got %d: %+v", len(c.Legend), c.Legend)
|
|
}
|
|
last := c.Legend[3]
|
|
if last.Name != "other" {
|
|
t.Errorf("the tail should be named, got %q", last.Name)
|
|
}
|
|
// m4+m5+m6 at the same instant is 6, not any one of them.
|
|
if last.Last != "6" {
|
|
t.Errorf("the tail should be summed, got %q", last.Last)
|
|
}
|
|
}
|
|
|
|
func TestTheTailIsSummedByTimestampNotByPosition(t *testing.T) {
|
|
// Two tail series sampled at different times. Added by index their points
|
|
// would slide sideways onto each other's clock.
|
|
a := promq.Series{Name: "a", Points: []promq.Point{{At: time.Unix(100, 0), Value: 1}}}
|
|
b := promq.Series{Name: "b", Points: []promq.Point{{At: time.Unix(200, 0), Value: 2}}}
|
|
big := ts("big", 9, 9, 9)
|
|
|
|
c := LineChart([]promq.Series{big, ts("x", 8, 8), ts("y", 7, 7), a, b}, "", 4)
|
|
if c.Legend[3].Name != "other" || c.Legend[3].Last != "2" {
|
|
t.Errorf("expected the latest tail point alone, got %+v", c.Legend[3])
|
|
}
|
|
}
|
|
|
|
func TestAFlatZeroIsDrawnRatherThanLeftBlank(t *testing.T) {
|
|
c := LineChart([]promq.Series{ts("idle", 0, 0, 0)}, "", 4)
|
|
|
|
// Nothing happening is a real answer, and a line along the bottom says so
|
|
// where an empty frame reads as a broken chart.
|
|
if c.Empty {
|
|
t.Fatal("a quiet series is not an absent one")
|
|
}
|
|
if !strings.Contains(string(c.SVG), "polyline") {
|
|
t.Error("the line should be drawn")
|
|
}
|
|
}
|
|
|
|
func TestNothingToDrawSaysSoInsteadOfDrawingAFrame(t *testing.T) {
|
|
if got := LineChart(nil, "", 4); !got.Empty {
|
|
t.Error("no series is empty")
|
|
}
|
|
// One instant is not a line: drawn, it is a dot in the corner.
|
|
one := promq.Series{Name: "a", Points: []promq.Point{{At: time.Unix(1, 0), Value: 5}}}
|
|
if got := LineChart([]promq.Series{one}, "", 4); !got.Empty {
|
|
t.Error("a single sample is not a line")
|
|
}
|
|
}
|
|
|
|
func TestTheAxisTopsOutAtANumberAPersonWouldChoose(t *testing.T) {
|
|
c := LineChart([]promq.Series{ts("a", 0.1, 0.8347)}, "", 4)
|
|
// 0.8347 as the top makes every gridline unreadable, and the shape of the
|
|
// data is what the chart is for.
|
|
if c.Peak != "1" {
|
|
t.Errorf("expected a rounded ceiling, got %q", c.Peak)
|
|
}
|
|
}
|
|
|
|
func TestPercentagesAndSecondsReadAsThemselves(t *testing.T) {
|
|
if got := formatValue(0.42, "%"); got != "42%" {
|
|
t.Errorf("percent: %q", got)
|
|
}
|
|
if got := formatValue(0.035, "s"); got != "35ms" {
|
|
t.Errorf("sub-second latency should read in ms, got %q", got)
|
|
}
|
|
if got := formatValue(12.5, "s"); got != "12.5s" {
|
|
t.Errorf("seconds: %q", got)
|
|
}
|
|
}
|
|
|
|
func TestAValueAboveTheCeilingIsClampedNotDrawnOffTheChart(t *testing.T) {
|
|
// Only reachable if a series changes between the bounds pass and drawing,
|
|
// but an SVG point outside the viewBox is a line that leaves the frame.
|
|
if got := clamp01(1.4); got != 1 {
|
|
t.Errorf("expected clamping, got %v", got)
|
|
}
|
|
if got := clamp01(-2); got != 0 {
|
|
t.Errorf("expected clamping, got %v", got)
|
|
}
|
|
}
|
|
|
|
// fakeHistory stands in for a metrics store.
|
|
type fakeHistory struct {
|
|
byExpr map[string][]promq.Series
|
|
err error
|
|
asked []string
|
|
}
|
|
|
|
func (f *fakeHistory) RangeQuery(_ context.Context, expr, _ string,
|
|
_, _ time.Duration,
|
|
) ([]promq.Series, error) {
|
|
f.asked = append(f.asked, expr)
|
|
return f.byExpr[expr], f.err
|
|
}
|
|
|
|
func TestTheFleetPanelsAreEachQueried(t *testing.T) {
|
|
h := &fakeHistory{byExpr: map[string][]promq.Series{
|
|
"llmgw_pool_saturation": {ts("qwen", 0.1, 0.4)},
|
|
"llmgw_endpoint_waiting": {ts("r1", 0, 2)},
|
|
"llmgw_endpoint_kv_cache_usage": {ts("r1", 0.1, 0.2)},
|
|
"llmgw_model_replicas_ready": {ts("qwen", 3, 3)},
|
|
}}
|
|
got := LoadFleetCharts(context.Background(), h, time.Hour, nil)
|
|
|
|
if len(h.asked) != 4 {
|
|
t.Fatalf("expected one query per panel, got %v", h.asked)
|
|
}
|
|
for name, c := range map[string]TimeSeries{
|
|
"saturation": got.Saturation, "queue": got.Queue,
|
|
"kv": got.KV, "ready": got.Ready,
|
|
} {
|
|
if c.Empty {
|
|
t.Errorf("%s came back empty", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOneFailedQueryFailsThemAll(t *testing.T) {
|
|
h := &fakeHistory{err: errors.New("reaching Prometheus: refused")}
|
|
got := LoadFleetCharts(context.Background(), h, time.Hour, nil)
|
|
|
|
// Three charts drawn and one blank invites reading the blank one as
|
|
// "nothing is happening there". Either the store answered or it did not.
|
|
if got.Err == "" {
|
|
t.Fatal("the failure should be reported")
|
|
}
|
|
if !strings.Contains(got.Err, "refused") {
|
|
t.Errorf("the reason should survive: %q", got.Err)
|
|
}
|
|
}
|
|
|
|
func TestWithNoMetricsStoreThereAreNoPanels(t *testing.T) {
|
|
// A deployment with no Prometheus leaves them off rather than showing
|
|
// empty frames where charts should be.
|
|
if got := LoadFleetCharts(context.Background(), nil, time.Hour, nil); got != nil {
|
|
t.Errorf("expected nothing, got %+v", got)
|
|
}
|
|
}
|