Files
llmgw/internal/promq/promq.go
T
Ilker ManapandClaude Opus 5 c6df0659af Put the cluster's own charts on its own admin page
The dashboard charted what the cluster had been billed for — requests, tokens,
spend, all from llmgw's own store — and nothing about whether it was healthy.
Saturation, queue depth, cache occupancy and replica counts were numbers in a
table at the instant the page loaded, with no way to see that one of them had
been climbing for an hour.

Four line charts now, read back from wherever the metrics were scraped to.

Read back rather than kept twice. llmgw publishes these already; what a gauge
cannot do is remember, and a ring buffer in this process would be a rival copy
of the same numbers with its own retention, lost on every restart and
disagreeing with the one an operator has on screen elsewhere. monitoring.prometheus
is empty by default and the panels are then absent rather than empty.

One axis per chart, always. Two measures of different scale belong in two
charts: a second y-axis lets any two lines be made to cross wherever the author
likes and the reader cannot tell it happened.

Four colours, handed out in fixed order, never cycled. A fifth series is summed
into "other" and drawn in grey — dropping it would make a chart that does not
add up, and inventing a hue for it would put two lines on the page that nobody
can tell apart. The fold is by timestamp, not by position: the tail's series do
not share sample times, and adding them by index slides points sideways onto
each other's clock.

A legend for every chart with more than one line, because colour alone is not
an encoding a reader can rely on.

A flat zero is drawn along the bottom rather than left blank: nothing happening
is a real answer, and an empty frame reads as a broken chart. A single sample
is not a line and says so instead of putting a dot in the corner.

One failed query fails all four panels. Three drawn and one blank invites
reading the blank one as "nothing is happening there".

NaN is dropped rather than plotted. Prometheus sends it as a string for a
sample it has no value for, Go's ParseFloat accepts it happily, and one NaN
coordinate is an SVG path that does not draw at all — taking the rest of the
line with it.

Axis maxima are rounded to a number a person would pick. An axis topping out at
0.8347 makes every gridline unreadable, and the shape is what a chart is for.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-27 18:25:56 +02:00

208 lines
5.5 KiB
Go

// Package promq reads time series back out of Prometheus.
//
// Only enough to draw a chart: a range query, and the labels needed to name
// each line. llmgw already publishes these metrics; what it has never had is
// the history, because a gauge knows the current value and nothing about the
// ten minutes before it.
//
// Reading them back rather than keeping a second copy. A ring buffer in the
// control plane would be a rival store of the same numbers with its own
// retention, lost on every restart, and disagreeing with the one an operator
// already has on screen in Grafana.
package promq
import (
"context"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"time"
)
// Client queries a Prometheus server.
type Client struct {
base string
hc *http.Client
}
// New builds a client, or nil when no address is configured.
//
// Nil rather than an error: a deployment with no Prometheus is an ordinary one,
// and the dashboard leaves the panels out rather than showing an error where a
// chart should be.
func New(addr string, timeout time.Duration) *Client {
addr = strings.TrimSuffix(strings.TrimSpace(addr), "/")
if addr == "" {
return nil
}
if timeout <= 0 {
// Short. This runs while somebody is waiting for a page, and a
// monitoring system that is slow today is better skipped than waited
// for — the rest of the dashboard does not need it.
timeout = 3 * time.Second
}
return &Client{base: addr, hc: &http.Client{Timeout: timeout}}
}
// Series is one line: a name and its points, oldest first.
type Series struct {
Name string
Points []Point
}
// Point is one sample.
type Point struct {
At time.Time
Value float64
}
// Max is the largest value in the series, and 0 for an empty one.
func (s Series) Max() float64 {
m := 0.0
for _, p := range s.Points {
if p.Value > m {
m = p.Value
}
}
return m
}
// Last is the most recent value, and 0 for an empty series.
func (s Series) Last() float64 {
if len(s.Points) == 0 {
return 0
}
return s.Points[len(s.Points)-1].Value
}
// RangeQuery reads a metric over a window, one series per label combination.
//
// nameBy picks the label that names each line. A series whose chosen label is
// missing is named by the metric itself, because a chart legend reading "{}"
// helps nobody and dropping the line hides data.
func (c *Client) RangeQuery(ctx context.Context, expr, nameBy string,
window, step time.Duration,
) ([]Series, error) {
if c == nil {
return nil, nil
}
if step <= 0 {
step = window / 60
}
if step < time.Second {
step = time.Second
}
end := time.Now()
q := url.Values{
"query": {expr},
"start": {strconv.FormatInt(end.Add(-window).Unix(), 10)},
"end": {strconv.FormatInt(end.Unix(), 10)},
"step": {strconv.Itoa(int(step.Seconds()))},
}
var body struct {
Status string `json:"status"`
Error string `json:"error"`
Data struct {
Result []struct {
Metric map[string]string `json:"metric"`
Values [][2]any `json:"values"`
} `json:"result"`
} `json:"data"`
}
if err := c.get(ctx, "/api/v1/query_range?"+q.Encode(), &body); err != nil {
return nil, err
}
if body.Status != "success" {
return nil, fmt.Errorf("prometheus refused the query: %s", body.Error)
}
out := make([]Series, 0, len(body.Data.Result))
for _, r := range body.Data.Result {
s := Series{Name: r.Metric[nameBy]}
if s.Name == "" {
s.Name = r.Metric["__name__"]
}
if s.Name == "" {
s.Name = "series"
}
for _, v := range r.Values {
at, ok := toFloat(v[0])
if !ok {
continue
}
val, ok := toFloat(v[1])
if !ok {
// A sample Prometheus could not represent — NaN comes back as
// the string "NaN". Skipped rather than drawn as zero, which
// would put a line on the floor where there is no data.
continue
}
s.Points = append(s.Points, Point{At: time.Unix(int64(at), 0), Value: val})
}
if len(s.Points) > 0 {
out = append(out, s)
}
}
// Busiest first, so the lines that matter get the first colours and
// whatever is folded into "other" is genuinely the tail.
sort.SliceStable(out, func(i, j int) bool { return out[i].Max() > out[j].Max() })
return out, nil
}
// toFloat accepts the two shapes a Prometheus sample uses: a number for the
// timestamp and a quoted string for the value.
//
// NaN and the infinities are refused, which ParseFloat will happily return for
// them: Prometheus sends "NaN" as a string for a sample it has no value for,
// and one of those in a chart is an SVG coordinate of NaN — a path that does
// not draw at all, taking the rest of the line with it.
func toFloat(v any) (float64, bool) {
var f float64
switch t := v.(type) {
case float64:
f = t
case string:
parsed, err := strconv.ParseFloat(t, 64)
if err != nil {
return 0, false
}
f = parsed
default:
return 0, false
}
if math.IsNaN(f) || math.IsInf(f, 0) {
return 0, false
}
return f, true
}
func (c *Client) get(ctx context.Context, path string, out any) error {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
if err != nil {
return err
}
resp, err := c.hc.Do(req)
if err != nil {
return fmt.Errorf("reaching Prometheus at %s: %w", c.base, err)
}
defer resp.Body.Close()
// Capped: a range query with a careless step can return a great deal, and
// a page that renders it should not be handed all of it.
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
if err != nil {
return err
}
if resp.StatusCode >= 300 && len(raw) == 0 {
return fmt.Errorf("prometheus answered %d", resp.StatusCode)
}
return json.Unmarshal(raw, out)
}