CI / test (push) Has been cancelled
Usage could be read as a total — per key, per user, per model, per group — and the cluster's own charts covered the fleet's hardware. What nobody could see was when a group used it. "Research doubled its traffic three weeks ago" is not a question a column of totals can be asked, and it is the question somebody asks before a quota conversation. /admin/usage draws requests, tokens and spend over a chosen window, grouped by any level of the tenancy tree or by person. Person is not a level of that tree — one person holds keys in several projects — so the grouping is its own type rather than store.Scope with a fourth value nothing else could use. Three things learned from the charts already here. Buckets are gap-filled per group, because a line drawn straight across an absence claims traffic that did not happen, and because the chart sums its tail by timestamp and can only do that if every series shares a clock. Series are ranked by total rather than by peak — a counter's question is "who used the most", and ranking by peak would colour the team that made one burst over the team that ran steadily all week. And the spend chart is replaced by a sentence when no model has a price, rather than drawn flat along the bottom. The bucket query groups by ordinal and was planned against the cluster's PostgreSQL before being believed, with a test alongside the others that run there: three reports have shipped working on SQLite and failing on it. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
221 lines
5.9 KiB
Go
221 lines
5.9 KiB
Go
// Package promq reads time series back out of Prometheus.
|
|
//
|
|
// Only enough to draw a chart: a range query, and the labels needed to name
|
|
// each line. llmgw already publishes these metrics; what it has never had is
|
|
// the history, because a gauge knows the current value and nothing about the
|
|
// ten minutes before it.
|
|
//
|
|
// Reading them back rather than keeping a second copy. A ring buffer in the
|
|
// control plane would be a rival store of the same numbers with its own
|
|
// retention, lost on every restart, and disagreeing with the one an operator
|
|
// already has on screen in Grafana.
|
|
package promq
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Client queries a Prometheus server.
|
|
type Client struct {
|
|
base string
|
|
hc *http.Client
|
|
}
|
|
|
|
// New builds a client, or nil when no address is configured.
|
|
//
|
|
// Nil rather than an error: a deployment with no Prometheus is an ordinary one,
|
|
// and the dashboard leaves the panels out rather than showing an error where a
|
|
// chart should be.
|
|
func New(addr string, timeout time.Duration) *Client {
|
|
addr = strings.TrimSuffix(strings.TrimSpace(addr), "/")
|
|
if addr == "" {
|
|
return nil
|
|
}
|
|
if timeout <= 0 {
|
|
// Short. This runs while somebody is waiting for a page, and a
|
|
// monitoring system that is slow today is better skipped than waited
|
|
// for — the rest of the dashboard does not need it.
|
|
timeout = 3 * time.Second
|
|
}
|
|
return &Client{base: addr, hc: &http.Client{Timeout: timeout}}
|
|
}
|
|
|
|
// Series is one line: a name and its points, oldest first.
|
|
type Series struct {
|
|
Name string
|
|
Points []Point
|
|
}
|
|
|
|
// Point is one sample.
|
|
type Point struct {
|
|
At time.Time
|
|
Value float64
|
|
}
|
|
|
|
// Max is the largest value in the series, and 0 for an empty one.
|
|
func (s Series) Max() float64 {
|
|
m := 0.0
|
|
for _, p := range s.Points {
|
|
if p.Value > m {
|
|
m = p.Value
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Sum is every value added together, and 0 for an empty series.
|
|
//
|
|
// Meaningful for a counter sampled into buckets, where each point is what
|
|
// happened during that bucket, and not for a gauge, where adding successive
|
|
// readings of the same thing produces a number that means nothing.
|
|
func (s Series) Sum() float64 {
|
|
total := 0.0
|
|
for _, p := range s.Points {
|
|
total += p.Value
|
|
}
|
|
return total
|
|
}
|
|
|
|
// Last is the most recent value, and 0 for an empty series.
|
|
func (s Series) Last() float64 {
|
|
if len(s.Points) == 0 {
|
|
return 0
|
|
}
|
|
return s.Points[len(s.Points)-1].Value
|
|
}
|
|
|
|
// RangeQuery reads a metric over a window, one series per label combination.
|
|
//
|
|
// nameBy picks the label that names each line. A series whose chosen label is
|
|
// missing is named by the metric itself, because a chart legend reading "{}"
|
|
// helps nobody and dropping the line hides data.
|
|
func (c *Client) RangeQuery(ctx context.Context, expr, nameBy string,
|
|
window, step time.Duration,
|
|
) ([]Series, error) {
|
|
if c == nil {
|
|
return nil, nil
|
|
}
|
|
if step <= 0 {
|
|
step = window / 60
|
|
}
|
|
if step < time.Second {
|
|
step = time.Second
|
|
}
|
|
end := time.Now()
|
|
q := url.Values{
|
|
"query": {expr},
|
|
"start": {strconv.FormatInt(end.Add(-window).Unix(), 10)},
|
|
"end": {strconv.FormatInt(end.Unix(), 10)},
|
|
"step": {strconv.Itoa(int(step.Seconds()))},
|
|
}
|
|
|
|
var body struct {
|
|
Status string `json:"status"`
|
|
Error string `json:"error"`
|
|
Data struct {
|
|
Result []struct {
|
|
Metric map[string]string `json:"metric"`
|
|
Values [][2]any `json:"values"`
|
|
} `json:"result"`
|
|
} `json:"data"`
|
|
}
|
|
if err := c.get(ctx, "/api/v1/query_range?"+q.Encode(), &body); err != nil {
|
|
return nil, err
|
|
}
|
|
if body.Status != "success" {
|
|
return nil, fmt.Errorf("prometheus refused the query: %s", body.Error)
|
|
}
|
|
|
|
out := make([]Series, 0, len(body.Data.Result))
|
|
for _, r := range body.Data.Result {
|
|
s := Series{Name: r.Metric[nameBy]}
|
|
if s.Name == "" {
|
|
s.Name = r.Metric["__name__"]
|
|
}
|
|
if s.Name == "" {
|
|
s.Name = "series"
|
|
}
|
|
for _, v := range r.Values {
|
|
at, ok := toFloat(v[0])
|
|
if !ok {
|
|
continue
|
|
}
|
|
val, ok := toFloat(v[1])
|
|
if !ok {
|
|
// A sample Prometheus could not represent — NaN comes back as
|
|
// the string "NaN". Skipped rather than drawn as zero, which
|
|
// would put a line on the floor where there is no data.
|
|
continue
|
|
}
|
|
s.Points = append(s.Points, Point{At: time.Unix(int64(at), 0), Value: val})
|
|
}
|
|
if len(s.Points) > 0 {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
// Busiest first, so the lines that matter get the first colours and
|
|
// whatever is folded into "other" is genuinely the tail.
|
|
sort.SliceStable(out, func(i, j int) bool { return out[i].Max() > out[j].Max() })
|
|
return out, nil
|
|
}
|
|
|
|
// toFloat accepts the two shapes a Prometheus sample uses: a number for the
|
|
// timestamp and a quoted string for the value.
|
|
//
|
|
// NaN and the infinities are refused, which ParseFloat will happily return for
|
|
// them: Prometheus sends "NaN" as a string for a sample it has no value for,
|
|
// and one of those in a chart is an SVG coordinate of NaN — a path that does
|
|
// not draw at all, taking the rest of the line with it.
|
|
func toFloat(v any) (float64, bool) {
|
|
var f float64
|
|
switch t := v.(type) {
|
|
case float64:
|
|
f = t
|
|
case string:
|
|
parsed, err := strconv.ParseFloat(t, 64)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
f = parsed
|
|
default:
|
|
return 0, false
|
|
}
|
|
if math.IsNaN(f) || math.IsInf(f, 0) {
|
|
return 0, false
|
|
}
|
|
return f, true
|
|
}
|
|
|
|
func (c *Client) get(ctx context.Context, path string, out any) error {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := c.hc.Do(req)
|
|
if err != nil {
|
|
return fmt.Errorf("reaching Prometheus at %s: %w", c.base, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Capped: a range query with a careless step can return a great deal, and
|
|
// a page that renders it should not be handed all of it.
|
|
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.StatusCode >= 300 && len(raw) == 0 {
|
|
return fmt.Errorf("prometheus answered %d", resp.StatusCode)
|
|
}
|
|
return json.Unmarshal(raw, out)
|
|
}
|