// Package promq reads time series back out of Prometheus. // // Only enough to draw a chart: a range query, and the labels needed to name // each line. llmgw already publishes these metrics; what it has never had is // the history, because a gauge knows the current value and nothing about the // ten minutes before it. // // Reading them back rather than keeping a second copy. A ring buffer in the // control plane would be a rival store of the same numbers with its own // retention, lost on every restart, and disagreeing with the one an operator // already has on screen in Grafana. package promq import ( "context" "encoding/json" "fmt" "io" "math" "net/http" "net/url" "sort" "strconv" "strings" "time" ) // Client queries a Prometheus server. type Client struct { base string hc *http.Client } // New builds a client, or nil when no address is configured. // // Nil rather than an error: a deployment with no Prometheus is an ordinary one, // and the dashboard leaves the panels out rather than showing an error where a // chart should be. func New(addr string, timeout time.Duration) *Client { addr = strings.TrimSuffix(strings.TrimSpace(addr), "/") if addr == "" { return nil } if timeout <= 0 { // Short. This runs while somebody is waiting for a page, and a // monitoring system that is slow today is better skipped than waited // for — the rest of the dashboard does not need it. timeout = 3 * time.Second } return &Client{base: addr, hc: &http.Client{Timeout: timeout}} } // Series is one line: a name and its points, oldest first. type Series struct { Name string Points []Point } // Point is one sample. type Point struct { At time.Time Value float64 } // Max is the largest value in the series, and 0 for an empty one. func (s Series) Max() float64 { m := 0.0 for _, p := range s.Points { if p.Value > m { m = p.Value } } return m } // Sum is every value added together, and 0 for an empty series. // // Meaningful for a counter sampled into buckets, where each point is what // happened during that bucket, and not for a gauge, where adding successive // readings of the same thing produces a number that means nothing. func (s Series) Sum() float64 { total := 0.0 for _, p := range s.Points { total += p.Value } return total } // Last is the most recent value, and 0 for an empty series. func (s Series) Last() float64 { if len(s.Points) == 0 { return 0 } return s.Points[len(s.Points)-1].Value } // RangeQuery reads a metric over a window, one series per label combination. // // nameBy picks the label that names each line. A series whose chosen label is // missing is named by the metric itself, because a chart legend reading "{}" // helps nobody and dropping the line hides data. func (c *Client) RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration, ) ([]Series, error) { if c == nil { return nil, nil } if step <= 0 { step = window / 60 } if step < time.Second { step = time.Second } end := time.Now() q := url.Values{ "query": {expr}, "start": {strconv.FormatInt(end.Add(-window).Unix(), 10)}, "end": {strconv.FormatInt(end.Unix(), 10)}, "step": {strconv.Itoa(int(step.Seconds()))}, } var body struct { Status string `json:"status"` Error string `json:"error"` Data struct { Result []struct { Metric map[string]string `json:"metric"` Values [][2]any `json:"values"` } `json:"result"` } `json:"data"` } if err := c.get(ctx, "/api/v1/query_range?"+q.Encode(), &body); err != nil { return nil, err } if body.Status != "success" { return nil, fmt.Errorf("prometheus refused the query: %s", body.Error) } out := make([]Series, 0, len(body.Data.Result)) for _, r := range body.Data.Result { s := Series{Name: r.Metric[nameBy]} if s.Name == "" { s.Name = r.Metric["__name__"] } if s.Name == "" { s.Name = "series" } for _, v := range r.Values { at, ok := toFloat(v[0]) if !ok { continue } val, ok := toFloat(v[1]) if !ok { // A sample Prometheus could not represent — NaN comes back as // the string "NaN". Skipped rather than drawn as zero, which // would put a line on the floor where there is no data. continue } s.Points = append(s.Points, Point{At: time.Unix(int64(at), 0), Value: val}) } if len(s.Points) > 0 { out = append(out, s) } } // Busiest first, so the lines that matter get the first colours and // whatever is folded into "other" is genuinely the tail. sort.SliceStable(out, func(i, j int) bool { return out[i].Max() > out[j].Max() }) return out, nil } // toFloat accepts the two shapes a Prometheus sample uses: a number for the // timestamp and a quoted string for the value. // // NaN and the infinities are refused, which ParseFloat will happily return for // them: Prometheus sends "NaN" as a string for a sample it has no value for, // and one of those in a chart is an SVG coordinate of NaN — a path that does // not draw at all, taking the rest of the line with it. func toFloat(v any) (float64, bool) { var f float64 switch t := v.(type) { case float64: f = t case string: parsed, err := strconv.ParseFloat(t, 64) if err != nil { return 0, false } f = parsed default: return 0, false } if math.IsNaN(f) || math.IsInf(f, 0) { return 0, false } return f, true } func (c *Client) get(ctx context.Context, path string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil) if err != nil { return err } resp, err := c.hc.Do(req) if err != nil { return fmt.Errorf("reaching Prometheus at %s: %w", c.base, err) } defer resp.Body.Close() // Capped: a range query with a careless step can return a great deal, and // a page that renders it should not be handed all of it. raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) if err != nil { return err } if resp.StatusCode >= 300 && len(raw) == 0 { return fmt.Errorf("prometheus answered %d", resp.StatusCode) } return json.Unmarshal(raw, out) }