diff --git a/docs/reference.md b/docs/reference.md index 9ae7312..0220e05 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -94,6 +94,9 @@ rather than repeated. | `context_length` | `int` | [Model](#model) | | `max_output` | `int` | [Model](#model) | | `capabilities` | `Capabilities` | [Model](#model) | +| `prometheus` | `string` | [Monitoring](#monitoring) | +| `window` | `time.Duration` | [Monitoring](#monitoring) | +| `timeout` | `time.Duration` | [Monitoring](#monitoring) | | `nvidia_smi` | `string` | [Options](#options) | | `ib_root` | `string` | [Options](#options) | | `prefer_nvml` | `bool` | [Options](#options) | @@ -155,6 +158,7 @@ rather than repeated. | `threads` | `Threads` | [Server](#server) | | `landing` | `Landing` | [Server](#server) | | `agent_tls` | `tlsconf.Config` | [Server](#server) | +| `monitoring` | `Monitoring` | [Server](#server) | | `mirror` | `Mirror` | [Server](#server) | | `router` | `Router` | [Server](#server) | | `control` | `Control` | [Server](#server) | @@ -707,6 +711,29 @@ Capabilities is what this model can do beyond plain completion. Takes a [Capabilities](#capabilities) block. +## Monitoring + +### `prometheus` + +`string` + +Prometheus is the server's address, for example +http://monitoring.internal:9090. Empty leaves the charts off the +dashboard rather than showing an error where one should be. + +### `window` + +`time.Duration` + +Window is how far back the charts look. Zero takes an hour. + +### `timeout` + +`time.Duration` + +Timeout bounds one query. It runs while somebody waits for a page, so a +monitoring system that is slow today is skipped rather than waited for. + ## Options ### `nvidia_smi` @@ -1133,6 +1160,14 @@ trusted certificate, while this one is mutual and private. Takes a [Config](#config) block. +### `monitoring` + +`Monitoring` + +Monitoring is where the dashboard reads its history from. + +Takes a [Monitoring](#monitoring) block. + ### `mirror` `Mirror` diff --git a/internal/cli/server.go b/internal/cli/server.go index 3adcc0a..f3adefd 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -29,6 +29,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" "gitea.mynodes.xyz/ilker/llmgw/internal/router" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" @@ -189,6 +190,14 @@ func newServerCmd() *cobra.Command { // session and an admin flag. Handing the router the data rather // than the port keeps pprof where it is. rt.SetClusterView(api, usage) + // The dashboard's operational charts. llmgw publishes these + // metrics; what it cannot do is remember them, so the history + // is read back from wherever they were scraped to. + if h := promq.New(cfg.Monitoring.Prometheus, cfg.Monitoring.Timeout); h != nil { + rt.SetHistory(h, cfg.Monitoring.Window) + log.Info("dashboard history configured", + "prometheus", cfg.Monitoring.Prometheus) + } } // The model mirror, when one is configured. Built here and not at diff --git a/internal/config/config.go b/internal/config/config.go index 80da07f..c1ef662 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,6 +65,9 @@ type Server struct { // trusted certificate, while this one is mutual and private. AgentTLS tlsconf.Config `koanf:"agent_tls"` + // Monitoring is where the dashboard reads its history from. + Monitoring Monitoring `koanf:"monitoring"` + // Mirror points the control plane at a model mirror it can administer. // Empty means there is none, and /admin/mirror stays unmounted. Mirror Mirror `koanf:"mirror"` @@ -79,6 +82,25 @@ type Server struct { Endpoints []Endpoint `koanf:"endpoints"` } +// Monitoring points the dashboard at a Prometheus holding this cluster's +// history. +// +// Read back rather than kept twice. llmgw publishes these metrics already; what +// a gauge cannot do is remember, and a ring buffer in this process would be a +// rival copy with its own retention, lost on every restart and disagreeing with +// the one an operator already has on screen. +type Monitoring struct { + // Prometheus is the server's address, for example + // http://monitoring.internal:9090. Empty leaves the charts off the + // dashboard rather than showing an error where one should be. + Prometheus string `koanf:"prometheus"` + // Window is how far back the charts look. Zero takes an hour. + Window time.Duration `koanf:"window"` + // Timeout bounds one query. It runs while somebody waits for a page, so a + // monitoring system that is slow today is skipped rather than waited for. + Timeout time.Duration `koanf:"timeout"` +} + // Mirror is a model mirror this control plane administers over its API. // // The mirror is a separate machine with a large disk, not part of the gateway, diff --git a/internal/promq/promq.go b/internal/promq/promq.go new file mode 100644 index 0000000..41e9013 --- /dev/null +++ b/internal/promq/promq.go @@ -0,0 +1,207 @@ +// Package promq reads time series back out of Prometheus. +// +// Only enough to draw a chart: a range query, and the labels needed to name +// each line. llmgw already publishes these metrics; what it has never had is +// the history, because a gauge knows the current value and nothing about the +// ten minutes before it. +// +// Reading them back rather than keeping a second copy. A ring buffer in the +// control plane would be a rival store of the same numbers with its own +// retention, lost on every restart, and disagreeing with the one an operator +// already has on screen in Grafana. +package promq + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +// Client queries a Prometheus server. +type Client struct { + base string + hc *http.Client +} + +// New builds a client, or nil when no address is configured. +// +// Nil rather than an error: a deployment with no Prometheus is an ordinary one, +// and the dashboard leaves the panels out rather than showing an error where a +// chart should be. +func New(addr string, timeout time.Duration) *Client { + addr = strings.TrimSuffix(strings.TrimSpace(addr), "/") + if addr == "" { + return nil + } + if timeout <= 0 { + // Short. This runs while somebody is waiting for a page, and a + // monitoring system that is slow today is better skipped than waited + // for — the rest of the dashboard does not need it. + timeout = 3 * time.Second + } + return &Client{base: addr, hc: &http.Client{Timeout: timeout}} +} + +// Series is one line: a name and its points, oldest first. +type Series struct { + Name string + Points []Point +} + +// Point is one sample. +type Point struct { + At time.Time + Value float64 +} + +// Max is the largest value in the series, and 0 for an empty one. +func (s Series) Max() float64 { + m := 0.0 + for _, p := range s.Points { + if p.Value > m { + m = p.Value + } + } + return m +} + +// Last is the most recent value, and 0 for an empty series. +func (s Series) Last() float64 { + if len(s.Points) == 0 { + return 0 + } + return s.Points[len(s.Points)-1].Value +} + +// RangeQuery reads a metric over a window, one series per label combination. +// +// nameBy picks the label that names each line. A series whose chosen label is +// missing is named by the metric itself, because a chart legend reading "{}" +// helps nobody and dropping the line hides data. +func (c *Client) RangeQuery(ctx context.Context, expr, nameBy string, + window, step time.Duration, +) ([]Series, error) { + if c == nil { + return nil, nil + } + if step <= 0 { + step = window / 60 + } + if step < time.Second { + step = time.Second + } + end := time.Now() + q := url.Values{ + "query": {expr}, + "start": {strconv.FormatInt(end.Add(-window).Unix(), 10)}, + "end": {strconv.FormatInt(end.Unix(), 10)}, + "step": {strconv.Itoa(int(step.Seconds()))}, + } + + var body struct { + Status string `json:"status"` + Error string `json:"error"` + Data struct { + Result []struct { + Metric map[string]string `json:"metric"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := c.get(ctx, "/api/v1/query_range?"+q.Encode(), &body); err != nil { + return nil, err + } + if body.Status != "success" { + return nil, fmt.Errorf("prometheus refused the query: %s", body.Error) + } + + out := make([]Series, 0, len(body.Data.Result)) + for _, r := range body.Data.Result { + s := Series{Name: r.Metric[nameBy]} + if s.Name == "" { + s.Name = r.Metric["__name__"] + } + if s.Name == "" { + s.Name = "series" + } + for _, v := range r.Values { + at, ok := toFloat(v[0]) + if !ok { + continue + } + val, ok := toFloat(v[1]) + if !ok { + // A sample Prometheus could not represent — NaN comes back as + // the string "NaN". Skipped rather than drawn as zero, which + // would put a line on the floor where there is no data. + continue + } + s.Points = append(s.Points, Point{At: time.Unix(int64(at), 0), Value: val}) + } + if len(s.Points) > 0 { + out = append(out, s) + } + } + // Busiest first, so the lines that matter get the first colours and + // whatever is folded into "other" is genuinely the tail. + sort.SliceStable(out, func(i, j int) bool { return out[i].Max() > out[j].Max() }) + return out, nil +} + +// toFloat accepts the two shapes a Prometheus sample uses: a number for the +// timestamp and a quoted string for the value. +// +// NaN and the infinities are refused, which ParseFloat will happily return for +// them: Prometheus sends "NaN" as a string for a sample it has no value for, +// and one of those in a chart is an SVG coordinate of NaN — a path that does +// not draw at all, taking the rest of the line with it. +func toFloat(v any) (float64, bool) { + var f float64 + switch t := v.(type) { + case float64: + f = t + case string: + parsed, err := strconv.ParseFloat(t, 64) + if err != nil { + return 0, false + } + f = parsed + default: + return 0, false + } + if math.IsNaN(f) || math.IsInf(f, 0) { + return 0, false + } + return f, true +} + +func (c *Client) get(ctx context.Context, path string, out any) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+path, nil) + if err != nil { + return err + } + resp, err := c.hc.Do(req) + if err != nil { + return fmt.Errorf("reaching Prometheus at %s: %w", c.base, err) + } + defer resp.Body.Close() + + // Capped: a range query with a careless step can return a great deal, and + // a page that renders it should not be handed all of it. + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return err + } + if resp.StatusCode >= 300 && len(raw) == 0 { + return fmt.Errorf("prometheus answered %d", resp.StatusCode) + } + return json.Unmarshal(raw, out) +} diff --git a/internal/promq/promq_test.go b/internal/promq/promq_test.go new file mode 100644 index 0000000..9546bb1 --- /dev/null +++ b/internal/promq/promq_test.go @@ -0,0 +1,99 @@ +package promq + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func server(t *testing.T, body string) *Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return New(srv.URL, time.Second) +} + +func TestSeriesAreNamedByTheLabelAsked(t *testing.T) { + c := server(t, `{"status":"success","data":{"resultType":"matrix","result":[ + {"metric":{"__name__":"llmgw_pool_saturation","model":"qwen"}, + "values":[[1700000000,"0.2"],[1700000015,"0.8"]]}]}}`) + + got, err := c.RangeQuery(t.Context(), "llmgw_pool_saturation", "model", time.Hour, time.Minute) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Name != "qwen" { + t.Fatalf("expected one series named for its model, got %+v", got) + } + if got[0].Last() != 0.8 || got[0].Max() != 0.8 { + t.Errorf("points did not decode: %+v", got[0].Points) + } +} + +func TestASeriesMissingThatLabelIsStillDrawn(t *testing.T) { + c := server(t, `{"status":"success","data":{"result":[ + {"metric":{"__name__":"llmgw_replicas"},"values":[[1700000000,"3"]]}]}}`) + + got, _ := c.RangeQuery(t.Context(), "x", "model", time.Hour, time.Minute) + // A legend reading "{}" helps nobody, and dropping the line hides data. + if len(got) != 1 || got[0].Name != "llmgw_replicas" { + t.Errorf("expected a fallback name, got %+v", got) + } +} + +func TestBusiestSeriesComeFirst(t *testing.T) { + c := server(t, `{"status":"success","data":{"result":[ + {"metric":{"model":"quiet"},"values":[[1,"0.1"]]}, + {"metric":{"model":"busy"},"values":[[1,"9"]]}]}}`) + + got, _ := c.RangeQuery(t.Context(), "x", "model", time.Hour, time.Minute) + // The first colours go to the lines that matter, and whatever is folded + // into "other" is then genuinely the tail. + if got[0].Name != "busy" { + t.Errorf("expected the busiest first, got %+v", got) + } +} + +func TestASampleThatIsNotANumberIsSkippedRatherThanZero(t *testing.T) { + c := server(t, `{"status":"success","data":{"result":[ + {"metric":{"model":"m"},"values":[[1,"1.5"],[2,"NaN"],[3,"2.5"]]}]}}`) + + got, _ := c.RangeQuery(t.Context(), "x", "model", time.Hour, time.Minute) + // Drawn as zero it would put a line on the floor where there is no data, + // which reads as an outage. + if len(got[0].Points) != 2 { + t.Errorf("expected the NaN dropped, got %+v", got[0].Points) + } +} + +func TestAnEmptySeriesIsNotReturned(t *testing.T) { + c := server(t, `{"status":"success","data":{"result":[{"metric":{"model":"m"},"values":[]}]}}`) + if got, _ := c.RangeQuery(t.Context(), "x", "model", time.Hour, time.Minute); len(got) != 0 { + t.Errorf("a series with no points is not a line, got %+v", got) + } +} + +func TestAQueryPrometheusRefusesSaysWhy(t *testing.T) { + c := server(t, `{"status":"error","errorType":"bad_data","error":"parse error at char 1"}`) + _, err := c.RangeQuery(t.Context(), "!!", "model", time.Hour, time.Minute) + if err == nil || !strings.Contains(err.Error(), "parse error") { + t.Errorf("expected the reason to survive, got %v", err) + } +} + +func TestNoAddressMeansNoClient(t *testing.T) { + // A deployment with no Prometheus is an ordinary one. The dashboard leaves + // the panels out rather than showing an error where a chart should be. + var c *Client = New(" ", 0) + if c != nil { + t.Fatal("an empty address should build no client") + } + got, err := c.RangeQuery(t.Context(), "x", "model", time.Hour, time.Minute) + if err != nil || got != nil { + t.Errorf("a nil client should answer nothing, quietly: %v %v", got, err) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 5f7091d..9e0383b 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -154,7 +154,9 @@ func (r *Router) Handler() http.Handler { // the heap that holds every API key in plaintext. mux.HandleFunc("GET /admin", pages.AdminPage(web.Dashboard( r.clusterView, r.clusterUsage, version.Version, 5*time.Second, false, - web.NavExtras{Mirror: r.mirror != nil}, r.log))) + web.NavExtras{ + Mirror: r.mirror != nil, History: r.history, Window: r.histWindow, + }, r.log))) pages.RegisterAdminUsers(mux, r.store) // Quotas are written on the same tree the accounts page builds, so // they are mounted together. diff --git a/internal/router/router.go b/internal/router/router.go index 025585a..02f39f2 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -108,6 +108,10 @@ type Router struct { // catalogAdmin puts a mirrored model into service. Nil without a store, // where a model added at runtime would not survive a restart. catalogAdmin web.CatalogAdmin + // history is the metrics store the dashboard charts read from. Nil where + // none is configured, which leaves the charts off the page. + history web.History + histWindow time.Duration } // SetClusterView gives the router the data the /admin page renders. @@ -131,6 +135,11 @@ func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } // SetCatalogAdmin lets the mirror page put a model into service. func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c } +// SetHistory gives the dashboard somewhere to read the cluster's past. +func (r *Router) SetHistory(h web.History, window time.Duration) { + r.history, r.histWindow = h, window +} + // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } diff --git a/internal/web/dashboard.go b/internal/web/dashboard.go index 9e94808..7ebe3fa 100644 --- a/internal/web/dashboard.go +++ b/internal/web/dashboard.go @@ -58,6 +58,9 @@ type DashboardData struct { // the other into a 404 — or, on the public port, into "no api key was // provided", which is a far more confusing way to say the same thing. OnAdminListener bool + // Fleet is the operational charts, nil when no metrics store is configured. + Fleet *FleetCharts + // Nav says which optional admin pages are mounted. Asked rather than // assumed: a link to a page that was never registered lands on whatever the // catch-all route serves, which on the client-facing port is the landing @@ -118,6 +121,11 @@ type UsageSource interface { type NavExtras struct { // Mirror is true when /admin/mirror is mounted. Mirror bool + // History reads the cluster's past for the operational charts. Nil leaves + // them off the page rather than drawing empty frames. + History History + // Window is how far back those charts look. + Window time.Duration } func Dashboard(src Source, usage UsageSource, version string, refresh time.Duration, @@ -133,6 +141,7 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat d := DashboardData{ OnAdminListener: onAdminListener, Nav: nav, + Fleet: LoadFleetCharts(r.Context(), nav.History, nav.Window, log), RefreshSec: int(refresh.Seconds()), Version: version, Now: time.Now().UTC().Format(time.RFC3339), diff --git a/internal/web/dashcharts.go b/internal/web/dashcharts.go new file mode 100644 index 0000000..369a514 --- /dev/null +++ b/internal/web/dashcharts.go @@ -0,0 +1,92 @@ +package web + +import ( + "context" + "log/slog" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" +) + +// History reads the cluster's past out of a metrics store. +type History interface { + RangeQuery(ctx context.Context, expr, nameBy string, window, step time.Duration) ([]promq.Series, error) +} + +// FleetCharts are the operational panels: what the cluster is doing, rather +// than what it has been billed for. +// +// The usage charts already on this page come from llmgw's own store and answer +// a different question — who spent what. These answer whether the thing is +// healthy, and they need history llmgw does not keep. +type FleetCharts struct { + Saturation TimeSeries + Queue TimeSeries + KV TimeSeries + Ready TimeSeries + Window string + // Err is set when the metrics store could not be read. Shown as a line of + // text where the charts would be: an empty panel reads as a healthy cluster + // with nothing happening, which is the opposite of not knowing. + Err string +} + +// fleetPanel is one chart and the query behind it. +var fleetPanels = []struct { + name string + expr string + nameBy string + unit string + into func(*FleetCharts, TimeSeries) +}{ + { + "saturation", "llmgw_pool_saturation", "model", "", + func(f *FleetCharts, t TimeSeries) { f.Saturation = t }, + }, + { + "queue depth", "llmgw_endpoint_waiting", "replica", "", + func(f *FleetCharts, t TimeSeries) { f.Queue = t }, + }, + { + "kv cache", "llmgw_endpoint_kv_cache_usage", "replica", "%", + func(f *FleetCharts, t TimeSeries) { f.KV = t }, + }, + { + "replicas ready", "llmgw_model_replicas_ready", "model", "", + func(f *FleetCharts, t TimeSeries) { f.Ready = t }, + }, +} + +// LoadFleetCharts queries each panel and renders it. +// +// One failure fails them all, deliberately: four panels where three are drawn +// and one is blank invites reading the blank one as "nothing is happening +// there". Either the metrics store answered or it did not. +func LoadFleetCharts(ctx context.Context, h History, window time.Duration, + log *slog.Logger, +) *FleetCharts { + if h == nil { + return nil + } + if window <= 0 { + window = time.Hour + } + // Sixty points across the window. More is detail nobody can see at this + // size and a bigger response to parse on every page load. + step := window / 60 + + out := &FleetCharts{Window: window.String()} + for _, p := range fleetPanels { + series, err := h.RangeQuery(ctx, p.expr, p.nameBy, window, step) + if err != nil { + if log != nil { + log.Warn("could not read history for the dashboard", + "panel", p.name, "err", err) + } + out.Err = err.Error() + return out + } + p.into(out, LineChart(series, p.unit, 4)) + } + return out +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 5e5fc82..5431edc 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -282,3 +282,17 @@ tr:has(+ .url-row) td, tr:has(+ .error-row) td { border-bottom: none; } inside a table cell breaks the ruling of every column beside it. */ tr.editing > td { background: color-mix(in srgb, var(--accent) 8%, var(--card)); } tr.editing form { margin: .4rem 0; } + +/* Multi-line time series. Thin marks, recessive grid: the data is the darkest + thing on the chart, which is the only rule that makes a chart readable. */ +.chart-lines { height: 150px; } +.chart-lines .line { stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.chart-lines .axis { stroke: var(--viz-grid); stroke-width: 1; } +.chart-lines .tick { fill: var(--muted); font-size: 9px; text-anchor: end; } +/* A legend is always present for two lines or more: colour alone is not an + encoding a reader can rely on. */ +.legend { display: flex; flex-wrap: wrap; gap: .1rem 1rem; margin: .2rem 0 1rem; font-size: .8rem; } +.legend span { display: inline-flex; align-items: center; gap: .35rem; color: var(--muted); } +.legend i { width: 10px; height: 2px; border-radius: 1px; display: inline-block; } +.legend b { font-weight: 600; color: var(--fg); font-variant-numeric: tabular-nums; } +.chart-title { font-size: .9rem; font-weight: 600; margin: 0 0 .3rem; } diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index bcd9d94..dbfbdb9 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -120,6 +120,28 @@ requested whenever a fallback served it.
{{else}}No agents have registered.
{{end}}No history: {{.Err}}
+{{else}} +| Model | Aliases | Ready | Wanted | Profiles | Fallbacks |
|---|---|---|---|---|---|