From 9f479b9a702f640450a2a1469a96b999e8577d2a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:50:16 +0200 Subject: [PATCH] Show each team's first-token and latency percentiles, errors and refusals Percentiles are read from the histograms kept with the usage sums and shown per group on the usage tab and in the people page's panels. Requests refused by a limit are recorded as 429s against their owner, free of charge, so a team hitting its limits shows as such. Usage can be grouped by model. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 ++ internal/accounting/accounting.go | 11 ++ internal/router/handlers.go | 5 + internal/router/pricing_test.go | 36 +++++ internal/store/latency.go | 132 ++++++++++++++++++ internal/store/latency_test.go | 61 ++++++++ .../migrations/postgres/0015_usage_rollup.sql | 18 ++- .../migrations/sqlite/0015_usage_rollup.sql | 18 ++- internal/store/rollup.go | 17 ++- internal/store/usageseries.go | 10 +- internal/web/admin_people.go | 17 ++- internal/web/admin_people_test.go | 24 ++++ internal/web/admin_usage.go | 50 +++++++ internal/web/admin_usage_test.go | 12 +- internal/web/templates/admin_people.html | 7 + internal/web/templates/admin_usage.html | 11 +- internal/web/web.go | 3 + 17 files changed, 418 insertions(+), 29 deletions(-) create mode 100644 internal/store/latency.go create mode 100644 internal/store/latency_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d2de1..f39601e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,21 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration sums — but `ctl request` finds nothing older. Unset keeps every record, as before. +### How each team's requests fared, not only what they cost + +- **Percentiles from the sums.** First token and latency, p50, p95 and p99, + per organization, team, project, person or model, read from the histograms + kept with the usage sums: on the usage tab's table, and in a branch's or + person's panel on the people page, beside the error rate and requests + refused by a limit. +- **Refused requests are counted against their owner.** A request a rate, + token or budget limit refused was counted only in Prometheus, by model; a + team running into its limit looked like a team that had gone quiet. It is + now recorded as a 429 against its key, person and project — no tokens, no + cost — and shown as "Limited". +- **Usage by model** is a grouping of the usage tab, by the model that + answered. + ### People, limits and usage are one page Managing a team meant its members on the people page, its quota on the quotas diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index 66c42e3..a6ada98 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -316,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { } } +// RecordRefused queues the row of a request a limit refused: nothing is +// charged — it used nothing — but it is counted, so a team running into its +// limits shows as one rather than as a team that went quiet. +func (a *Accountant) RecordRefused(u store.Usage) { + if u.KeyID == "" || a.recorder == nil { + return + } + u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0 + a.recorder.Record(u) +} + // Status returns what a key has used, for headers and `ctl`. func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index b758f11..e48a60c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -537,6 +537,11 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.acct.RecordRefused(store.Usage{ + At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, + Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests, + LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, + }) r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start)) } diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 40d06ae..408f681 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -188,3 +188,39 @@ func TestAnIDThatIsNotOneIsReplaced(t *testing.T) { t.Error("a plain id was not kept") } } + +func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) { + // Refused requests were counted only in Prometheus, by model: a team + // running into its limit looked, on every usage page, like a team that + // had gone quiet. + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + rpm := 1 + if err := rt.store.PutLimits(t.Context(), store.Limits{Scope: store.ScopeKey, ScopeID: "key-1", RPM: &rpm}); err != nil { + t.Fatal(err) + } + body := map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "hi"}}} + auth := map[string]string{"Authorization": "Bearer " + secret} + codes := []int{post(t, rt.Handler(), body, auth).Code, post(t, rt.Handler(), body, auth).Code} + if codes[1] != 429 { + t.Fatalf("the second request in the minute answered %v, want 429", codes) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + stats, err := rt.store.StatsBy(t.Context(), store.GroupByProject, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Limited == 1 && s.Requests == 2 { + spend, _ := rt.store.SpendSince(t.Context(), time.Now().Add(-time.Hour)) + if spend["key-1"] == 0 { + t.Error("the request that was served cost nothing") + } + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the refused request was never counted against its project") +} diff --git a/internal/store/latency.go b/internal/store/latency.go new file mode 100644 index 0000000..e3699a3 --- /dev/null +++ b/internal/store/latency.go @@ -0,0 +1,132 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Quantiles are a duration's percentiles, in milliseconds, read from the +// histogram kept with the usage sums. Within a bucket the value is +// interpolated; above the last bucket's edge it is that edge, and Over says +// so. N is how many durations they are read from — zero is no reading. +type Quantiles struct { + P50, P95, P99 int64 + N int64 + Over bool +} + +// GroupStats is one group's requests, how they ended, and how long they took. +type GroupStats struct { + ID string + Requests int64 + Errors int64 + // Limited is requests refused by a rate, token or budget limit. + Limited int64 + Latency Quantiles + // TTFT is time to first token, for requests that streamed one. + TTFT Quantiles +} + +// ErrorRate is the share of requests that failed, limits included, in percent. +func (g GroupStats) ErrorRate() float64 { + if g.Requests == 0 { + return 0 + } + return 100 * float64(g.Errors) / float64(g.Requests) +} + +// StatsBy is each group's request outcomes and latency percentiles since a +// moment, keyed by the group's id. +// +// Read from the hourly sums for a window of two days or more and from the +// five-minute ones otherwise, starting at the slice the moment falls in: +// percentiles are a shape, and a window starting up to one slice early does +// not change one. Budgets, which must be exact, are not read this way. +func (s *DB) StatsBy(ctx context.Context, g Grouping, since time.Time) (map[string]GroupStats, error) { + c, ok := groupColumns[g] + if !ok { + return nil, fmt.Errorf("usage cannot be grouped by %q", g) + } + t := tier5m + if time.Since(since) >= 48*time.Hour { + t = tier1h + } + n := len(LatencyBounds) + 1 + var sums []string + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.lat_%d)", i)) + } + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.ttft_%d)", i)) + } + rows, err := s.db.QueryContext(ctx, ` + SELECT `+c.id+`, SUM(u.requests), SUM(u.errors), SUM(u.limited), `+strings.Join(sums, ", ")+` + FROM `+t.table+` u`+c.joins+` + WHERE u.bucket >= ? + GROUP BY 1`, t.floor(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[string]GroupStats{} + for rows.Next() { + var gs GroupStats + lat, ttft := make([]int64, n), make([]int64, n) + dst := []any{&gs.ID, &gs.Requests, &gs.Errors, &gs.Limited} + for i := range lat { + dst = append(dst, &lat[i]) + } + for i := range ttft { + dst = append(dst, &ttft[i]) + } + if err := rows.Scan(dst...); err != nil { + return nil, err + } + gs.Latency, gs.TTFT = quantiles(lat), quantiles(ttft) + out[gs.ID] = gs + } + return out, rows.Err() +} + +// quantiles reads p50, p95 and p99 from counts per LatencyBounds bucket. +func quantiles(counts []int64) Quantiles { + var q Quantiles + for _, c := range counts { + q.N += c + } + if q.N == 0 { + return q + } + at := func(p float64) (int64, bool) { + rank := p * float64(q.N) + var seen float64 + for i, c := range counts { + if c == 0 { + continue + } + if seen+float64(c) >= rank { + if i >= len(LatencyBounds) { + return LatencyBounds[len(LatencyBounds)-1], true + } + lo := int64(0) + if i > 0 { + lo = LatencyBounds[i-1] + } + hi := LatencyBounds[i] + frac := (rank - seen) / float64(c) + return lo + int64(frac*float64(hi-lo)), false + } + seen += float64(c) + } + return LatencyBounds[len(LatencyBounds)-1], true + } + var over bool + q.P50, _ = at(0.50) + q.P95, _ = at(0.95) + q.P99, over = at(0.99) + q.Over = over + return q +} diff --git a/internal/store/latency_test.go b/internal/store/latency_test.go new file mode 100644 index 0000000..1fabc66 --- /dev/null +++ b/internal/store/latency_test.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "testing" + "time" +) + +func TestQuantilesAreReadFromTheHistogram(t *testing.T) { + // 100 durations: 90 at <=100 ms, 8 in (1000, 2500], 2 above two minutes. + counts := make([]int64, len(LatencyBounds)+1) + counts[0], counts[4], counts[len(counts)-1] = 90, 8, 2 + q := quantiles(counts) + if q.N != 100 || q.P50 > 100 || q.P95 <= 1000 || q.P95 > 2500 || !q.Over { + t.Errorf("%+v: want p50 within 100 ms, p95 in (1000, 2500], p99 past the last edge", q) + } + if (quantiles(make([]int64, len(LatencyBounds)+1)) != Quantiles{}) { + t.Error("no durations read as some") + } +} + +func TestStatsSayHowEachGroupFaredAndHowLongItTook(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + var batch []Usage + for i := range 20 { + batch = append(batch, Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", + ServedModel: "big", Status: 200, LatencyMs: 4000, TTFTMs: int64(200 + i)}) + } + batch = append(batch, + Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", ServedModel: "big", Status: 502, LatencyMs: 50}, + Usage{At: at, KeyID: "k2", UserID: "bora", ProjectID: "p", Model: "small", ServedModel: "small", Status: 429}, + ) + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + byUser, err := s.StatsBy(ctx, GroupByUser, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + a, b := byUser["ayse"], byUser["bora"] + if a.Requests != 21 || a.Errors != 1 || a.Limited != 0 || b.Limited != 1 || b.Errors != 1 { + t.Errorf("outcomes: ayse %+v, bora %+v", a, b) + } + if a.TTFT.N != 20 || a.TTFT.P95 <= 100 || a.TTFT.P95 > 250 || a.Latency.P95 <= 2500 || a.Latency.P95 > 5000 { + t.Errorf("ayse's first token %+v and latency %+v", a.TTFT, a.Latency) + } + byModel, err := s.StatsBy(ctx, GroupByModel, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if byModel["big"].Requests != 21 || byModel["small"].Requests != 1 { + t.Errorf("by model: %+v", byModel) + } + series, err := s.UsageBucketsBy(ctx, GroupByModel, at.Add(-time.Hour), time.Hour) + if err != nil || len(series) != 2 { + t.Fatalf("usage over time by model: %v, %+v", err, series) + } + }) +} diff --git a/internal/store/migrations/postgres/0015_usage_rollup.sql b/internal/store/migrations/postgres/0015_usage_rollup.sql index 39abcf7..9121268 100644 --- a/internal/store/migrations/postgres/0015_usage_rollup.sql +++ b/internal/store/migrations/postgres/0015_usage_rollup.sql @@ -21,7 +21,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket BIGINT NOT NULL, @@ -32,6 +33,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -70,12 +72,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -92,6 +94,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -130,12 +133,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -152,6 +155,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -190,12 +194,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/migrations/sqlite/0015_usage_rollup.sql b/internal/store/migrations/sqlite/0015_usage_rollup.sql index 17d9ae9..abe08f8 100644 --- a/internal/store/migrations/sqlite/0015_usage_rollup.sql +++ b/internal/store/migrations/sqlite/0015_usage_rollup.sql @@ -20,7 +20,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket INTEGER NOT NULL, @@ -31,6 +32,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -69,12 +71,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -91,6 +93,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -129,12 +132,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -151,6 +154,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -189,12 +193,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/rollup.go b/internal/store/rollup.go index 4333ee2..fa0e296 100644 --- a/internal/store/rollup.go +++ b/internal/store/rollup.go @@ -68,9 +68,10 @@ type rollupKey struct { // rollupRow is what a batch adds to one row. type rollupRow struct { - requests, errors, prompt, completion, cost int64 - latency, queue, ttft, ttftN, last int64 - lat, ttftH []int64 + requests, errors, limited, prompt int64 + completion, cost int64 + latency, queue, ttft, ttftN, last int64 + lat, ttftH []int64 } // rollupBatch sums a batch of records into the rows of one tier they add to. @@ -91,6 +92,9 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey if r.Status >= 400 { row.errors++ } + if r.Status == 429 { + row.limited++ + } row.prompt += r.PromptTokens row.completion += r.CompletionTokens row.cost += r.CostMicros @@ -111,7 +115,7 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey var rollupColumns = func() []string { n := len(LatencyBounds) + 1 cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model", - "requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros", + "requests", "errors", "limited", "prompt_tokens", "completion_tokens", "cost_micros", "latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"} for i := range n { cols = append(cols, fmt.Sprintf("lat_%d", i)) @@ -151,7 +155,7 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { for _, k := range order { r := rows[k] args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel, - r.requests, r.errors, r.prompt, r.completion, r.cost, + r.requests, r.errors, r.limited, r.prompt, r.completion, r.cost, r.latency, r.queue, r.ttft, r.ttftN, r.last} for _, v := range r.lat { args = append(args, v) @@ -182,10 +186,11 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { func usageSince(since time.Time) (string, []any) { from := millis(since) to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since) - const cols = `key_id, user_id, project_id, model, served_model, requests, errors, + const cols = `key_id, user_id, project_id, model, served_model, requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts` return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests, CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors, + CASE WHEN status = 429 THEN 1 ELSE 0 END AS limited, prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts FROM usage WHERE ts >= ? AND ts < ? UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ? diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go index 398adf7..d5e6eee 100644 --- a/internal/store/usageseries.go +++ b/internal/store/usageseries.go @@ -21,6 +21,9 @@ const ( GroupByTeam Grouping = "team" GroupByProject Grouping = "project" GroupByUser Grouping = "user" + // GroupByModel is by the model that answered, which differs from the one + // asked for when a fallback served it. + GroupByModel Grouping = "model" ) // Valid reports whether this is a grouping usage can be read by. @@ -32,7 +35,7 @@ func (g Grouping) Valid() bool { // Groupings lists them in the order a page should offer them: widest first, // then the one that is not a level at all. func Groupings() []Grouping { - return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser} + return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser, GroupByModel} } // UsageSeries is one group's usage over time. @@ -87,8 +90,13 @@ var groupColumns = map[Grouping]struct{ id, name, joins string }{ // rather than hides. GroupByUser: {"u.user_id", "COALESCE(us.email, '')", ` LEFT JOIN users us ON us.id = NULLIF(u.user_id, '')`}, + GroupByModel: {servedModel, servedModel, ""}, } +// servedModel is what answered: the served model, or the one asked for on a +// row from before the two were told apart. +const servedModel = "CASE WHEN u.served_model != '' THEN u.served_model ELSE u.model END" + // UsageBucketsBy returns each group's usage over time. // // One query for every group rather than one per group: the page shows them diff --git a/internal/web/admin_people.go b/internal/web/admin_people.go index 7cfe35d..5eae074 100644 --- a/internal/web/admin_people.go +++ b/internal/web/admin_people.go @@ -66,10 +66,14 @@ type peopleDetail struct { BudgetsBind bool // Its use over the last week, and the totals beside it. - HasUsage bool - Requests, Tokens, Spend TimeSeries - Totals usageRow - Priced bool + HasUsage bool + Requests, Tokens, Spend TimeSeries + Totals usageRow + Priced bool + // Stats is how its requests ended and how long they took, over the + // same week; HasStats is false where the store does not say. + Stats store.GroupStats + HasStats bool UsageWindow, QuotaWindow string CSRF string @@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) { } d.HasUsage = true d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced() + if sr, ok := p.usageStore.(StatsReader); ok { + if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil { + d.Stats, d.HasStats = stats[d.ID], true + } + } for _, s := range rows { if s.ID == d.ID { var table []usageRow diff --git a/internal/web/admin_people_test.go b/internal/web/admin_people_test.go index c9e1fed..4039f03 100644 --- a/internal/web/admin_people_test.go +++ b/internal/web/admin_people_test.go @@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) { t.Error("a panel opened for an organization that does not exist") } } + +func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) { + stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25, + TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150}, + Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}} + u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200, + Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}}, + }} + mux := peoplePage(t, oneTeam(), u) + + table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ") + for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} { + if !strings.Contains(table, want) { + t.Errorf("the usage table lacks %q", want) + } + } + panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ") + for _, want := range []string{"
Refused by a limit
", "p95 900 ms", "p99 > 120 s"} { + if !strings.Contains(panel, want) { + t.Errorf("the panel lacks %q", want) + } + } +} diff --git a/internal/web/admin_usage.go b/internal/web/admin_usage.go index 4f0d0b2..ebcf0a7 100644 --- a/internal/web/admin_usage.go +++ b/internal/web/admin_usage.go @@ -2,6 +2,7 @@ package web import ( "context" + "fmt" "net/http" "sort" "time" @@ -16,6 +17,35 @@ type UsageAdmin interface { since time.Time, width time.Duration) ([]store.UsageSeries, error) } +// StatsReader is how each group fared and how long its requests took; the +// store implements it. Optional: without it the usage table has no such +// columns. +type StatsReader interface { + StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error) +} + +// pctile is a percentile from the usage histograms as a person reads it: +// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read. +func pctile(q store.Quantiles, which int) string { + if q.N == 0 { + return "—" + } + v := q.P50 + switch which { + case 95: + v = q.P95 + case 99: + v = q.P99 + if q.Over { + return fmt.Sprintf("> %d s", v/1000) + } + } + if v < 1000 { + return fmt.Sprintf("%d ms", v) + } + return fmt.Sprintf("%.1f s", float64(v)/1000) +} + // usageWindow is one length of history the page offers, and the slice it is cut // into. // @@ -54,11 +84,16 @@ type usageGroup struct { // usageRow is one group's totals, shown under the charts. type usageRow struct { + ID string Name string Requests int64 Tokens int64 Spend string Errors int64 + // Stats is how its requests ended and how long they took; HasStats is + // false where the store does not say. + Stats store.GroupStats + HasStats bool } type adminUsageData struct { @@ -86,6 +121,8 @@ type adminUsageData struct { // model on a GPU. GPUHeld TimeSeries GPUHours []gpuHoursRow + // HasStats adds outcomes and latency percentiles to the table. + HasStats bool Error string Version string @@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced) } d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows) + if sr, ok := s.(StatsReader); ok { + stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span)) + if err != nil { + p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err) + } + for i := range d.Rows { + d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil + } + d.HasStats = stats != nil + } d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win) return d } @@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string { return "project" case store.GroupByUser: return "person" + case store.GroupByModel: + return "model" default: return string(g) } @@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table [] }), "", 4) for _, u := range rows { table = append(table, usageRow{ + ID: u.ID, Name: displayName(u), Requests: u.Requests, Tokens: u.Tokens(), diff --git a/internal/web/admin_usage_test.go b/internal/web/admin_usage_test.go index c157d15..72de07c 100644 --- a/internal/web/admin_usage_test.go +++ b/internal/web/admin_usage_test.go @@ -15,9 +15,11 @@ import ( type fakeScopeSeries struct { byGroup map[store.Grouping][]store.UsageSeries - asked []store.Grouping - width time.Duration - since time.Time + // stats, when set, is what StatsBy answers for every grouping. + stats map[string]store.GroupStats + asked []store.Grouping + width time.Duration + since time.Time } func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, @@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, return f.byGroup[g], nil } +func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) { + return f.stats, nil +} + func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} diff --git a/internal/web/templates/admin_people.html b/internal/web/templates/admin_people.html index bf53062..51357b6 100644 --- a/internal/web/templates/admin_people.html +++ b/internal/web/templates/admin_people.html @@ -140,6 +140,13 @@

Use, {{.UsageWindow}}

{{if .Totals.Name}}

{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}

+{{if and .HasStats .Stats.Requests}}{{with .Stats}} +
+
Error rate
{{pct1 .ErrorRate}}% +
Refused by a limit
{{if .Limited}}{{compact .Limited}}{{else}}none{{end}} +
First token
p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}
+
Latency
p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}
+
{{end}}{{end}} {{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}} {{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}} {{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}} diff --git a/internal/web/templates/admin_usage.html b/internal/web/templates/admin_usage.html index 456960b..ad6d637 100644 --- a/internal/web/templates/admin_usage.html +++ b/internal/web/templates/admin_usage.html @@ -40,7 +40,11 @@ {{if .Rows}} - + + {{if .HasStats}} + + + {{end}} {{range .Rows}} @@ -48,6 +52,11 @@ + {{if $.HasStats}}{{with .Stats}} + + + + {{end}}{{end}} {{end}}
GroupRequestsTokensSpendErrors
GroupRequestsTokensSpendErrorsError rateLimitedFirst token p50 / p95Latency p95
{{.Name}}{{compact .Tokens}} {{.Spend}} {{if .Errors}}{{.Errors}}{{else}}—{{end}}{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}{{pctile .TTFT 50}} / {{pctile .TTFT 95}}{{pctile .Latency 95}}
diff --git a/internal/web/web.go b/internal/web/web.go index 8f44fe9..e870014 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -22,6 +22,7 @@ import ( "io/fs" "net/http" "slices" + "strconv" "strings" "sync" "time" @@ -90,6 +91,8 @@ var funcs = template.FuncMap{ return strings.Join(v, ", ") }, "latency": humanLatency, + "pctile": pctile, + "pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) }, "duration": func(sec int64) string { return time.Duration(sec * int64(time.Second)).Round(time.Second).String() },