Show each team's first-token and latency percentiles, errors and refusals

Percentiles are read from the histograms kept with the usage sums and shown
per group on the usage tab and in the people page's panels. Requests refused
by a limit are recorded as 429s against their owner, free of charge, so a team
hitting its limits shows as such. Usage can be grouped by model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 19:50:16 +02:00
1 parent 4454e1322c
commit 9f479b9a70
17 files changed
+418 -29

No files matched your search

+15
View File
@@ -97,6 +97,21 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration
sums — but `ctl request` finds nothing older. Unset keeps every record, as
before.
### How each team's requests fared, not only what they cost
- **Percentiles from the sums.** First token and latency, p50, p95 and p99,
per organization, team, project, person or model, read from the histograms
kept with the usage sums: on the usage tab's table, and in a branch's or
person's panel on the people page, beside the error rate and requests
refused by a limit.
- **Refused requests are counted against their owner.** A request a rate,
token or budget limit refused was counted only in Prometheus, by model; a
team running into its limit looked like a team that had gone quiet. It is
now recorded as a 429 against its key, person and project — no tokens, no
cost — and shown as "Limited".
- **Usage by model** is a grouping of the usage tab, by the model that
answered.
### People, limits and usage are one page
Managing a team meant its members on the people page, its quota on the quotas
+11
View File
@@ -316,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
}
}
// RecordRefused queues the row of a request a limit refused: nothing is
// charged — it used nothing — but it is counted, so a team running into its
// limits shows as one rather than as a team that went quiet.
func (a *Accountant) RecordRefused(u store.Usage) {
if u.KeyID == "" || a.recorder == nil {
return
}
u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0
a.recorder.Record(u)
}
// Status returns what a key has used, for headers and `ctl`.
func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
if p.KeyID == "" {
+5
View File
@@ -537,6 +537,11 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
// which is absent when waiting does not help.
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0)
r.acct.RecordRefused(store.Usage{
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests,
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID,
})
r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start))
}
+36
View File
@@ -188,3 +188,39 @@ func TestAnIDThatIsNotOneIsReplaced(t *testing.T) {
t.Error("a plain id was not kept")
}
}
func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) {
// Refused requests were counted only in Prometheus, by model: a team
// running into its limit looked, on every usage page, like a team that
// had gone quiet.
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
rpm := 1
if err := rt.store.PutLimits(t.Context(), store.Limits{Scope: store.ScopeKey, ScopeID: "key-1", RPM: &rpm}); err != nil {
t.Fatal(err)
}
body := map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
auth := map[string]string{"Authorization": "Bearer " + secret}
codes := []int{post(t, rt.Handler(), body, auth).Code, post(t, rt.Handler(), body, auth).Code}
if codes[1] != 429 {
t.Fatalf("the second request in the minute answered %v, want 429", codes)
}
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
stats, err := rt.store.StatsBy(t.Context(), store.GroupByProject, time.Now().Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
for _, s := range stats {
if s.Limited == 1 && s.Requests == 2 {
spend, _ := rt.store.SpendSince(t.Context(), time.Now().Add(-time.Hour))
if spend["key-1"] == 0 {
t.Error("the request that was served cost nothing")
}
return
}
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("the refused request was never counted against its project")
}
+132
View File
@@ -0,0 +1,132 @@
package store
import (
"context"
"fmt"
"strings"
"time"
)
// Quantiles are a duration's percentiles, in milliseconds, read from the
// histogram kept with the usage sums. Within a bucket the value is
// interpolated; above the last bucket's edge it is that edge, and Over says
// so. N is how many durations they are read from — zero is no reading.
type Quantiles struct {
P50, P95, P99 int64
N int64
Over bool
}
// GroupStats is one group's requests, how they ended, and how long they took.
type GroupStats struct {
ID string
Requests int64
Errors int64
// Limited is requests refused by a rate, token or budget limit.
Limited int64
Latency Quantiles
// TTFT is time to first token, for requests that streamed one.
TTFT Quantiles
}
// ErrorRate is the share of requests that failed, limits included, in percent.
func (g GroupStats) ErrorRate() float64 {
if g.Requests == 0 {
return 0
}
return 100 * float64(g.Errors) / float64(g.Requests)
}
// StatsBy is each group's request outcomes and latency percentiles since a
// moment, keyed by the group's id.
//
// Read from the hourly sums for a window of two days or more and from the
// five-minute ones otherwise, starting at the slice the moment falls in:
// percentiles are a shape, and a window starting up to one slice early does
// not change one. Budgets, which must be exact, are not read this way.
func (s *DB) StatsBy(ctx context.Context, g Grouping, since time.Time) (map[string]GroupStats, error) {
c, ok := groupColumns[g]
if !ok {
return nil, fmt.Errorf("usage cannot be grouped by %q", g)
}
t := tier5m
if time.Since(since) >= 48*time.Hour {
t = tier1h
}
n := len(LatencyBounds) + 1
var sums []string
for i := range n {
sums = append(sums, fmt.Sprintf("SUM(u.lat_%d)", i))
}
for i := range n {
sums = append(sums, fmt.Sprintf("SUM(u.ttft_%d)", i))
}
rows, err := s.db.QueryContext(ctx, `
SELECT `+c.id+`, SUM(u.requests), SUM(u.errors), SUM(u.limited), `+strings.Join(sums, ", ")+`
FROM `+t.table+` u`+c.joins+`
WHERE u.bucket >= ?
GROUP BY 1`, t.floor(since))
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]GroupStats{}
for rows.Next() {
var gs GroupStats
lat, ttft := make([]int64, n), make([]int64, n)
dst := []any{&gs.ID, &gs.Requests, &gs.Errors, &gs.Limited}
for i := range lat {
dst = append(dst, &lat[i])
}
for i := range ttft {
dst = append(dst, &ttft[i])
}
if err := rows.Scan(dst...); err != nil {
return nil, err
}
gs.Latency, gs.TTFT = quantiles(lat), quantiles(ttft)
out[gs.ID] = gs
}
return out, rows.Err()
}
// quantiles reads p50, p95 and p99 from counts per LatencyBounds bucket.
func quantiles(counts []int64) Quantiles {
var q Quantiles
for _, c := range counts {
q.N += c
}
if q.N == 0 {
return q
}
at := func(p float64) (int64, bool) {
rank := p * float64(q.N)
var seen float64
for i, c := range counts {
if c == 0 {
continue
}
if seen+float64(c) >= rank {
if i >= len(LatencyBounds) {
return LatencyBounds[len(LatencyBounds)-1], true
}
lo := int64(0)
if i > 0 {
lo = LatencyBounds[i-1]
}
hi := LatencyBounds[i]
frac := (rank - seen) / float64(c)
return lo + int64(frac*float64(hi-lo)), false
}
seen += float64(c)
}
return LatencyBounds[len(LatencyBounds)-1], true
}
var over bool
q.P50, _ = at(0.50)
q.P95, _ = at(0.95)
q.P99, over = at(0.99)
q.Over = over
return q
}
+61
View File
@@ -0,0 +1,61 @@
package store
import (
"context"
"testing"
"time"
)
func TestQuantilesAreReadFromTheHistogram(t *testing.T) {
// 100 durations: 90 at <=100 ms, 8 in (1000, 2500], 2 above two minutes.
counts := make([]int64, len(LatencyBounds)+1)
counts[0], counts[4], counts[len(counts)-1] = 90, 8, 2
q := quantiles(counts)
if q.N != 100 || q.P50 > 100 || q.P95 <= 1000 || q.P95 > 2500 || !q.Over {
t.Errorf("%+v: want p50 within 100 ms, p95 in (1000, 2500], p99 past the last edge", q)
}
if (quantiles(make([]int64, len(LatencyBounds)+1)) != Quantiles{}) {
t.Error("no durations read as some")
}
}
func TestStatsSayHowEachGroupFaredAndHowLongItTook(t *testing.T) {
onBoth(t, func(t *testing.T, s *DB) {
ctx := context.Background()
at := slice()
var batch []Usage
for i := range 20 {
batch = append(batch, Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big",
ServedModel: "big", Status: 200, LatencyMs: 4000, TTFTMs: int64(200 + i)})
}
batch = append(batch,
Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", ServedModel: "big", Status: 502, LatencyMs: 50},
Usage{At: at, KeyID: "k2", UserID: "bora", ProjectID: "p", Model: "small", ServedModel: "small", Status: 429},
)
if err := s.WriteUsage(ctx, batch); err != nil {
t.Fatal(err)
}
byUser, err := s.StatsBy(ctx, GroupByUser, at.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
a, b := byUser["ayse"], byUser["bora"]
if a.Requests != 21 || a.Errors != 1 || a.Limited != 0 || b.Limited != 1 || b.Errors != 1 {
t.Errorf("outcomes: ayse %+v, bora %+v", a, b)
}
if a.TTFT.N != 20 || a.TTFT.P95 <= 100 || a.TTFT.P95 > 250 || a.Latency.P95 <= 2500 || a.Latency.P95 > 5000 {
t.Errorf("ayse's first token %+v and latency %+v", a.TTFT, a.Latency)
}
byModel, err := s.StatsBy(ctx, GroupByModel, at.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if byModel["big"].Requests != 21 || byModel["small"].Requests != 1 {
t.Errorf("by model: %+v", byModel)
}
series, err := s.UsageBucketsBy(ctx, GroupByModel, at.Add(-time.Hour), time.Hour)
if err != nil || len(series) != 2 {
t.Fatalf("usage over time by model: %v, %+v", err, series)
}
})
}
@@ -21,7 +21,8 @@
-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000,
-- and above), which is what a percentile per team or model is read from after
-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose
-- first token was measured.
-- first token was measured. limited counts requests refused with 429 — a rate,
-- token or budget limit — which are recorded with no tokens and no cost.
CREATE TABLE usage_5m (
bucket BIGINT NOT NULL,
@@ -32,6 +33,7 @@ CREATE TABLE usage_5m (
served_model TEXT NOT NULL,
requests BIGINT NOT NULL DEFAULT 0,
errors BIGINT NOT NULL DEFAULT 0,
limited BIGINT NOT NULL DEFAULT 0,
prompt_tokens BIGINT NOT NULL DEFAULT 0,
completion_tokens BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0,
@@ -70,12 +72,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket);
CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket);
INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
@@ -92,6 +94,7 @@ CREATE TABLE usage_1h (
served_model TEXT NOT NULL,
requests BIGINT NOT NULL DEFAULT 0,
errors BIGINT NOT NULL DEFAULT 0,
limited BIGINT NOT NULL DEFAULT 0,
prompt_tokens BIGINT NOT NULL DEFAULT 0,
completion_tokens BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0,
@@ -130,12 +133,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket);
CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket);
INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
@@ -152,6 +155,7 @@ CREATE TABLE usage_1d (
served_model TEXT NOT NULL,
requests BIGINT NOT NULL DEFAULT 0,
errors BIGINT NOT NULL DEFAULT 0,
limited BIGINT NOT NULL DEFAULT 0,
prompt_tokens BIGINT NOT NULL DEFAULT 0,
completion_tokens BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0,
@@ -190,12 +194,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket);
CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket);
INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
@@ -20,7 +20,8 @@
-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000,
-- and above), which is what a percentile per team or model is read from after
-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose
-- first token was measured.
-- first token was measured. limited counts requests refused with 429 — a rate,
-- token or budget limit — which are recorded with no tokens and no cost.
CREATE TABLE usage_5m (
bucket INTEGER NOT NULL,
@@ -31,6 +32,7 @@ CREATE TABLE usage_5m (
served_model TEXT NOT NULL,
requests INTEGER NOT NULL DEFAULT 0,
errors INTEGER NOT NULL DEFAULT 0,
limited INTEGER NOT NULL DEFAULT 0,
prompt_tokens INTEGER NOT NULL DEFAULT 0,
completion_tokens INTEGER NOT NULL DEFAULT 0,
cost_micros INTEGER NOT NULL DEFAULT 0,
@@ -69,12 +71,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket);
CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket);
INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
@@ -91,6 +93,7 @@ CREATE TABLE usage_1h (
served_model TEXT NOT NULL,
requests INTEGER NOT NULL DEFAULT 0,
errors INTEGER NOT NULL DEFAULT 0,
limited INTEGER NOT NULL DEFAULT 0,
prompt_tokens INTEGER NOT NULL DEFAULT 0,
completion_tokens INTEGER NOT NULL DEFAULT 0,
cost_micros INTEGER NOT NULL DEFAULT 0,
@@ -129,12 +132,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket);
CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket);
INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
@@ -151,6 +154,7 @@ CREATE TABLE usage_1d (
served_model TEXT NOT NULL,
requests INTEGER NOT NULL DEFAULT 0,
errors INTEGER NOT NULL DEFAULT 0,
limited INTEGER NOT NULL DEFAULT 0,
prompt_tokens INTEGER NOT NULL DEFAULT 0,
completion_tokens INTEGER NOT NULL DEFAULT 0,
cost_micros INTEGER NOT NULL DEFAULT 0,
@@ -189,12 +193,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket);
CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket);
INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model,
requests, errors, prompt_tokens, completion_tokens, cost_micros,
requests, errors, limited, prompt_tokens, completion_tokens, cost_micros,
latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts,
lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10,
ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10)
SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model,
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END),
COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END),
SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros),
SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts),
SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END),
+11 -6
View File
@@ -68,9 +68,10 @@ type rollupKey struct {
// rollupRow is what a batch adds to one row.
type rollupRow struct {
requests, errors, prompt, completion, cost int64
latency, queue, ttft, ttftN, last int64
lat, ttftH []int64
requests, errors, limited, prompt int64
completion, cost int64
latency, queue, ttft, ttftN, last int64
lat, ttftH []int64
}
// rollupBatch sums a batch of records into the rows of one tier they add to.
@@ -91,6 +92,9 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey
if r.Status >= 400 {
row.errors++
}
if r.Status == 429 {
row.limited++
}
row.prompt += r.PromptTokens
row.completion += r.CompletionTokens
row.cost += r.CostMicros
@@ -111,7 +115,7 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey
var rollupColumns = func() []string {
n := len(LatencyBounds) + 1
cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model",
"requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros",
"requests", "errors", "limited", "prompt_tokens", "completion_tokens", "cost_micros",
"latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"}
for i := range n {
cols = append(cols, fmt.Sprintf("lat_%d", i))
@@ -151,7 +155,7 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error {
for _, k := range order {
r := rows[k]
args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel,
r.requests, r.errors, r.prompt, r.completion, r.cost,
r.requests, r.errors, r.limited, r.prompt, r.completion, r.cost,
r.latency, r.queue, r.ttft, r.ttftN, r.last}
for _, v := range r.lat {
args = append(args, v)
@@ -182,10 +186,11 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error {
func usageSince(since time.Time) (string, []any) {
from := millis(since)
to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since)
const cols = `key_id, user_id, project_id, model, served_model, requests, errors,
const cols = `key_id, user_id, project_id, model, served_model, requests, errors, limited,
prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts`
return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests,
CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors,
CASE WHEN status = 429 THEN 1 ELSE 0 END AS limited,
prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts
FROM usage WHERE ts >= ? AND ts < ?
UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ?
+9 -1
View File
@@ -21,6 +21,9 @@ const (
GroupByTeam Grouping = "team"
GroupByProject Grouping = "project"
GroupByUser Grouping = "user"
// GroupByModel is by the model that answered, which differs from the one
// asked for when a fallback served it.
GroupByModel Grouping = "model"
)
// Valid reports whether this is a grouping usage can be read by.
@@ -32,7 +35,7 @@ func (g Grouping) Valid() bool {
// Groupings lists them in the order a page should offer them: widest first,
// then the one that is not a level at all.
func Groupings() []Grouping {
return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser}
return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser, GroupByModel}
}
// UsageSeries is one group's usage over time.
@@ -87,8 +90,13 @@ var groupColumns = map[Grouping]struct{ id, name, joins string }{
// rather than hides.
GroupByUser: {"u.user_id", "COALESCE(us.email, '')", `
LEFT JOIN users us ON us.id = NULLIF(u.user_id, '')`},
GroupByModel: {servedModel, servedModel, ""},
}
// servedModel is what answered: the served model, or the one asked for on a
// row from before the two were told apart.
const servedModel = "CASE WHEN u.served_model != '' THEN u.served_model ELSE u.model END"
// UsageBucketsBy returns each group's usage over time.
//
// One query for every group rather than one per group: the page shows them
+13 -4
View File
@@ -66,10 +66,14 @@ type peopleDetail struct {
BudgetsBind bool
// Its use over the last week, and the totals beside it.
HasUsage bool
Requests, Tokens, Spend TimeSeries
Totals usageRow
Priced bool
HasUsage bool
Requests, Tokens, Spend TimeSeries
Totals usageRow
Priced bool
// Stats is how its requests ended and how long they took, over the
// same week; HasStats is false where the store does not say.
Stats store.GroupStats
HasStats bool
UsageWindow, QuotaWindow string
CSRF string
@@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) {
}
d.HasUsage = true
d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced()
if sr, ok := p.usageStore.(StatsReader); ok {
if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil {
d.Stats, d.HasStats = stats[d.ID], true
}
}
for _, s := range rows {
if s.ID == d.ID {
var table []usageRow
+24
View File
@@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) {
t.Error("a panel opened for an organization that does not exist")
}
}
func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) {
stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25,
TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150},
Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}}
u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{
store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200,
Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}},
}}
mux := peoplePage(t, oneTeam(), u)
table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ")
for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} {
if !strings.Contains(table, want) {
t.Errorf("the usage table lacks %q", want)
}
}
panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ")
for _, want := range []string{"<dt>Refused by a limit</dt>", "p95 900 ms", "p99 &gt; 120 s"} {
if !strings.Contains(panel, want) {
t.Errorf("the panel lacks %q", want)
}
}
}
+50
View File
@@ -2,6 +2,7 @@ package web
import (
"context"
"fmt"
"net/http"
"sort"
"time"
@@ -16,6 +17,35 @@ type UsageAdmin interface {
since time.Time, width time.Duration) ([]store.UsageSeries, error)
}
// StatsReader is how each group fared and how long its requests took; the
// store implements it. Optional: without it the usage table has no such
// columns.
type StatsReader interface {
StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error)
}
// pctile is a percentile from the usage histograms as a person reads it:
// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read.
func pctile(q store.Quantiles, which int) string {
if q.N == 0 {
return "—"
}
v := q.P50
switch which {
case 95:
v = q.P95
case 99:
v = q.P99
if q.Over {
return fmt.Sprintf("> %d s", v/1000)
}
}
if v < 1000 {
return fmt.Sprintf("%d ms", v)
}
return fmt.Sprintf("%.1f s", float64(v)/1000)
}
// usageWindow is one length of history the page offers, and the slice it is cut
// into.
//
@@ -54,11 +84,16 @@ type usageGroup struct {
// usageRow is one group's totals, shown under the charts.
type usageRow struct {
ID string
Name string
Requests int64
Tokens int64
Spend string
Errors int64
// Stats is how its requests ended and how long they took; HasStats is
// false where the store does not say.
Stats store.GroupStats
HasStats bool
}
type adminUsageData struct {
@@ -86,6 +121,8 @@ type adminUsageData struct {
// model on a GPU.
GPUHeld TimeSeries
GPUHours []gpuHoursRow
// HasStats adds outcomes and latency percentiles to the table.
HasStats bool
Error string
Version string
@@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced)
}
d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows)
if sr, ok := s.(StatsReader); ok {
stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span))
if err != nil {
p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err)
}
for i := range d.Rows {
d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil
}
d.HasStats = stats != nil
}
d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win)
return d
}
@@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string {
return "project"
case store.GroupByUser:
return "person"
case store.GroupByModel:
return "model"
default:
return string(g)
}
@@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table []
}), "", 4)
for _, u := range rows {
table = append(table, usageRow{
ID: u.ID,
Name: displayName(u),
Requests: u.Requests,
Tokens: u.Tokens(),
+9 -3
View File
@@ -15,9 +15,11 @@ import (
type fakeScopeSeries struct {
byGroup map[store.Grouping][]store.UsageSeries
asked []store.Grouping
width time.Duration
since time.Time
// stats, when set, is what StatsBy answers for every grouping.
stats map[string]store.GroupStats
asked []store.Grouping
width time.Duration
since time.Time
}
func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
@@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
return f.byGroup[g], nil
}
func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) {
return f.stats, nil
}
func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
+7
View File
@@ -140,6 +140,13 @@
<h3>Use, {{.UsageWindow}}</h3>
{{if .Totals.Name}}
<p class="pd-sub">{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}</p>
{{if and .HasStats .Stats.Requests}}{{with .Stats}}
<dl class="pd-limits">
<dt>Error rate</dt><dd{{if ge .ErrorRate 5.0}} class="q-warn"{{end}}>{{pct1 .ErrorRate}}%</dd>
<dt>Refused by a limit</dt><dd{{if .Limited}} class="q-warn"{{end}}>{{if .Limited}}{{compact .Limited}}{{else}}none{{end}}</dd>
<dt>First token</dt><dd>p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}</dd>
<dt>Latency</dt><dd>p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}</dd>
</dl>{{end}}{{end}}
{{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}}
{{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}}
{{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}}
+10 -1
View File
@@ -40,7 +40,11 @@
{{if .Rows}}
<table class="rows">
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th>
{{if .HasStats}}<th class="num" title="Failed requests, refusals by a limit included">Error rate</th>
<th class="num" title="Refused by a rate, token or budget limit">Limited</th>
<th class="num" title="Time to first token, for streamed requests">First token p50 / p95</th>
<th class="num">Latency p95</th>{{end}}</tr>
{{range .Rows}}
<tr>
<td>{{.Name}}</td>
@@ -48,6 +52,11 @@
<td class="num">{{compact .Tokens}}</td>
<td class="num">{{.Spend}}</td>
<td class="num">{{if .Errors}}{{.Errors}}{{else}}—{{end}}</td>
{{if $.HasStats}}{{with .Stats}}
<td class="num{{if ge .ErrorRate 5.0}} q-warn{{end}}">{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}</td>
<td class="num{{if .Limited}} q-warn{{end}}">{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}</td>
<td class="num">{{pctile .TTFT 50}} / {{pctile .TTFT 95}}</td>
<td class="num">{{pctile .Latency 95}}</td>{{end}}{{end}}
</tr>
{{end}}
</table>
+3
View File
@@ -22,6 +22,7 @@ import (
"io/fs"
"net/http"
"slices"
"strconv"
"strings"
"sync"
"time"
@@ -90,6 +91,8 @@ var funcs = template.FuncMap{
return strings.Join(v, ", ")
},
"latency": humanLatency,
"pctile": pctile,
"pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) },
"duration": func(sec int64) string {
return time.Duration(sec * int64(time.Second)).Round(time.Second).String()
},