diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d2de1..f39601e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,21 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration sums — but `ctl request` finds nothing older. Unset keeps every record, as before. +### How each team's requests fared, not only what they cost + +- **Percentiles from the sums.** First token and latency, p50, p95 and p99, + per organization, team, project, person or model, read from the histograms + kept with the usage sums: on the usage tab's table, and in a branch's or + person's panel on the people page, beside the error rate and requests + refused by a limit. +- **Refused requests are counted against their owner.** A request a rate, + token or budget limit refused was counted only in Prometheus, by model; a + team running into its limit looked like a team that had gone quiet. It is + now recorded as a 429 against its key, person and project — no tokens, no + cost — and shown as "Limited". +- **Usage by model** is a grouping of the usage tab, by the model that + answered. + ### People, limits and usage are one page Managing a team meant its members on the people page, its quota on the quotas diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index 66c42e3..a6ada98 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -316,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { } } +// RecordRefused queues the row of a request a limit refused: nothing is +// charged — it used nothing — but it is counted, so a team running into its +// limits shows as one rather than as a team that went quiet. +func (a *Accountant) RecordRefused(u store.Usage) { + if u.KeyID == "" || a.recorder == nil { + return + } + u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0 + a.recorder.Record(u) +} + // Status returns what a key has used, for headers and `ctl`. func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index b758f11..e48a60c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -537,6 +537,11 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.acct.RecordRefused(store.Usage{ + At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, + Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests, + LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, + }) r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start)) } diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 40d06ae..408f681 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -188,3 +188,39 @@ func TestAnIDThatIsNotOneIsReplaced(t *testing.T) { t.Error("a plain id was not kept") } } + +func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) { + // Refused requests were counted only in Prometheus, by model: a team + // running into its limit looked, on every usage page, like a team that + // had gone quiet. + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + rpm := 1 + if err := rt.store.PutLimits(t.Context(), store.Limits{Scope: store.ScopeKey, ScopeID: "key-1", RPM: &rpm}); err != nil { + t.Fatal(err) + } + body := map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "hi"}}} + auth := map[string]string{"Authorization": "Bearer " + secret} + codes := []int{post(t, rt.Handler(), body, auth).Code, post(t, rt.Handler(), body, auth).Code} + if codes[1] != 429 { + t.Fatalf("the second request in the minute answered %v, want 429", codes) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + stats, err := rt.store.StatsBy(t.Context(), store.GroupByProject, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Limited == 1 && s.Requests == 2 { + spend, _ := rt.store.SpendSince(t.Context(), time.Now().Add(-time.Hour)) + if spend["key-1"] == 0 { + t.Error("the request that was served cost nothing") + } + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the refused request was never counted against its project") +} diff --git a/internal/store/latency.go b/internal/store/latency.go new file mode 100644 index 0000000..e3699a3 --- /dev/null +++ b/internal/store/latency.go @@ -0,0 +1,132 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Quantiles are a duration's percentiles, in milliseconds, read from the +// histogram kept with the usage sums. Within a bucket the value is +// interpolated; above the last bucket's edge it is that edge, and Over says +// so. N is how many durations they are read from — zero is no reading. +type Quantiles struct { + P50, P95, P99 int64 + N int64 + Over bool +} + +// GroupStats is one group's requests, how they ended, and how long they took. +type GroupStats struct { + ID string + Requests int64 + Errors int64 + // Limited is requests refused by a rate, token or budget limit. + Limited int64 + Latency Quantiles + // TTFT is time to first token, for requests that streamed one. + TTFT Quantiles +} + +// ErrorRate is the share of requests that failed, limits included, in percent. +func (g GroupStats) ErrorRate() float64 { + if g.Requests == 0 { + return 0 + } + return 100 * float64(g.Errors) / float64(g.Requests) +} + +// StatsBy is each group's request outcomes and latency percentiles since a +// moment, keyed by the group's id. +// +// Read from the hourly sums for a window of two days or more and from the +// five-minute ones otherwise, starting at the slice the moment falls in: +// percentiles are a shape, and a window starting up to one slice early does +// not change one. Budgets, which must be exact, are not read this way. +func (s *DB) StatsBy(ctx context.Context, g Grouping, since time.Time) (map[string]GroupStats, error) { + c, ok := groupColumns[g] + if !ok { + return nil, fmt.Errorf("usage cannot be grouped by %q", g) + } + t := tier5m + if time.Since(since) >= 48*time.Hour { + t = tier1h + } + n := len(LatencyBounds) + 1 + var sums []string + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.lat_%d)", i)) + } + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.ttft_%d)", i)) + } + rows, err := s.db.QueryContext(ctx, ` + SELECT `+c.id+`, SUM(u.requests), SUM(u.errors), SUM(u.limited), `+strings.Join(sums, ", ")+` + FROM `+t.table+` u`+c.joins+` + WHERE u.bucket >= ? + GROUP BY 1`, t.floor(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[string]GroupStats{} + for rows.Next() { + var gs GroupStats + lat, ttft := make([]int64, n), make([]int64, n) + dst := []any{&gs.ID, &gs.Requests, &gs.Errors, &gs.Limited} + for i := range lat { + dst = append(dst, &lat[i]) + } + for i := range ttft { + dst = append(dst, &ttft[i]) + } + if err := rows.Scan(dst...); err != nil { + return nil, err + } + gs.Latency, gs.TTFT = quantiles(lat), quantiles(ttft) + out[gs.ID] = gs + } + return out, rows.Err() +} + +// quantiles reads p50, p95 and p99 from counts per LatencyBounds bucket. +func quantiles(counts []int64) Quantiles { + var q Quantiles + for _, c := range counts { + q.N += c + } + if q.N == 0 { + return q + } + at := func(p float64) (int64, bool) { + rank := p * float64(q.N) + var seen float64 + for i, c := range counts { + if c == 0 { + continue + } + if seen+float64(c) >= rank { + if i >= len(LatencyBounds) { + return LatencyBounds[len(LatencyBounds)-1], true + } + lo := int64(0) + if i > 0 { + lo = LatencyBounds[i-1] + } + hi := LatencyBounds[i] + frac := (rank - seen) / float64(c) + return lo + int64(frac*float64(hi-lo)), false + } + seen += float64(c) + } + return LatencyBounds[len(LatencyBounds)-1], true + } + var over bool + q.P50, _ = at(0.50) + q.P95, _ = at(0.95) + q.P99, over = at(0.99) + q.Over = over + return q +} diff --git a/internal/store/latency_test.go b/internal/store/latency_test.go new file mode 100644 index 0000000..1fabc66 --- /dev/null +++ b/internal/store/latency_test.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "testing" + "time" +) + +func TestQuantilesAreReadFromTheHistogram(t *testing.T) { + // 100 durations: 90 at <=100 ms, 8 in (1000, 2500], 2 above two minutes. + counts := make([]int64, len(LatencyBounds)+1) + counts[0], counts[4], counts[len(counts)-1] = 90, 8, 2 + q := quantiles(counts) + if q.N != 100 || q.P50 > 100 || q.P95 <= 1000 || q.P95 > 2500 || !q.Over { + t.Errorf("%+v: want p50 within 100 ms, p95 in (1000, 2500], p99 past the last edge", q) + } + if (quantiles(make([]int64, len(LatencyBounds)+1)) != Quantiles{}) { + t.Error("no durations read as some") + } +} + +func TestStatsSayHowEachGroupFaredAndHowLongItTook(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + var batch []Usage + for i := range 20 { + batch = append(batch, Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", + ServedModel: "big", Status: 200, LatencyMs: 4000, TTFTMs: int64(200 + i)}) + } + batch = append(batch, + Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", ServedModel: "big", Status: 502, LatencyMs: 50}, + Usage{At: at, KeyID: "k2", UserID: "bora", ProjectID: "p", Model: "small", ServedModel: "small", Status: 429}, + ) + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + byUser, err := s.StatsBy(ctx, GroupByUser, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + a, b := byUser["ayse"], byUser["bora"] + if a.Requests != 21 || a.Errors != 1 || a.Limited != 0 || b.Limited != 1 || b.Errors != 1 { + t.Errorf("outcomes: ayse %+v, bora %+v", a, b) + } + if a.TTFT.N != 20 || a.TTFT.P95 <= 100 || a.TTFT.P95 > 250 || a.Latency.P95 <= 2500 || a.Latency.P95 > 5000 { + t.Errorf("ayse's first token %+v and latency %+v", a.TTFT, a.Latency) + } + byModel, err := s.StatsBy(ctx, GroupByModel, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if byModel["big"].Requests != 21 || byModel["small"].Requests != 1 { + t.Errorf("by model: %+v", byModel) + } + series, err := s.UsageBucketsBy(ctx, GroupByModel, at.Add(-time.Hour), time.Hour) + if err != nil || len(series) != 2 { + t.Fatalf("usage over time by model: %v, %+v", err, series) + } + }) +} diff --git a/internal/store/migrations/postgres/0015_usage_rollup.sql b/internal/store/migrations/postgres/0015_usage_rollup.sql index 39abcf7..9121268 100644 --- a/internal/store/migrations/postgres/0015_usage_rollup.sql +++ b/internal/store/migrations/postgres/0015_usage_rollup.sql @@ -21,7 +21,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket BIGINT NOT NULL, @@ -32,6 +33,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -70,12 +72,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -92,6 +94,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -130,12 +133,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -152,6 +155,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -190,12 +194,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/migrations/sqlite/0015_usage_rollup.sql b/internal/store/migrations/sqlite/0015_usage_rollup.sql index 17d9ae9..abe08f8 100644 --- a/internal/store/migrations/sqlite/0015_usage_rollup.sql +++ b/internal/store/migrations/sqlite/0015_usage_rollup.sql @@ -20,7 +20,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket INTEGER NOT NULL, @@ -31,6 +32,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -69,12 +71,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -91,6 +93,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -129,12 +132,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -151,6 +154,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -189,12 +193,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/rollup.go b/internal/store/rollup.go index 4333ee2..fa0e296 100644 --- a/internal/store/rollup.go +++ b/internal/store/rollup.go @@ -68,9 +68,10 @@ type rollupKey struct { // rollupRow is what a batch adds to one row. type rollupRow struct { - requests, errors, prompt, completion, cost int64 - latency, queue, ttft, ttftN, last int64 - lat, ttftH []int64 + requests, errors, limited, prompt int64 + completion, cost int64 + latency, queue, ttft, ttftN, last int64 + lat, ttftH []int64 } // rollupBatch sums a batch of records into the rows of one tier they add to. @@ -91,6 +92,9 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey if r.Status >= 400 { row.errors++ } + if r.Status == 429 { + row.limited++ + } row.prompt += r.PromptTokens row.completion += r.CompletionTokens row.cost += r.CostMicros @@ -111,7 +115,7 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey var rollupColumns = func() []string { n := len(LatencyBounds) + 1 cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model", - "requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros", + "requests", "errors", "limited", "prompt_tokens", "completion_tokens", "cost_micros", "latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"} for i := range n { cols = append(cols, fmt.Sprintf("lat_%d", i)) @@ -151,7 +155,7 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { for _, k := range order { r := rows[k] args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel, - r.requests, r.errors, r.prompt, r.completion, r.cost, + r.requests, r.errors, r.limited, r.prompt, r.completion, r.cost, r.latency, r.queue, r.ttft, r.ttftN, r.last} for _, v := range r.lat { args = append(args, v) @@ -182,10 +186,11 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { func usageSince(since time.Time) (string, []any) { from := millis(since) to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since) - const cols = `key_id, user_id, project_id, model, served_model, requests, errors, + const cols = `key_id, user_id, project_id, model, served_model, requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts` return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests, CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors, + CASE WHEN status = 429 THEN 1 ELSE 0 END AS limited, prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts FROM usage WHERE ts >= ? AND ts < ? UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ? diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go index 398adf7..d5e6eee 100644 --- a/internal/store/usageseries.go +++ b/internal/store/usageseries.go @@ -21,6 +21,9 @@ const ( GroupByTeam Grouping = "team" GroupByProject Grouping = "project" GroupByUser Grouping = "user" + // GroupByModel is by the model that answered, which differs from the one + // asked for when a fallback served it. + GroupByModel Grouping = "model" ) // Valid reports whether this is a grouping usage can be read by. @@ -32,7 +35,7 @@ func (g Grouping) Valid() bool { // Groupings lists them in the order a page should offer them: widest first, // then the one that is not a level at all. func Groupings() []Grouping { - return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser} + return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser, GroupByModel} } // UsageSeries is one group's usage over time. @@ -87,8 +90,13 @@ var groupColumns = map[Grouping]struct{ id, name, joins string }{ // rather than hides. GroupByUser: {"u.user_id", "COALESCE(us.email, '')", ` LEFT JOIN users us ON us.id = NULLIF(u.user_id, '')`}, + GroupByModel: {servedModel, servedModel, ""}, } +// servedModel is what answered: the served model, or the one asked for on a +// row from before the two were told apart. +const servedModel = "CASE WHEN u.served_model != '' THEN u.served_model ELSE u.model END" + // UsageBucketsBy returns each group's usage over time. // // One query for every group rather than one per group: the page shows them diff --git a/internal/web/admin_people.go b/internal/web/admin_people.go index 7cfe35d..5eae074 100644 --- a/internal/web/admin_people.go +++ b/internal/web/admin_people.go @@ -66,10 +66,14 @@ type peopleDetail struct { BudgetsBind bool // Its use over the last week, and the totals beside it. - HasUsage bool - Requests, Tokens, Spend TimeSeries - Totals usageRow - Priced bool + HasUsage bool + Requests, Tokens, Spend TimeSeries + Totals usageRow + Priced bool + // Stats is how its requests ended and how long they took, over the + // same week; HasStats is false where the store does not say. + Stats store.GroupStats + HasStats bool UsageWindow, QuotaWindow string CSRF string @@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) { } d.HasUsage = true d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced() + if sr, ok := p.usageStore.(StatsReader); ok { + if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil { + d.Stats, d.HasStats = stats[d.ID], true + } + } for _, s := range rows { if s.ID == d.ID { var table []usageRow diff --git a/internal/web/admin_people_test.go b/internal/web/admin_people_test.go index c9e1fed..4039f03 100644 --- a/internal/web/admin_people_test.go +++ b/internal/web/admin_people_test.go @@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) { t.Error("a panel opened for an organization that does not exist") } } + +func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) { + stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25, + TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150}, + Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}} + u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200, + Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}}, + }} + mux := peoplePage(t, oneTeam(), u) + + table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ") + for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} { + if !strings.Contains(table, want) { + t.Errorf("the usage table lacks %q", want) + } + } + panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ") + for _, want := range []string{"
Refused by a limit
", "p95 900 ms", "p99 > 120 s"} { + if !strings.Contains(panel, want) { + t.Errorf("the panel lacks %q", want) + } + } +} diff --git a/internal/web/admin_usage.go b/internal/web/admin_usage.go index 4f0d0b2..ebcf0a7 100644 --- a/internal/web/admin_usage.go +++ b/internal/web/admin_usage.go @@ -2,6 +2,7 @@ package web import ( "context" + "fmt" "net/http" "sort" "time" @@ -16,6 +17,35 @@ type UsageAdmin interface { since time.Time, width time.Duration) ([]store.UsageSeries, error) } +// StatsReader is how each group fared and how long its requests took; the +// store implements it. Optional: without it the usage table has no such +// columns. +type StatsReader interface { + StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error) +} + +// pctile is a percentile from the usage histograms as a person reads it: +// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read. +func pctile(q store.Quantiles, which int) string { + if q.N == 0 { + return "—" + } + v := q.P50 + switch which { + case 95: + v = q.P95 + case 99: + v = q.P99 + if q.Over { + return fmt.Sprintf("> %d s", v/1000) + } + } + if v < 1000 { + return fmt.Sprintf("%d ms", v) + } + return fmt.Sprintf("%.1f s", float64(v)/1000) +} + // usageWindow is one length of history the page offers, and the slice it is cut // into. // @@ -54,11 +84,16 @@ type usageGroup struct { // usageRow is one group's totals, shown under the charts. type usageRow struct { + ID string Name string Requests int64 Tokens int64 Spend string Errors int64 + // Stats is how its requests ended and how long they took; HasStats is + // false where the store does not say. + Stats store.GroupStats + HasStats bool } type adminUsageData struct { @@ -86,6 +121,8 @@ type adminUsageData struct { // model on a GPU. GPUHeld TimeSeries GPUHours []gpuHoursRow + // HasStats adds outcomes and latency percentiles to the table. + HasStats bool Error string Version string @@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced) } d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows) + if sr, ok := s.(StatsReader); ok { + stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span)) + if err != nil { + p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err) + } + for i := range d.Rows { + d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil + } + d.HasStats = stats != nil + } d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win) return d } @@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string { return "project" case store.GroupByUser: return "person" + case store.GroupByModel: + return "model" default: return string(g) } @@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table [] }), "", 4) for _, u := range rows { table = append(table, usageRow{ + ID: u.ID, Name: displayName(u), Requests: u.Requests, Tokens: u.Tokens(), diff --git a/internal/web/admin_usage_test.go b/internal/web/admin_usage_test.go index c157d15..72de07c 100644 --- a/internal/web/admin_usage_test.go +++ b/internal/web/admin_usage_test.go @@ -15,9 +15,11 @@ import ( type fakeScopeSeries struct { byGroup map[store.Grouping][]store.UsageSeries - asked []store.Grouping - width time.Duration - since time.Time + // stats, when set, is what StatsBy answers for every grouping. + stats map[string]store.GroupStats + asked []store.Grouping + width time.Duration + since time.Time } func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, @@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, return f.byGroup[g], nil } +func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) { + return f.stats, nil +} + func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} diff --git a/internal/web/templates/admin_people.html b/internal/web/templates/admin_people.html index bf53062..51357b6 100644 --- a/internal/web/templates/admin_people.html +++ b/internal/web/templates/admin_people.html @@ -140,6 +140,13 @@

Use, {{.UsageWindow}}

{{if .Totals.Name}}

{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}

+{{if and .HasStats .Stats.Requests}}{{with .Stats}} +
+
Error rate
{{pct1 .ErrorRate}}% +
Refused by a limit
{{if .Limited}}{{compact .Limited}}{{else}}none{{end}} +
First token
p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}
+
Latency
p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}
+
{{end}}{{end}} {{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}} {{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}} {{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}} diff --git a/internal/web/templates/admin_usage.html b/internal/web/templates/admin_usage.html index 456960b..ad6d637 100644 --- a/internal/web/templates/admin_usage.html +++ b/internal/web/templates/admin_usage.html @@ -40,7 +40,11 @@ {{if .Rows}} - + + {{if .HasStats}} + + + {{end}} {{range .Rows}} @@ -48,6 +52,11 @@ + {{if $.HasStats}}{{with .Stats}} + + + + {{end}}{{end}} {{end}}
GroupRequestsTokensSpendErrors
GroupRequestsTokensSpendErrorsError rateLimitedFirst token p50 / p95Latency p95
{{.Name}}{{compact .Tokens}} {{.Spend}} {{if .Errors}}{{.Errors}}{{else}}—{{end}}{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}{{pctile .TTFT 50}} / {{pctile .TTFT 95}}{{pctile .Latency 95}}
diff --git a/internal/web/web.go b/internal/web/web.go index 8f44fe9..e870014 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -22,6 +22,7 @@ import ( "io/fs" "net/http" "slices" + "strconv" "strings" "sync" "time" @@ -90,6 +91,8 @@ var funcs = template.FuncMap{ return strings.Join(v, ", ") }, "latency": humanLatency, + "pctile": pctile, + "pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) }, "duration": func(sec int64) string { return time.Duration(sec * int64(time.Second)).Round(time.Second).String() },