Show each team's first-token and latency percentiles, errors and refusals

Percentiles are read from the histograms kept with the usage sums and shown
per group on the usage tab and in the people page's panels. Requests refused
by a limit are recorded as 429s against their owner, free of charge, so a team
hitting its limits shows as such. Usage can be grouped by model.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 19:50:16 +02:00
1 parent 4454e1322c
commit 9f479b9a70
17 files changed
+418 -29

No files matched your search

+13 -4
View File
@@ -66,10 +66,14 @@ type peopleDetail struct {
BudgetsBind bool
// Its use over the last week, and the totals beside it.
HasUsage bool
Requests, Tokens, Spend TimeSeries
Totals usageRow
Priced bool
HasUsage bool
Requests, Tokens, Spend TimeSeries
Totals usageRow
Priced bool
// Stats is how its requests ended and how long they took, over the
// same week; HasStats is false where the store does not say.
Stats store.GroupStats
HasStats bool
UsageWindow, QuotaWindow string
CSRF string
@@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) {
}
d.HasUsage = true
d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced()
if sr, ok := p.usageStore.(StatsReader); ok {
if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil {
d.Stats, d.HasStats = stats[d.ID], true
}
}
for _, s := range rows {
if s.ID == d.ID {
var table []usageRow
+24
View File
@@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) {
t.Error("a panel opened for an organization that does not exist")
}
}
func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) {
stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25,
TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150},
Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}}
u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{
store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200,
Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}},
}}
mux := peoplePage(t, oneTeam(), u)
table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ")
for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} {
if !strings.Contains(table, want) {
t.Errorf("the usage table lacks %q", want)
}
}
panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ")
for _, want := range []string{"<dt>Refused by a limit</dt>", "p95 900 ms", "p99 &gt; 120 s"} {
if !strings.Contains(panel, want) {
t.Errorf("the panel lacks %q", want)
}
}
}
+50
View File
@@ -2,6 +2,7 @@ package web
import (
"context"
"fmt"
"net/http"
"sort"
"time"
@@ -16,6 +17,35 @@ type UsageAdmin interface {
since time.Time, width time.Duration) ([]store.UsageSeries, error)
}
// StatsReader is how each group fared and how long its requests took; the
// store implements it. Optional: without it the usage table has no such
// columns.
type StatsReader interface {
StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error)
}
// pctile is a percentile from the usage histograms as a person reads it:
// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read.
func pctile(q store.Quantiles, which int) string {
if q.N == 0 {
return "—"
}
v := q.P50
switch which {
case 95:
v = q.P95
case 99:
v = q.P99
if q.Over {
return fmt.Sprintf("> %d s", v/1000)
}
}
if v < 1000 {
return fmt.Sprintf("%d ms", v)
}
return fmt.Sprintf("%.1f s", float64(v)/1000)
}
// usageWindow is one length of history the page offers, and the slice it is cut
// into.
//
@@ -54,11 +84,16 @@ type usageGroup struct {
// usageRow is one group's totals, shown under the charts.
type usageRow struct {
ID string
Name string
Requests int64
Tokens int64
Spend string
Errors int64
// Stats is how its requests ended and how long they took; HasStats is
// false where the store does not say.
Stats store.GroupStats
HasStats bool
}
type adminUsageData struct {
@@ -86,6 +121,8 @@ type adminUsageData struct {
// model on a GPU.
GPUHeld TimeSeries
GPUHours []gpuHoursRow
// HasStats adds outcomes and latency percentiles to the table.
HasStats bool
Error string
Version string
@@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced)
}
d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows)
if sr, ok := s.(StatsReader); ok {
stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span))
if err != nil {
p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err)
}
for i := range d.Rows {
d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil
}
d.HasStats = stats != nil
}
d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win)
return d
}
@@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string {
return "project"
case store.GroupByUser:
return "person"
case store.GroupByModel:
return "model"
default:
return string(g)
}
@@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table []
}), "", 4)
for _, u := range rows {
table = append(table, usageRow{
ID: u.ID,
Name: displayName(u),
Requests: u.Requests,
Tokens: u.Tokens(),
+9 -3
View File
@@ -15,9 +15,11 @@ import (
type fakeScopeSeries struct {
byGroup map[store.Grouping][]store.UsageSeries
asked []store.Grouping
width time.Duration
since time.Time
// stats, when set, is what StatsBy answers for every grouping.
stats map[string]store.GroupStats
asked []store.Grouping
width time.Duration
since time.Time
}
func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
@@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
return f.byGroup[g], nil
}
func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) {
return f.stats, nil
}
func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
+7
View File
@@ -140,6 +140,13 @@
<h3>Use, {{.UsageWindow}}</h3>
{{if .Totals.Name}}
<p class="pd-sub">{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}</p>
{{if and .HasStats .Stats.Requests}}{{with .Stats}}
<dl class="pd-limits">
<dt>Error rate</dt><dd{{if ge .ErrorRate 5.0}} class="q-warn"{{end}}>{{pct1 .ErrorRate}}%</dd>
<dt>Refused by a limit</dt><dd{{if .Limited}} class="q-warn"{{end}}>{{if .Limited}}{{compact .Limited}}{{else}}none{{end}}</dd>
<dt>First token</dt><dd>p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}</dd>
<dt>Latency</dt><dd>p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}</dd>
</dl>{{end}}{{end}}
{{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}}
{{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}}
{{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}}
+10 -1
View File
@@ -40,7 +40,11 @@
{{if .Rows}}
<table class="rows">
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th>
{{if .HasStats}}<th class="num" title="Failed requests, refusals by a limit included">Error rate</th>
<th class="num" title="Refused by a rate, token or budget limit">Limited</th>
<th class="num" title="Time to first token, for streamed requests">First token p50 / p95</th>
<th class="num">Latency p95</th>{{end}}</tr>
{{range .Rows}}
<tr>
<td>{{.Name}}</td>
@@ -48,6 +52,11 @@
<td class="num">{{compact .Tokens}}</td>
<td class="num">{{.Spend}}</td>
<td class="num">{{if .Errors}}{{.Errors}}{{else}}—{{end}}</td>
{{if $.HasStats}}{{with .Stats}}
<td class="num{{if ge .ErrorRate 5.0}} q-warn{{end}}">{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}</td>
<td class="num{{if .Limited}} q-warn{{end}}">{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}</td>
<td class="num">{{pctile .TTFT 50}} / {{pctile .TTFT 95}}</td>
<td class="num">{{pctile .Latency 95}}</td>{{end}}{{end}}
</tr>
{{end}}
</table>
+3
View File
@@ -22,6 +22,7 @@ import (
"io/fs"
"net/http"
"slices"
"strconv"
"strings"
"sync"
"time"
@@ -90,6 +91,8 @@ var funcs = template.FuncMap{
return strings.Join(v, ", ")
},
"latency": humanLatency,
"pctile": pctile,
"pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) },
"duration": func(sec int64) string {
return time.Duration(sec * int64(time.Second)).Round(time.Second).String()
},