Show each team's first-token and latency percentiles, errors and refusals
Percentiles are read from the histograms kept with the usage sums and shown per group on the usage tab and in the people page's panels. Requests refused by a limit are recorded as 429s against their owner, free of charge, so a team hitting its limits shows as such. Usage can be grouped by model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4454e1322c
commit
9f479b9a70
17 files changed
+418
-29
No files matched your search
@@ -66,10 +66,14 @@ type peopleDetail struct {
|
||||
BudgetsBind bool
|
||||
|
||||
// Its use over the last week, and the totals beside it.
|
||||
HasUsage bool
|
||||
Requests, Tokens, Spend TimeSeries
|
||||
Totals usageRow
|
||||
Priced bool
|
||||
HasUsage bool
|
||||
Requests, Tokens, Spend TimeSeries
|
||||
Totals usageRow
|
||||
Priced bool
|
||||
// Stats is how its requests ended and how long they took, over the
|
||||
// same week; HasStats is false where the store does not say.
|
||||
Stats store.GroupStats
|
||||
HasStats bool
|
||||
UsageWindow, QuotaWindow string
|
||||
|
||||
CSRF string
|
||||
@@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) {
|
||||
}
|
||||
d.HasUsage = true
|
||||
d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced()
|
||||
if sr, ok := p.usageStore.(StatsReader); ok {
|
||||
if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil {
|
||||
d.Stats, d.HasStats = stats[d.ID], true
|
||||
}
|
||||
}
|
||||
for _, s := range rows {
|
||||
if s.ID == d.ID {
|
||||
var table []usageRow
|
||||
|
||||
@@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) {
|
||||
t.Error("a panel opened for an organization that does not exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) {
|
||||
stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25,
|
||||
TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150},
|
||||
Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}}
|
||||
u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{
|
||||
store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200,
|
||||
Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}},
|
||||
}}
|
||||
mux := peoplePage(t, oneTeam(), u)
|
||||
|
||||
table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ")
|
||||
for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} {
|
||||
if !strings.Contains(table, want) {
|
||||
t.Errorf("the usage table lacks %q", want)
|
||||
}
|
||||
}
|
||||
panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ")
|
||||
for _, want := range []string{"<dt>Refused by a limit</dt>", "p95 900 ms", "p99 > 120 s"} {
|
||||
if !strings.Contains(panel, want) {
|
||||
t.Errorf("the panel lacks %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -16,6 +17,35 @@ type UsageAdmin interface {
|
||||
since time.Time, width time.Duration) ([]store.UsageSeries, error)
|
||||
}
|
||||
|
||||
// StatsReader is how each group fared and how long its requests took; the
|
||||
// store implements it. Optional: without it the usage table has no such
|
||||
// columns.
|
||||
type StatsReader interface {
|
||||
StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error)
|
||||
}
|
||||
|
||||
// pctile is a percentile from the usage histograms as a person reads it:
|
||||
// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read.
|
||||
func pctile(q store.Quantiles, which int) string {
|
||||
if q.N == 0 {
|
||||
return "—"
|
||||
}
|
||||
v := q.P50
|
||||
switch which {
|
||||
case 95:
|
||||
v = q.P95
|
||||
case 99:
|
||||
v = q.P99
|
||||
if q.Over {
|
||||
return fmt.Sprintf("> %d s", v/1000)
|
||||
}
|
||||
}
|
||||
if v < 1000 {
|
||||
return fmt.Sprintf("%d ms", v)
|
||||
}
|
||||
return fmt.Sprintf("%.1f s", float64(v)/1000)
|
||||
}
|
||||
|
||||
// usageWindow is one length of history the page offers, and the slice it is cut
|
||||
// into.
|
||||
//
|
||||
@@ -54,11 +84,16 @@ type usageGroup struct {
|
||||
|
||||
// usageRow is one group's totals, shown under the charts.
|
||||
type usageRow struct {
|
||||
ID string
|
||||
Name string
|
||||
Requests int64
|
||||
Tokens int64
|
||||
Spend string
|
||||
Errors int64
|
||||
// Stats is how its requests ended and how long they took; HasStats is
|
||||
// false where the store does not say.
|
||||
Stats store.GroupStats
|
||||
HasStats bool
|
||||
}
|
||||
|
||||
type adminUsageData struct {
|
||||
@@ -86,6 +121,8 @@ type adminUsageData struct {
|
||||
// model on a GPU.
|
||||
GPUHeld TimeSeries
|
||||
GPUHours []gpuHoursRow
|
||||
// HasStats adds outcomes and latency percentiles to the table.
|
||||
HasStats bool
|
||||
|
||||
Error string
|
||||
Version string
|
||||
@@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced)
|
||||
}
|
||||
|
||||
d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows)
|
||||
if sr, ok := s.(StatsReader); ok {
|
||||
stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span))
|
||||
if err != nil {
|
||||
p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err)
|
||||
}
|
||||
for i := range d.Rows {
|
||||
d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil
|
||||
}
|
||||
d.HasStats = stats != nil
|
||||
}
|
||||
d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win)
|
||||
return d
|
||||
}
|
||||
@@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string {
|
||||
return "project"
|
||||
case store.GroupByUser:
|
||||
return "person"
|
||||
case store.GroupByModel:
|
||||
return "model"
|
||||
default:
|
||||
return string(g)
|
||||
}
|
||||
@@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table []
|
||||
}), "", 4)
|
||||
for _, u := range rows {
|
||||
table = append(table, usageRow{
|
||||
ID: u.ID,
|
||||
Name: displayName(u),
|
||||
Requests: u.Requests,
|
||||
Tokens: u.Tokens(),
|
||||
|
||||
@@ -15,9 +15,11 @@ import (
|
||||
|
||||
type fakeScopeSeries struct {
|
||||
byGroup map[store.Grouping][]store.UsageSeries
|
||||
asked []store.Grouping
|
||||
width time.Duration
|
||||
since time.Time
|
||||
// stats, when set, is what StatsBy answers for every grouping.
|
||||
stats map[string]store.GroupStats
|
||||
asked []store.Grouping
|
||||
width time.Duration
|
||||
since time.Time
|
||||
}
|
||||
|
||||
func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
|
||||
@@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
|
||||
return f.byGroup[g], nil
|
||||
}
|
||||
|
||||
func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) {
|
||||
return f.stats, nil
|
||||
}
|
||||
|
||||
func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux {
|
||||
t.Helper()
|
||||
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
|
||||
|
||||
@@ -140,6 +140,13 @@
|
||||
<h3>Use, {{.UsageWindow}}</h3>
|
||||
{{if .Totals.Name}}
|
||||
<p class="pd-sub">{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}</p>
|
||||
{{if and .HasStats .Stats.Requests}}{{with .Stats}}
|
||||
<dl class="pd-limits">
|
||||
<dt>Error rate</dt><dd{{if ge .ErrorRate 5.0}} class="q-warn"{{end}}>{{pct1 .ErrorRate}}%</dd>
|
||||
<dt>Refused by a limit</dt><dd{{if .Limited}} class="q-warn"{{end}}>{{if .Limited}}{{compact .Limited}}{{else}}none{{end}}</dd>
|
||||
<dt>First token</dt><dd>p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}</dd>
|
||||
<dt>Latency</dt><dd>p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}</dd>
|
||||
</dl>{{end}}{{end}}
|
||||
{{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}}
|
||||
{{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}}
|
||||
{{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}}
|
||||
|
||||
@@ -40,7 +40,11 @@
|
||||
|
||||
{{if .Rows}}
|
||||
<table class="rows">
|
||||
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
|
||||
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th>
|
||||
{{if .HasStats}}<th class="num" title="Failed requests, refusals by a limit included">Error rate</th>
|
||||
<th class="num" title="Refused by a rate, token or budget limit">Limited</th>
|
||||
<th class="num" title="Time to first token, for streamed requests">First token p50 / p95</th>
|
||||
<th class="num">Latency p95</th>{{end}}</tr>
|
||||
{{range .Rows}}
|
||||
<tr>
|
||||
<td>{{.Name}}</td>
|
||||
@@ -48,6 +52,11 @@
|
||||
<td class="num">{{compact .Tokens}}</td>
|
||||
<td class="num">{{.Spend}}</td>
|
||||
<td class="num">{{if .Errors}}{{.Errors}}{{else}}—{{end}}</td>
|
||||
{{if $.HasStats}}{{with .Stats}}
|
||||
<td class="num{{if ge .ErrorRate 5.0}} q-warn{{end}}">{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}</td>
|
||||
<td class="num{{if .Limited}} q-warn{{end}}">{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{pctile .TTFT 50}} / {{pctile .TTFT 95}}</td>
|
||||
<td class="num">{{pctile .Latency 95}}</td>{{end}}{{end}}
|
||||
</tr>
|
||||
{{end}}
|
||||
</table>
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -90,6 +91,8 @@ var funcs = template.FuncMap{
|
||||
return strings.Join(v, ", ")
|
||||
},
|
||||
"latency": humanLatency,
|
||||
"pctile": pctile,
|
||||
"pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) },
|
||||
"duration": func(sec int64) string {
|
||||
return time.Duration(sec * int64(time.Second)).Round(time.Second).String()
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user