Show each team's first-token and latency percentiles, errors and refusals
Percentiles are read from the histograms kept with the usage sums and shown per group on the usage tab and in the people page's panels. Requests refused by a limit are recorded as 429s against their owner, free of charge, so a team hitting its limits shows as such. Usage can be grouped by model. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4454e1322c
commit
9f479b9a70
17 files changed
+418
-29
No files matched your search
@@ -2,6 +2,7 @@ package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -16,6 +17,35 @@ type UsageAdmin interface {
|
||||
since time.Time, width time.Duration) ([]store.UsageSeries, error)
|
||||
}
|
||||
|
||||
// StatsReader is how each group fared and how long its requests took; the
|
||||
// store implements it. Optional: without it the usage table has no such
|
||||
// columns.
|
||||
type StatsReader interface {
|
||||
StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error)
|
||||
}
|
||||
|
||||
// pctile is a percentile from the usage histograms as a person reads it:
|
||||
// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read.
|
||||
func pctile(q store.Quantiles, which int) string {
|
||||
if q.N == 0 {
|
||||
return "—"
|
||||
}
|
||||
v := q.P50
|
||||
switch which {
|
||||
case 95:
|
||||
v = q.P95
|
||||
case 99:
|
||||
v = q.P99
|
||||
if q.Over {
|
||||
return fmt.Sprintf("> %d s", v/1000)
|
||||
}
|
||||
}
|
||||
if v < 1000 {
|
||||
return fmt.Sprintf("%d ms", v)
|
||||
}
|
||||
return fmt.Sprintf("%.1f s", float64(v)/1000)
|
||||
}
|
||||
|
||||
// usageWindow is one length of history the page offers, and the slice it is cut
|
||||
// into.
|
||||
//
|
||||
@@ -54,11 +84,16 @@ type usageGroup struct {
|
||||
|
||||
// usageRow is one group's totals, shown under the charts.
|
||||
type usageRow struct {
|
||||
ID string
|
||||
Name string
|
||||
Requests int64
|
||||
Tokens int64
|
||||
Spend string
|
||||
Errors int64
|
||||
// Stats is how its requests ended and how long they took; HasStats is
|
||||
// false where the store does not say.
|
||||
Stats store.GroupStats
|
||||
HasStats bool
|
||||
}
|
||||
|
||||
type adminUsageData struct {
|
||||
@@ -86,6 +121,8 @@ type adminUsageData struct {
|
||||
// model on a GPU.
|
||||
GPUHeld TimeSeries
|
||||
GPUHours []gpuHoursRow
|
||||
// HasStats adds outcomes and latency percentiles to the table.
|
||||
HasStats bool
|
||||
|
||||
Error string
|
||||
Version string
|
||||
@@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced)
|
||||
}
|
||||
|
||||
d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows)
|
||||
if sr, ok := s.(StatsReader); ok {
|
||||
stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span))
|
||||
if err != nil {
|
||||
p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err)
|
||||
}
|
||||
for i := range d.Rows {
|
||||
d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil
|
||||
}
|
||||
d.HasStats = stats != nil
|
||||
}
|
||||
d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win)
|
||||
return d
|
||||
}
|
||||
@@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string {
|
||||
return "project"
|
||||
case store.GroupByUser:
|
||||
return "person"
|
||||
case store.GroupByModel:
|
||||
return "model"
|
||||
default:
|
||||
return string(g)
|
||||
}
|
||||
@@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table []
|
||||
}), "", 4)
|
||||
for _, u := range rows {
|
||||
table = append(table, usageRow{
|
||||
ID: u.ID,
|
||||
Name: displayName(u),
|
||||
Requests: u.Requests,
|
||||
Tokens: u.Tokens(),
|
||||
|
||||
Reference in new issue
Block a user