Show usage over time, per organization, team, project and person
CI / test (push) Has been cancelled

Usage could be read as a total — per key, per user, per model, per group —
and the cluster's own charts covered the fleet's hardware. What nobody could
see was when a group used it. "Research doubled its traffic three weeks ago"
is not a question a column of totals can be asked, and it is the question
somebody asks before a quota conversation.

/admin/usage draws requests, tokens and spend over a chosen window, grouped
by any level of the tenancy tree or by person. Person is not a level of that
tree — one person holds keys in several projects — so the grouping is its
own type rather than store.Scope with a fourth value nothing else could use.

Three things learned from the charts already here. Buckets are gap-filled
per group, because a line drawn straight across an absence claims traffic
that did not happen, and because the chart sums its tail by timestamp and
can only do that if every series shares a clock. Series are ranked by total
rather than by peak — a counter's question is "who used the most", and
ranking by peak would colour the team that made one burst over the team that
ran steadily all week. And the spend chart is replaced by a sentence when no
model has a price, rather than drawn flat along the bottom.

The bucket query groups by ordinal and was planned against the cluster's
PostgreSQL before being believed, with a test alongside the others that run
there: three reports have shipped working on SQLite and failing on it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-28 08:11:31 +02:00
1 parent f80eef9fa6
commit 07809490e3
15 files changed
+963 -4

No files matched your search

+207
View File
@@ -0,0 +1,207 @@
package web
import (
"context"
"net/http"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// UsageAdmin is the store half of the usage report.
type UsageAdmin interface {
UsageBucketsBy(ctx context.Context, g store.Grouping,
since time.Time, width time.Duration) ([]store.UsageSeries, error)
}
// usageWindow is one length of history the page offers, and the slice it is cut
// into.
//
// The bucket width is chosen with the window rather than separately. A day in
// five-minute slices is 288 points across 720 pixels, which is a texture; a
// month in five-minute slices is 8,640, which is a smear. Roughly fifty to a
// hundred points is what a line of this size can actually show.
type usageWindow struct {
Key string
Label string
Span time.Duration
Width time.Duration
}
var usageWindows = []usageWindow{
{"24h", "last 24 hours", 24 * time.Hour, 20 * time.Minute},
{"7d", "last 7 days", 7 * 24 * time.Hour, 2 * time.Hour},
{"30d", "last 30 days", 30 * 24 * time.Hour, 12 * time.Hour},
}
func windowFor(key string) usageWindow {
for _, w := range usageWindows {
if w.Key == key {
return w
}
}
return usageWindows[1]
}
// usageGroup names one grouping for the chooser.
type usageGroup struct {
Key string
Label string
Selected bool
}
// usageRow is one group's totals, shown under the charts.
type usageRow struct {
Name string
Requests int64
Tokens int64
Spend string
Errors int64
}
type adminUsageData struct {
Groups []usageGroup
Windows []struct {
Key, Label string
Selected bool
}
By string
Window string
Requests TimeSeries
Tokens TimeSeries
Spend TimeSeries
Rows []usageRow
// Priced is false when no model has a price, which makes the spend chart a
// flat zero however busy the cluster is. Said rather than drawn, because a
// chart along the bottom reads as "nobody spent anything".
Priced bool
Error string
Version string
Now string
}
// RegisterAdminUsage mounts the usage report under /admin/usage.
//
// Separate from the quotas page although both read the same tree. Quotas is a
// page for writing a number and seeing whether it is being approached; this one
// answers "who is using the cluster, and when" — including per person, which is
// not a level quotas can be written on at all.
func (p *AccountPages) RegisterAdminUsage(mux *http.ServeMux, s UsageAdmin, priced Priced) {
mux.HandleFunc("GET /admin/usage", p.AdminPage(
func(w http.ResponseWriter, r *http.Request) {
p.renderUsage(w, r, s, priced)
}))
}
func (p *AccountPages) renderUsage(w http.ResponseWriter, r *http.Request,
s UsageAdmin, priced Priced,
) {
by := store.Grouping(r.URL.Query().Get("by"))
if !by.Valid() {
by = store.GroupByTeam
}
win := windowFor(r.URL.Query().Get("window"))
d := adminUsageData{
By: string(by), Window: win.Key,
Priced: priced != nil && priced.AnythingPriced(),
Version: p.version,
Now: time.Now().UTC().Format(time.RFC3339),
}
for _, g := range store.Groupings() {
d.Groups = append(d.Groups, usageGroup{
Key: string(g), Label: groupLabel(g), Selected: g == by,
})
}
for _, uw := range usageWindows {
d.Windows = append(d.Windows, struct {
Key, Label string
Selected bool
}{uw.Key, uw.Label, uw.Key == win.Key})
}
rows, err := s.UsageBucketsBy(r.Context(), by, time.Now().Add(-win.Span), win.Width)
if err != nil {
p.log.Error("could not read usage over time", "by", by, "err", err)
d.Error = "could not read usage"
render(w, "admin_usage", d)
return
}
d.Requests = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
return float64(b.Requests)
}), "", 4)
d.Tokens = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
return float64(b.Prompt + b.Completion)
}), "", 4)
d.Spend = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
// Shown in whole units, as every other spend figure is: a budget is
// written in units and a chart in micros would not compare with it.
return float64(b.CostMicros) / 1e6
}), "", 4)
for _, u := range rows {
d.Rows = append(d.Rows, usageRow{
Name: displayName(u),
Requests: u.Requests,
Tokens: u.Tokens(),
Spend: unitsOf(u.CostMicros),
Errors: u.Errors,
})
}
render(w, "admin_usage", d)
}
// seriesOf turns each group's buckets into a line of one measure.
//
// One pass per measure rather than one query per measure: the buckets are
// already in memory and already share a clock, and three queries that could
// disagree about the window is a worse answer than three loops.
func seriesOf(rows []store.UsageSeries, value func(store.Bucket) float64) []promq.Series {
out := make([]promq.Series, 0, len(rows))
for _, u := range rows {
s := promq.Series{Name: displayName(u)}
for _, b := range u.Buckets {
s.Points = append(s.Points, promq.Point{At: b.Start, Value: value(b)})
}
out = append(out, s)
}
return out
}
// displayName is what a line and a row are labelled with.
//
// A person with no account is named rather than left blank: keys issued before
// accounts existed carry real traffic, and an unlabelled line in a legend is
// the one nobody can act on.
func displayName(u store.UsageSeries) string {
if u.Name != "" {
return u.Name
}
if u.Grouping == store.GroupByUser {
return "keys with no owner"
}
if u.ID != "" {
return u.ID
}
return "unknown"
}
func groupLabel(g store.Grouping) string {
switch g {
case store.GroupByOrg:
return "organization"
case store.GroupByTeam:
return "team"
case store.GroupByProject:
return "project"
case store.GroupByUser:
return "person"
default:
return string(g)
}
}
+164
View File
@@ -0,0 +1,164 @@
package web
import (
"context"
"io"
"log/slog"
"net/http"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
type fakeScopeSeries struct {
byGroup map[store.Grouping][]store.UsageSeries
asked []store.Grouping
width time.Duration
since time.Time
}
func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
since time.Time, width time.Duration,
) ([]store.UsageSeries, error) {
f.asked = append(f.asked, g)
f.width, f.since = width, since
return f.byGroup[g], nil
}
func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux {
t.Helper()
a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
mux := http.NewServeMux()
p.RegisterAdminUsage(mux, s, priced)
return mux
}
// hourly builds a group whose traffic rises over six slices, so a chart drawn
// from it has a shape rather than a dot.
func hourly(g store.Grouping, id, name string, scale int64) store.UsageSeries {
u := store.UsageSeries{Grouping: g, ID: id, Name: name}
base := time.Now().Add(-6 * time.Hour).Truncate(time.Hour)
for i := int64(1); i <= 6; i++ {
b := store.Bucket{
Start: base.Add(time.Duration(i) * time.Hour),
Requests: i * scale,
Prompt: i * scale * 100,
Completion: i * scale * 20,
CostMicros: i * scale * 1000,
}
u.Buckets = append(u.Buckets, b)
u.Requests += b.Requests
u.Prompt += b.Prompt
u.Completion += b.Completion
u.CostMicros += b.CostMicros
}
return u
}
func TestTheUsagePageDrawsAndTabulatesEachGroup(t *testing.T) {
s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
store.GroupByTeam: {
hourly(store.GroupByTeam, "t1", "platform", 3),
hourly(store.GroupByTeam, "t2", "research", 1),
},
}}
body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage").Body.String()
for _, want := range []string{"platform", "research"} {
if !strings.Contains(body, want) {
t.Errorf("%q is missing from the page", want)
}
}
// A chart is an SVG with lines in it, not a frame saying there is nothing.
if strings.Count(body, "<svg") < 3 {
t.Errorf("expected a chart for requests, tokens and spend; got %s", body)
}
if strings.Contains(body, "Nothing recorded in this window") {
t.Error("the table was left out although there are two groups")
}
}
func TestTeamIsTheDefaultGrouping(t *testing.T) {
// The middle of the tree: an organization is usually one on a first
// cluster, and a project list is long before it is informative.
s := &fakeScopeSeries{}
mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage")
if len(s.asked) != 1 || s.asked[0] != store.GroupByTeam {
t.Errorf("asked for %v, expected one query grouped by team", s.asked)
}
}
func TestTheGroupingComesFromTheURL(t *testing.T) {
s := &fakeScopeSeries{}
mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=user")
if len(s.asked) != 1 || s.asked[0] != store.GroupByUser {
t.Errorf("asked for %v, expected a query grouped by user", s.asked)
}
}
func TestAnUnknownGroupingFallsBackRatherThanFailing(t *testing.T) {
// The grouping reaches SQL, so it is validated before it is used. A page
// that 500s on a mistyped query string is worse than one that shows the
// default, and the chooser makes the mistake obvious.
s := &fakeScopeSeries{}
rec := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=drop+table")
if rec.Code != 200 {
t.Fatalf("status %d", rec.Code)
}
if len(s.asked) != 1 || s.asked[0] != store.GroupByTeam {
t.Errorf("asked for %v, expected the default", s.asked)
}
}
func TestTheWindowChoosesItsOwnBucketWidth(t *testing.T) {
// A month in twenty-minute slices is two thousand points across seven
// hundred pixels, which is a smear rather than a chart.
s := &fakeScopeSeries{}
mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?window=30d")
wide := s.width
s2 := &fakeScopeSeries{}
mirrorGet(t, usagePage(t, s2, pricedYes{}), "/admin/usage?window=24h")
if !(wide > s2.width) {
t.Errorf("a 30-day window bucketed at %s, a 24-hour one at %s", wide, s2.width)
}
if s2.width > time.Hour {
t.Errorf("a day in %s slices is too coarse to read", s2.width)
}
}
func TestWithNoPricesTheSpendChartIsReplacedBySayingWhy(t *testing.T) {
s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
store.GroupByTeam: {hourly(store.GroupByTeam, "t1", "platform", 3)},
}}
body := mirrorGet(t, usagePage(t, s, pricedNo{}), "/admin/usage").Body.String()
// A spend chart flat along the bottom reads as a quiet month rather than as
// a cluster where nothing is priced.
if !strings.Contains(body, "No model has a price") {
t.Errorf("the page should say why spend is empty; got %s", body)
}
if strings.Count(body, "<svg") > 2 {
t.Error("a spend chart was drawn although nothing is priced")
}
}
func TestAPersonWithNoAccountIsNamedRatherThanBlank(t *testing.T) {
// Keys issued by an operator before accounts existed carry real traffic,
// and an unlabelled line in a legend is the one nobody can act on.
s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
store.GroupByUser: {hourly(store.GroupByUser, "", "", 2)},
}}
body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=user").Body.String()
if !strings.Contains(body, "keys with no owner") {
t.Errorf("unowned traffic was left unlabelled; got %s", body)
}
}
+7
View File
@@ -3,6 +3,7 @@ package web
import (
"fmt"
"html/template"
"strconv"
"strings"
"time"
)
@@ -120,6 +121,12 @@ func compact(v float64) string {
return fmt.Sprintf("%.1fk", v/1e3)
case v == float64(int64(v)):
return fmt.Sprintf("%d", int64(v))
case v > 0 && v < 1:
// Two significant digits rather than one decimal place. A spend of
// 0.03 printed as "0.0" says nothing was spent, which is the one thing
// a figure must not say when something was. Counts never reach this
// branch: a whole number is caught above, and zero is whole.
return strconv.FormatFloat(v, 'g', 2, 64)
default:
return fmt.Sprintf("%.1f", v)
}
+20
View File
@@ -104,3 +104,23 @@ func TestHourLabelsSayWhenAndHowMuch(t *testing.T) {
t.Errorf("label = %q", got)
}
}
func TestASmallFigureIsNotRenderedAsZero(t *testing.T) {
// A spend of 0.03 units shown as "0.0" says nothing was spent. On a cluster
// whose prices make a month's traffic a fraction of a unit, that is every
// figure on the page.
for _, c := range []struct {
in float64
want string
}{
{0.03, "0.03"},
{0.5, "0.5"},
{0, "0"},
{1, "1"},
{1500, "1.5k"},
} {
if got := compact(c.in); got != c.want {
t.Errorf("compact(%v) = %q, want %q", c.in, got, c.want)
}
}
}
+21
View File
@@ -296,3 +296,24 @@ tr.editing form { margin: .4rem 0; }
.legend i { width: 10px; height: 2px; border-radius: 1px; display: inline-block; }
.legend b { font-weight: 600; color: var(--fg); font-variant-numeric: tabular-nums; }
.chart-title { font-size: .9rem; font-weight: 600; margin: 0 0 .3rem; }
/* The filter row above a report's charts.
One row, above the charts rather than beside them: the reader chooses what
they are looking at before they look, and a control column on the right is
found after the question has already been answered wrongly. Links rather
than a form, so the view is in the URL and can be shared. */
.filters { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap;
margin: 0 0 1rem; }
.filter-label { color: var(--muted); font-size: .78rem; text-transform: uppercase;
letter-spacing: .04em; margin-right: .2rem; }
.filter-label + .chip { margin-right: 0; }
.filters .filter-label:not(:first-child) { margin-left: 1rem; }
.chip { font-size: .85rem; text-decoration: none; color: var(--fg);
background: var(--card); border: 1px solid var(--line);
border-radius: 999px; padding: .25rem .7rem; }
.chip:hover { border-color: var(--accent); }
/* The selected one is filled, not merely outlined: an outline reads as
available rather than as chosen, and with four of them side by side the
difference has to survive a glance. */
.chip.on { background: var(--accent); border-color: var(--accent); color: var(--bg);
font-weight: 600; }
+1 -1
View File
@@ -7,7 +7,7 @@
<h1>Quotas</h1>
<p class="sub">What each organization, team and project may use · spend over the last {{.Since}}</p>
<nav><a href="/admin">cluster</a> <a href="/admin/users">accounts</a>
<a href="/profile">your keys</a></nav>
<a href="/admin/usage">usage</a> <a href="/profile">your keys</a></nav>
</header>
{{if not .BudgetsBind}}
+80
View File
@@ -0,0 +1,80 @@
{{define "admin_usage"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Usage · llmgw</title><link rel="stylesheet" href="/_static/base.css">
</head><body><div class="wrap">
<header>
<h1>Usage over time</h1>
<p class="sub">Who is using the cluster, and when</p>
<nav><a href="/admin">cluster</a> <a href="/admin/users">accounts</a>
<a href="/admin/quotas">quotas</a> <a href="/profile">your keys</a></nav>
</header>
{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
{{/* One row of controls above the charts, which is where a reader looks for
them. Links rather than a script: the choice is in the URL, so a view worth
coming back to can be bookmarked or pasted to somebody else. */}}
<div class="filters">
<span class="filter-label">Group by</span>
{{range .Groups}}<a class="chip{{if .Selected}} on{{end}}"
href="/admin/usage?by={{.Key}}&amp;window={{$.Window}}">{{.Label}}</a>{{end}}
<span class="filter-label">Window</span>
{{range .Windows}}<a class="chip{{if .Selected}} on{{end}}"
href="/admin/usage?by={{$.By}}&amp;window={{.Key}}">{{.Label}}</a>{{end}}
</div>
<div class="chart-row">
{{template "tschart" dict "t" .Requests "title" "Requests"
"note" "Requests served in each slice, counted where the key belongs now rather than where it was at the time."}}
{{template "tschart" dict "t" .Tokens "title" "Tokens"
"note" "Prompt and completion together, which is the figure a person recognises as what they consumed."}}
</div>
<div class="chart-row">
{{if .Priced}}
{{template "tschart" dict "t" .Spend "title" "Spend"
"note" "In whole units, the same ones a budget is written in."}}
{{else}}
{{/* Said rather than drawn. A spend chart flat along the bottom reads as a
quiet month, and this is not that. */}}
<div class="chart-wrap">
<h3 class="chart-title">Spend</h3>
<p class="empty">No model has a price, so every request costs zero and this
chart would be a flat line however busy the cluster is. Set prices in the
<code>pricing</code> block of the server configuration.</p>
</div>
{{end}}
</div>
{{if .Rows}}
<table>
<tr><th>Group</th><th>Requests</th><th>Tokens</th><th>Spend</th><th>Errors</th></tr>
{{range .Rows}}
<tr>
<td>{{.Name}}</td>
<td>{{compact .Requests}}</td>
<td>{{compact .Tokens}}</td>
<td>{{.Spend}}</td>
<td>{{if .Errors}}{{.Errors}}{{else}}—{{end}}</td>
</tr>
{{end}}
</table>
{{else}}
<p class="empty">Nothing recorded in this window.</p>
{{end}}
{{/* The charts fold their tail; the table does not. Both are here because they
answer different questions, and the one that is easy to get wrong is
thinking the four coloured lines are all of it. */}}
<p class="note">The charts draw the four busiest groups by total and sum the
rest into “other”, so they always add up to the whole cluster. The table below
is every group, in the same order.</p>
<p class="note">A person is not a level of the quota tree — one person holds
keys in several projects — so grouping by person crosses the tree rather than
naming a level of it. Quotas are written on the other three, on the
<a href="/admin/quotas">quotas page</a>.</p>
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div></body></html>{{end}}
+1
View File
@@ -9,6 +9,7 @@
<nav><a href="/admin">cluster</a>
{{if .HasMirror}} <a href="/admin/mirror">mirror</a>
{{end}}{{if .HasQuotas}} <a href="/admin/quotas">quotas</a>
<a href="/admin/usage">usage</a>
{{end}} <a href="/profile">your keys</a></nav>
</header>
+25 -3
View File
@@ -57,11 +57,30 @@ type LegendEntry struct {
// One axis, always. Two measures of different scale belong in two charts: a
// second y-axis lets any two lines be made to cross wherever the author likes,
// and the reader has no way to know it happened.
//
// Ranked by peak, which is what a gauge asks: the model that hit the ceiling is
// the one worth a colour. For a counter, see LineChartTotals.
func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries {
return lineChart(series, unit, maxSeries, promq.Series.Max)
}
// LineChartTotals is LineChart ranked by the sum over the window.
//
// Which is the right question of a counter. "Who used the most" is answered by
// the total, and ranking by peak would give a colour to the team that made one
// large burst over the team that ran steadily all week — the opposite of what a
// usage report is read for.
func LineChartTotals(series []promq.Series, unit string, maxSeries int) TimeSeries {
return lineChart(series, unit, maxSeries, promq.Series.Sum)
}
func lineChart(series []promq.Series, unit string, maxSeries int,
weight func(promq.Series) float64,
) TimeSeries {
if maxSeries <= 0 || maxSeries > len(vizColours) {
maxSeries = len(vizColours)
}
series = foldSeries(series, maxSeries)
series = foldSeries(series, maxSeries, weight)
if len(series) == 0 {
return TimeSeries{Empty: true}
}
@@ -107,7 +126,10 @@ const otherLabel = "other"
// Rather than dropping the tail, which would make a chart that does not add up,
// or colouring it, which needs hues the palette does not have. The fold is
// labelled so nobody reads it as a single model.
func foldSeries(in []promq.Series, keep int) []promq.Series {
//
// What "busiest" means is the caller's: a gauge is ranked by its peak and a
// counter by its total, and the two disagree exactly where it matters.
func foldSeries(in []promq.Series, keep int, weight func(promq.Series) float64) []promq.Series {
clean := make([]promq.Series, 0, len(in))
for _, s := range in {
if len(s.Points) > 0 {
@@ -117,7 +139,7 @@ func foldSeries(in []promq.Series, keep int) []promq.Series {
if len(clean) <= keep {
return clean
}
sort.SliceStable(clean, func(i, j int) bool { return clean[i].Max() > clean[j].Max() })
sort.SliceStable(clean, func(i, j int) bool { return weight(clean[i]) > weight(clean[j]) })
rest := clean[keep-1:]
folded := promq.Series{Name: otherLabel}
+8
View File
@@ -130,6 +130,14 @@ var funcs = template.FuncMap{
},
}
// unitsOf renders micro-units as the whole units a budget is written in.
//
// A function rather than only a template helper, because the usage report
// formats the same figure into a table row in Go and the two must not drift.
func unitsOf(micros int64) string {
return fmt.Sprintf("%.2f", float64(micros)/1e6)
}
var tmpl = template.Must(template.New("").Funcs(funcs).ParseFS(files, "templates/*.html"))
// Static serves the embedded assets, mounted on both listeners because both