Show usage over time, per organization, team, project and person

Usage could be read as a total — per key, per user, per model, per group —
and the cluster's own charts covered the fleet's hardware. What nobody could
see was when a group used it. "Research doubled its traffic three weeks ago"
is not a question a column of totals can be asked, and it is the question
somebody asks before a quota conversation.

/admin/usage draws requests, tokens and spend over a chosen window, grouped
by any level of the tenancy tree or by person. Person is not a level of that
tree — one person holds keys in several projects — so the grouping is its
own type rather than store.Scope with a fourth value nothing else could use.

Three things learned from the charts already here. Buckets are gap-filled
per group, because a line drawn straight across an absence claims traffic
that did not happen, and because the chart sums its tail by timestamp and
can only do that if every series shares a clock. Series are ranked by total
rather than by peak — a counter's question is "who used the most", and
ranking by peak would colour the team that made one burst over the team that
ran steadily all week. And the spend chart is replaced by a sentence when no
model has a price, rather than drawn flat along the bottom.

The bucket query groups by ordinal and was planned against the cluster's
PostgreSQL before being believed, with a test alongside the others that run
there: three reports have shipped working on SQLite and failing on it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/promq/promq.go

@@ -73,6 +73,19 @@ func (s Series) Max() float64 {
 	return m
 }
 
+// Sum is every value added together, and 0 for an empty series.
+//
+// Meaningful for a counter sampled into buckets, where each point is what
+// happened during that bucket, and not for a gauge, where adding successive
+// readings of the same thing produces a number that means nothing.
+func (s Series) Sum() float64 {
+	total := 0.0
+	for _, p := range s.Points {
+		total += p.Value
+	}
+	return total
+}
+
 // Last is the most recent value, and 0 for an empty series.
 func (s Series) Last() float64 {
 	if len(s.Points) == 0 {

internal/router/handlers.go

@@ -161,6 +161,8 @@ func (r *Router) Handler() http.Handler {
 			// Quotas are written on the same tree the accounts page builds, so
 			// they are mounted together.
 			pages.RegisterAdminQuotas(mux, r.store, r.acct)
+			// And the report of what that tree actually used.
+			pages.RegisterAdminUsage(mux, r.store, r.acct)
 		}
 
 		// Independent of the cluster view: the mirror is a separate machine

internal/store/postgres_test.go

@@ -205,3 +205,50 @@ func TestPostgresUpsertsRatherThanDuplicating(t *testing.T) {
 		t.Errorf("applying the tree twice produced %d orgs", len(orgs))
 	}
 }
+
+func TestPostgresBucketsUsageByEveryGrouping(t *testing.T) {
+	// The query that most deserves this. It selects a computed bucket
+	// expression and groups by ordinal, and three earlier reports shipped
+	// working on SQLite and failing here because PostgreSQL insists every
+	// non-aggregated column is grouped and matches grouped expressions to
+	// selected ones by their text.
+	ctx := context.Background()
+	s := openPG(t)
+	proj := tree(t, s)
+	_, k := mustKey(t, s, "k1", proj)
+
+	now := time.Now()
+	err := s.WriteUsage(ctx, []Usage{
+		{At: now.Add(-2 * time.Hour), KeyID: k.ID, ProjectID: proj,
+			Model: "m", ServedModel: "m",
+			PromptTokens: 100, CompletionTokens: 50, CostMicros: 900, Status: 200},
+		{At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m",
+			PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 500},
+	})
+	if err != nil {
+		t.Fatalf("writing usage: %v", err)
+	}
+
+	for _, g := range Groupings() {
+		rows, err := s.UsageBucketsBy(ctx, g, now.Add(-6*time.Hour), time.Hour)
+		if err != nil {
+			t.Fatalf("grouping by %s: %v", g, err)
+		}
+		if len(rows) != 1 {
+			t.Fatalf("grouping by %s returned %d groups, want 1", g, len(rows))
+		}
+		if rows[0].CostMicros != 1000 || rows[0].Requests != 2 || rows[0].Errors != 1 {
+			t.Errorf("grouping by %s: %d micros, %d requests, %d errors",
+				g, rows[0].CostMicros, rows[0].Requests, rows[0].Errors)
+		}
+		busy := 0
+		for _, b := range rows[0].Buckets {
+			if b.Requests > 0 {
+				busy++
+			}
+		}
+		if busy != 2 {
+			t.Errorf("grouping by %s put the traffic in %d buckets, want 2", g, busy)
+		}
+	}
+}

internal/store/usageseries.go

@@ -0,0 +1,185 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// Grouping is how a usage report is broken down.
+//
+// Not store.Scope with one more value, although three of the four agree with
+// it. Scope is where a limit hangs, and nothing hangs off a person: one person
+// holds keys in several projects, so a per-user total crosses the tree rather
+// than naming a level of it. Keeping the two apart means neither has to carry a
+// case the other does not have.
+type Grouping string
+
+// The groupings a usage report can be asked for.
+const (
+	GroupByOrg     Grouping = "org"
+	GroupByTeam    Grouping = "team"
+	GroupByProject Grouping = "project"
+	GroupByUser    Grouping = "user"
+)
+
+// Valid reports whether this is a grouping usage can be read by.
+func (g Grouping) Valid() bool {
+	_, ok := groupColumns[g]
+	return ok
+}
+
+// Groupings lists them in the order a page should offer them: widest first,
+// then the one that is not a level at all.
+func Groupings() []Grouping {
+	return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser}
+}
+
+// UsageSeries is one group's usage over time.
+//
+// Buckets are contiguous and gap-filled: a group that stopped using the cluster
+// has zeroes rather than an absence, because a line drawn straight across a gap
+// says the traffic continued. It also means every series in a report shares one
+// set of timestamps, which is what lets a chart sum the tail of them.
+type UsageSeries struct {
+	Grouping Grouping
+	ID       string
+	Name     string
+
+	Buckets []Bucket
+
+	// Totals over the whole window, so a table beside the chart does not have
+	// to add the buckets up and possibly disagree with them.
+	Requests   int64
+	Prompt     int64
+	Completion int64
+	CostMicros int64
+	Errors     int64
+}
+
+// Tokens is what the group consumed, prompt and completion together.
+func (u UsageSeries) Tokens() int64 { return u.Prompt + u.Completion }
+
+// groupColumns is the id and display name for each grouping, and how usage
+// reaches it.
+//
+// Written out per grouping rather than assembled from parts: these are table
+// and column names going into SQL, and the one place they must not come from
+// anywhere but here is the place they are interpolated. A grouping that is not
+// in this map is refused before a query is built.
+//
+// Joined through api_keys rather than read from usage.project_id, for the same
+// reason UsageByScope is: that column records where the key was when the
+// request was served, and the tree is what a quota is written against. They
+// agree until a key is moved, and a report that disagrees with the limit shown
+// beside it is the useless one.
+var groupColumns = map[Grouping]struct{ id, name, joins string }{
+	GroupByOrg: {"o.id", "o.name", `
+		JOIN api_keys k      ON k.id = u.key_id
+		JOIN projects p      ON p.id = k.project_id
+		JOIN teams t         ON t.id = p.team_id
+		JOIN organizations o ON o.id = t.org_id`},
+	GroupByTeam: {"t.id", "t.name", `
+		JOIN api_keys k ON k.id = u.key_id
+		JOIN projects p ON p.id = k.project_id
+		JOIN teams t    ON t.id = p.team_id`},
+	GroupByProject: {"p.id", "p.name", `
+		JOIN api_keys k ON k.id = u.key_id
+		JOIN projects p ON p.id = k.project_id`},
+	// Left joins, and coalesced: a key issued by an operator before accounts
+	// existed has no owner, and its traffic is real. It groups under an empty
+	// id, which the page names rather than hides.
+	GroupByUser: {"COALESCE(k.user_id, '')", "COALESCE(us.email, '')", `
+		LEFT JOIN api_keys k ON k.id = u.key_id
+		LEFT JOIN users us   ON us.id = k.user_id`},
+}
+
+// UsageBucketsBy returns each group's usage over time.
+//
+// One query for every group rather than one per group: the page shows them
+// together, and a query per organization is a loop whose cost grows with the
+// thing being reported on.
+func (s *DB) UsageBucketsBy(
+	ctx context.Context, g Grouping, since time.Time, width time.Duration,
+) ([]UsageSeries, error) {
+	c, ok := groupColumns[g]
+	if !ok {
+		return nil, fmt.Errorf(
+			"usage cannot be grouped by %q; try org, team, project or user", g)
+	}
+	if width <= 0 {
+		width = time.Hour
+	}
+	ms := width.Milliseconds()
+
+	// Grouped by ordinal. PostgreSQL matches a grouped expression to a selected
+	// one syntactically, so "(u.ts / ?) * ?" written twice is two expressions
+	// as far as it is concerned; the ordinal is the same one by construction,
+	// and both dialects accept it.
+	q := `
+		SELECT ` + c.id + `, ` + c.name + `, (u.ts / ?) * ?,
+		       COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens),
+		       SUM(u.cost_micros),
+		       SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END)
+		FROM usage u` + c.joins + `
+		WHERE u.ts >= ?
+		GROUP BY 1, 2, 3
+		ORDER BY 1, 3`
+
+	rows, err := s.db.QueryContext(ctx, q, ms, ms, millis(since))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	type cell struct {
+		name string
+		at   map[int64]Bucket
+	}
+	found := map[string]*cell{}
+	var order []string
+
+	for rows.Next() {
+		var id, name string
+		var start int64
+		var b Bucket
+		if err := rows.Scan(&id, &name, &start,
+			&b.Requests, &b.Prompt, &b.Completion, &b.CostMicros, &b.Errors); err != nil {
+			return nil, err
+		}
+		b.Start = fromMillis(start)
+		e, ok := found[id]
+		if !ok {
+			e = &cell{name: name, at: map[int64]Bucket{}}
+			found[id] = e
+			order = append(order, id)
+		}
+		e.at[start] = b
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	first := (millis(since) / ms) * ms
+	last := (millis(time.Now()) / ms) * ms
+
+	out := make([]UsageSeries, 0, len(order))
+	for _, id := range order {
+		e := found[id]
+		u := UsageSeries{Grouping: g, ID: id, Name: e.name}
+		for t := first; t <= last; t += ms {
+			b, ok := e.at[t]
+			if !ok {
+				b = Bucket{Start: fromMillis(t)}
+			}
+			u.Buckets = append(u.Buckets, b)
+			u.Requests += b.Requests
+			u.Prompt += b.Prompt
+			u.Completion += b.Completion
+			u.CostMicros += b.CostMicros
+			u.Errors += b.Errors
+		}
+		out = append(out, u)
+	}
+	return out, nil
+}

internal/store/usageseries_test.go

@@ -0,0 +1,182 @@
+package store
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// spreadOverTime is one org, two teams, and traffic at two separate hours, so a
+// bucketed report has something to put in more than one slice.
+func spreadOverTime(t *testing.T) (*DB, time.Time) {
+	t.Helper()
+	ctx := context.Background()
+	s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "u.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = s.Close() })
+
+	must := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	must(s.PutOrg(ctx, Org{ID: "o", Name: "Acme"}))
+	must(s.PutTeam(ctx, Team{ID: "t1", OrgID: "o", Name: "platform"}))
+	must(s.PutTeam(ctx, Team{ID: "t2", OrgID: "o", Name: "research"}))
+	must(s.PutProject(ctx, Project{ID: "p1", TeamID: "t1", Name: "gateway"}))
+	must(s.PutProject(ctx, Project{ID: "p2", TeamID: "t2", Name: "labs"}))
+	must(s.CreateUser(ctx, User{ID: "usr-1", Email: "ada@example.com", CreatedAt: time.Now()}))
+	for id, proj := range map[string]string{"k1": "p1", "k2": "p2"} {
+		_, k, _ := NewKey(id, "n", proj)
+		k.CreatedAt = time.Now()
+		must(s.CreateKey(ctx, k))
+	}
+
+	now := time.Now()
+	old := now.Add(-3 * time.Hour)
+	must(s.WriteUsage(ctx, []Usage{
+		// platform, three hours ago
+		{At: old, KeyID: "k1", ProjectID: "p1", Model: "m", ServedModel: "m",
+			PromptTokens: 100, CompletionTokens: 50, CostMicros: 900, Status: 200},
+		// platform, now
+		{At: now, KeyID: "k1", ProjectID: "p1", Model: "m", ServedModel: "m",
+			PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 200},
+		// research, now only
+		{At: now, KeyID: "k2", ProjectID: "p2", Model: "m", ServedModel: "m",
+			PromptTokens: 1, CompletionTokens: 1, CostMicros: 50, Status: 500},
+	}))
+	return s, now
+}
+
+func seriesFor(rows []UsageSeries, id string) (UsageSeries, bool) {
+	for _, r := range rows {
+		if r.ID == id {
+			return r, true
+		}
+	}
+	return UsageSeries{}, false
+}
+
+func TestUsageIsBucketedPerGroup(t *testing.T) {
+	s, _ := spreadOverTime(t)
+
+	rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam,
+		time.Now().Add(-6*time.Hour), time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) != 2 {
+		t.Fatalf("expected two teams, got %d", len(rows))
+	}
+
+	platform, ok := seriesFor(rows, "t1")
+	if !ok {
+		t.Fatal("platform is missing")
+	}
+	// The totals are the window's, not one bucket's: a table beside the chart
+	// reads these rather than adding the buckets up itself.
+	if platform.CostMicros != 1000 || platform.Requests != 2 {
+		t.Errorf("platform totals wrong: %d micros over %d requests",
+			platform.CostMicros, platform.Requests)
+	}
+
+	// Two of the seven hourly slices carry traffic, and they are not adjacent.
+	busy := 0
+	for _, b := range platform.Buckets {
+		if b.Requests > 0 {
+			busy++
+		}
+	}
+	if busy != 2 {
+		t.Errorf("expected traffic in two buckets, found %d", busy)
+	}
+}
+
+func TestEveryGroupGetsTheSameTimestamps(t *testing.T) {
+	// Gap-filled, and the same fill for everyone. A chart sums the tail of its
+	// series by timestamp, so two groups whose points do not line up would add
+	// up to a line that is wrong everywhere they disagree.
+	s, _ := spreadOverTime(t)
+
+	rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam,
+		time.Now().Add(-6*time.Hour), time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rows) < 2 {
+		t.Fatalf("expected two teams, got %d", len(rows))
+	}
+	a, b := rows[0].Buckets, rows[1].Buckets
+	if len(a) != len(b) {
+		t.Fatalf("one group has %d buckets, the other %d", len(a), len(b))
+	}
+	for i := range a {
+		if !a[i].Start.Equal(b[i].Start) {
+			t.Fatalf("bucket %d is %s for one group and %s for the other",
+				i, a[i].Start, b[i].Start)
+		}
+	}
+}
+
+func TestAGroupThatWentQuietReadsAsZeroRatherThanAbsent(t *testing.T) {
+	// research used the cluster once, in the last hour. The earlier slices must
+	// be present and zero: a line drawn straight from three hours ago to now
+	// would claim it was busy the whole time.
+	s, _ := spreadOverTime(t)
+
+	rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam,
+		time.Now().Add(-6*time.Hour), time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	research, ok := seriesFor(rows, "t2")
+	if !ok {
+		t.Fatal("research is missing")
+	}
+	if len(research.Buckets) < 6 {
+		t.Fatalf("expected the window filled in, got %d buckets", len(research.Buckets))
+	}
+	if research.Buckets[0].Requests != 0 {
+		t.Errorf("the first slice should be an explicit zero, got %d",
+			research.Buckets[0].Requests)
+	}
+	if research.Requests != 1 || research.Errors != 1 {
+		t.Errorf("research totals wrong: %d requests, %d errors",
+			research.Requests, research.Errors)
+	}
+}
+
+func TestUsageCanBeGroupedByPerson(t *testing.T) {
+	// A person is not a level of the tenancy tree — they hold keys across
+	// projects — which is why this grouping exists beside the other three.
+	s, _ := spreadOverTime(t)
+
+	rows, err := s.UsageBucketsBy(t.Context(), GroupByUser,
+		time.Now().Add(-6*time.Hour), time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Both keys were issued without an owner, so the traffic groups under the
+	// empty person rather than disappearing.
+	unowned, ok := seriesFor(rows, "")
+	if !ok {
+		t.Fatalf("keys with no owner were dropped: %+v", rows)
+	}
+	if unowned.Requests != 3 {
+		t.Errorf("expected all three requests, got %d", unowned.Requests)
+	}
+}
+
+func TestAnUnknownGroupingIsRefusedRatherThanBuiltIntoSQL(t *testing.T) {
+	s, _ := spreadOverTime(t)
+
+	_, err := s.UsageBucketsBy(t.Context(), Grouping("key_id; DROP TABLE usage"),
+		time.Now().Add(-time.Hour), time.Hour)
+	if err == nil {
+		t.Fatal("an unknown grouping was accepted")
+	}
+}

internal/web/admin_usage.go

@@ -0,0 +1,207 @@
+package web
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// UsageAdmin is the store half of the usage report.
+type UsageAdmin interface {
+	UsageBucketsBy(ctx context.Context, g store.Grouping,
+		since time.Time, width time.Duration) ([]store.UsageSeries, error)
+}
+
+// usageWindow is one length of history the page offers, and the slice it is cut
+// into.
+//
+// The bucket width is chosen with the window rather than separately. A day in
+// five-minute slices is 288 points across 720 pixels, which is a texture; a
+// month in five-minute slices is 8,640, which is a smear. Roughly fifty to a
+// hundred points is what a line of this size can actually show.
+type usageWindow struct {
+	Key   string
+	Label string
+	Span  time.Duration
+	Width time.Duration
+}
+
+var usageWindows = []usageWindow{
+	{"24h", "last 24 hours", 24 * time.Hour, 20 * time.Minute},
+	{"7d", "last 7 days", 7 * 24 * time.Hour, 2 * time.Hour},
+	{"30d", "last 30 days", 30 * 24 * time.Hour, 12 * time.Hour},
+}
+
+func windowFor(key string) usageWindow {
+	for _, w := range usageWindows {
+		if w.Key == key {
+			return w
+		}
+	}
+	return usageWindows[1]
+}
+
+// usageGroup names one grouping for the chooser.
+type usageGroup struct {
+	Key      string
+	Label    string
+	Selected bool
+}
+
+// usageRow is one group's totals, shown under the charts.
+type usageRow struct {
+	Name     string
+	Requests int64
+	Tokens   int64
+	Spend    string
+	Errors   int64
+}
+
+type adminUsageData struct {
+	Groups  []usageGroup
+	Windows []struct {
+		Key, Label string
+		Selected   bool
+	}
+	By     string
+	Window string
+
+	Requests TimeSeries
+	Tokens   TimeSeries
+	Spend    TimeSeries
+
+	Rows []usageRow
+	// Priced is false when no model has a price, which makes the spend chart a
+	// flat zero however busy the cluster is. Said rather than drawn, because a
+	// chart along the bottom reads as "nobody spent anything".
+	Priced bool
+
+	Error   string
+	Version string
+	Now     string
+}
+
+// RegisterAdminUsage mounts the usage report under /admin/usage.
+//
+// Separate from the quotas page although both read the same tree. Quotas is a
+// page for writing a number and seeing whether it is being approached; this one
+// answers "who is using the cluster, and when" — including per person, which is
+// not a level quotas can be written on at all.
+func (p *AccountPages) RegisterAdminUsage(mux *http.ServeMux, s UsageAdmin, priced Priced) {
+	mux.HandleFunc("GET /admin/usage", p.AdminPage(
+		func(w http.ResponseWriter, r *http.Request) {
+			p.renderUsage(w, r, s, priced)
+		}))
+}
+
+func (p *AccountPages) renderUsage(w http.ResponseWriter, r *http.Request,
+	s UsageAdmin, priced Priced,
+) {
+	by := store.Grouping(r.URL.Query().Get("by"))
+	if !by.Valid() {
+		by = store.GroupByTeam
+	}
+	win := windowFor(r.URL.Query().Get("window"))
+
+	d := adminUsageData{
+		By: string(by), Window: win.Key,
+		Priced:  priced != nil && priced.AnythingPriced(),
+		Version: p.version,
+		Now:     time.Now().UTC().Format(time.RFC3339),
+	}
+	for _, g := range store.Groupings() {
+		d.Groups = append(d.Groups, usageGroup{
+			Key: string(g), Label: groupLabel(g), Selected: g == by,
+		})
+	}
+	for _, uw := range usageWindows {
+		d.Windows = append(d.Windows, struct {
+			Key, Label string
+			Selected   bool
+		}{uw.Key, uw.Label, uw.Key == win.Key})
+	}
+
+	rows, err := s.UsageBucketsBy(r.Context(), by, time.Now().Add(-win.Span), win.Width)
+	if err != nil {
+		p.log.Error("could not read usage over time", "by", by, "err", err)
+		d.Error = "could not read usage"
+		render(w, "admin_usage", d)
+		return
+	}
+
+	d.Requests = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
+		return float64(b.Requests)
+	}), "", 4)
+	d.Tokens = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
+		return float64(b.Prompt + b.Completion)
+	}), "", 4)
+	d.Spend = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 {
+		// Shown in whole units, as every other spend figure is: a budget is
+		// written in units and a chart in micros would not compare with it.
+		return float64(b.CostMicros) / 1e6
+	}), "", 4)
+
+	for _, u := range rows {
+		d.Rows = append(d.Rows, usageRow{
+			Name:     displayName(u),
+			Requests: u.Requests,
+			Tokens:   u.Tokens(),
+			Spend:    unitsOf(u.CostMicros),
+			Errors:   u.Errors,
+		})
+	}
+	render(w, "admin_usage", d)
+}
+
+// seriesOf turns each group's buckets into a line of one measure.
+//
+// One pass per measure rather than one query per measure: the buckets are
+// already in memory and already share a clock, and three queries that could
+// disagree about the window is a worse answer than three loops.
+func seriesOf(rows []store.UsageSeries, value func(store.Bucket) float64) []promq.Series {
+	out := make([]promq.Series, 0, len(rows))
+	for _, u := range rows {
+		s := promq.Series{Name: displayName(u)}
+		for _, b := range u.Buckets {
+			s.Points = append(s.Points, promq.Point{At: b.Start, Value: value(b)})
+		}
+		out = append(out, s)
+	}
+	return out
+}
+
+// displayName is what a line and a row are labelled with.
+//
+// A person with no account is named rather than left blank: keys issued before
+// accounts existed carry real traffic, and an unlabelled line in a legend is
+// the one nobody can act on.
+func displayName(u store.UsageSeries) string {
+	if u.Name != "" {
+		return u.Name
+	}
+	if u.Grouping == store.GroupByUser {
+		return "keys with no owner"
+	}
+	if u.ID != "" {
+		return u.ID
+	}
+	return "unknown"
+}
+
+func groupLabel(g store.Grouping) string {
+	switch g {
+	case store.GroupByOrg:
+		return "organization"
+	case store.GroupByTeam:
+		return "team"
+	case store.GroupByProject:
+		return "project"
+	case store.GroupByUser:
+		return "person"
+	default:
+		return string(g)
+	}
+}

internal/web/admin_usage_test.go

@@ -0,0 +1,164 @@
+package web
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+type fakeScopeSeries struct {
+	byGroup map[store.Grouping][]store.UsageSeries
+	asked   []store.Grouping
+	width   time.Duration
+	since   time.Time
+}
+
+func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping,
+	since time.Time, width time.Duration,
+) ([]store.UsageSeries, error) {
+	f.asked = append(f.asked, g)
+	f.width, f.since = width, since
+	return f.byGroup[g], nil
+}
+
+func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux {
+	t.Helper()
+	a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
+	p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
+	mux := http.NewServeMux()
+	p.RegisterAdminUsage(mux, s, priced)
+	return mux
+}
+
+// hourly builds a group whose traffic rises over six slices, so a chart drawn
+// from it has a shape rather than a dot.
+func hourly(g store.Grouping, id, name string, scale int64) store.UsageSeries {
+	u := store.UsageSeries{Grouping: g, ID: id, Name: name}
+	base := time.Now().Add(-6 * time.Hour).Truncate(time.Hour)
+	for i := int64(1); i <= 6; i++ {
+		b := store.Bucket{
+			Start:      base.Add(time.Duration(i) * time.Hour),
+			Requests:   i * scale,
+			Prompt:     i * scale * 100,
+			Completion: i * scale * 20,
+			CostMicros: i * scale * 1000,
+		}
+		u.Buckets = append(u.Buckets, b)
+		u.Requests += b.Requests
+		u.Prompt += b.Prompt
+		u.Completion += b.Completion
+		u.CostMicros += b.CostMicros
+	}
+	return u
+}
+
+func TestTheUsagePageDrawsAndTabulatesEachGroup(t *testing.T) {
+	s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
+		store.GroupByTeam: {
+			hourly(store.GroupByTeam, "t1", "platform", 3),
+			hourly(store.GroupByTeam, "t2", "research", 1),
+		},
+	}}
+	body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage").Body.String()
+
+	for _, want := range []string{"platform", "research"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("%q is missing from the page", want)
+		}
+	}
+	// A chart is an SVG with lines in it, not a frame saying there is nothing.
+	if strings.Count(body, "<svg") < 3 {
+		t.Errorf("expected a chart for requests, tokens and spend; got %s", body)
+	}
+	if strings.Contains(body, "Nothing recorded in this window") {
+		t.Error("the table was left out although there are two groups")
+	}
+}
+
+func TestTeamIsTheDefaultGrouping(t *testing.T) {
+	// The middle of the tree: an organization is usually one on a first
+	// cluster, and a project list is long before it is informative.
+	s := &fakeScopeSeries{}
+	mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage")
+
+	if len(s.asked) != 1 || s.asked[0] != store.GroupByTeam {
+		t.Errorf("asked for %v, expected one query grouped by team", s.asked)
+	}
+}
+
+func TestTheGroupingComesFromTheURL(t *testing.T) {
+	s := &fakeScopeSeries{}
+	mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=user")
+
+	if len(s.asked) != 1 || s.asked[0] != store.GroupByUser {
+		t.Errorf("asked for %v, expected a query grouped by user", s.asked)
+	}
+}
+
+func TestAnUnknownGroupingFallsBackRatherThanFailing(t *testing.T) {
+	// The grouping reaches SQL, so it is validated before it is used. A page
+	// that 500s on a mistyped query string is worse than one that shows the
+	// default, and the chooser makes the mistake obvious.
+	s := &fakeScopeSeries{}
+	rec := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=drop+table")
+
+	if rec.Code != 200 {
+		t.Fatalf("status %d", rec.Code)
+	}
+	if len(s.asked) != 1 || s.asked[0] != store.GroupByTeam {
+		t.Errorf("asked for %v, expected the default", s.asked)
+	}
+}
+
+func TestTheWindowChoosesItsOwnBucketWidth(t *testing.T) {
+	// A month in twenty-minute slices is two thousand points across seven
+	// hundred pixels, which is a smear rather than a chart.
+	s := &fakeScopeSeries{}
+	mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?window=30d")
+	wide := s.width
+
+	s2 := &fakeScopeSeries{}
+	mirrorGet(t, usagePage(t, s2, pricedYes{}), "/admin/usage?window=24h")
+
+	if !(wide > s2.width) {
+		t.Errorf("a 30-day window bucketed at %s, a 24-hour one at %s", wide, s2.width)
+	}
+	if s2.width > time.Hour {
+		t.Errorf("a day in %s slices is too coarse to read", s2.width)
+	}
+}
+
+func TestWithNoPricesTheSpendChartIsReplacedBySayingWhy(t *testing.T) {
+	s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
+		store.GroupByTeam: {hourly(store.GroupByTeam, "t1", "platform", 3)},
+	}}
+	body := mirrorGet(t, usagePage(t, s, pricedNo{}), "/admin/usage").Body.String()
+
+	// A spend chart flat along the bottom reads as a quiet month rather than as
+	// a cluster where nothing is priced.
+	if !strings.Contains(body, "No model has a price") {
+		t.Errorf("the page should say why spend is empty; got %s", body)
+	}
+	if strings.Count(body, "<svg") > 2 {
+		t.Error("a spend chart was drawn although nothing is priced")
+	}
+}
+
+func TestAPersonWithNoAccountIsNamedRatherThanBlank(t *testing.T) {
+	// Keys issued by an operator before accounts existed carry real traffic,
+	// and an unlabelled line in a legend is the one nobody can act on.
+	s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{
+		store.GroupByUser: {hourly(store.GroupByUser, "", "", 2)},
+	}}
+	body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=user").Body.String()
+
+	if !strings.Contains(body, "keys with no owner") {
+		t.Errorf("unowned traffic was left unlabelled; got %s", body)
+	}
+}

internal/web/chart.go

@@ -3,6 +3,7 @@ package web
 import (
 	"fmt"
 	"html/template"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -120,6 +121,12 @@ func compact(v float64) string {
 		return fmt.Sprintf("%.1fk", v/1e3)
 	case v == float64(int64(v)):
 		return fmt.Sprintf("%d", int64(v))
+	case v > 0 && v < 1:
+		// Two significant digits rather than one decimal place. A spend of
+		// 0.03 printed as "0.0" says nothing was spent, which is the one thing
+		// a figure must not say when something was. Counts never reach this
+		// branch: a whole number is caught above, and zero is whole.
+		return strconv.FormatFloat(v, 'g', 2, 64)
 	default:
 		return fmt.Sprintf("%.1f", v)
 	}

internal/web/chart_test.go

@@ -104,3 +104,23 @@ func TestHourLabelsSayWhenAndHowMuch(t *testing.T) {
 		t.Errorf("label = %q", got)
 	}
 }
+
+func TestASmallFigureIsNotRenderedAsZero(t *testing.T) {
+	// A spend of 0.03 units shown as "0.0" says nothing was spent. On a cluster
+	// whose prices make a month's traffic a fraction of a unit, that is every
+	// figure on the page.
+	for _, c := range []struct {
+		in   float64
+		want string
+	}{
+		{0.03, "0.03"},
+		{0.5, "0.5"},
+		{0, "0"},
+		{1, "1"},
+		{1500, "1.5k"},
+	} {
+		if got := compact(c.in); got != c.want {
+			t.Errorf("compact(%v) = %q, want %q", c.in, got, c.want)
+		}
+	}
+}

internal/web/static/base.css

@@ -296,3 +296,24 @@ tr.editing form { margin: .4rem 0; }
 .legend i { width: 10px; height: 2px; border-radius: 1px; display: inline-block; }
 .legend b { font-weight: 600; color: var(--fg); font-variant-numeric: tabular-nums; }
 .chart-title { font-size: .9rem; font-weight: 600; margin: 0 0 .3rem; }
+
+/* The filter row above a report's charts.
+   One row, above the charts rather than beside them: the reader chooses what
+   they are looking at before they look, and a control column on the right is
+   found after the question has already been answered wrongly. Links rather
+   than a form, so the view is in the URL and can be shared. */
+.filters { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap;
+           margin: 0 0 1rem; }
+.filter-label { color: var(--muted); font-size: .78rem; text-transform: uppercase;
+                letter-spacing: .04em; margin-right: .2rem; }
+.filter-label + .chip { margin-right: 0; }
+.filters .filter-label:not(:first-child) { margin-left: 1rem; }
+.chip { font-size: .85rem; text-decoration: none; color: var(--fg);
+        background: var(--card); border: 1px solid var(--line);
+        border-radius: 999px; padding: .25rem .7rem; }
+.chip:hover { border-color: var(--accent); }
+/* The selected one is filled, not merely outlined: an outline reads as
+   available rather than as chosen, and with four of them side by side the
+   difference has to survive a glance. */
+.chip.on { background: var(--accent); border-color: var(--accent); color: var(--bg);
+           font-weight: 600; }

internal/web/templates/admin_quotas.html

@@ -7,7 +7,7 @@
   <h1>Quotas</h1>
   <p class="sub">What each organization, team and project may use · spend over the last {{.Since}}</p>
   <nav><a href="/admin">cluster</a> <a href="/admin/users">accounts</a>
-       <a href="/profile">your keys</a></nav>
+       <a href="/admin/usage">usage</a> <a href="/profile">your keys</a></nav>
 </header>
 
 {{if not .BudgetsBind}}

internal/web/templates/admin_usage.html

@@ -0,0 +1,80 @@
+{{define "admin_usage"}}<!doctype html>
+<html lang="en"><head>
+<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Usage · llmgw</title><link rel="stylesheet" href="/_static/base.css">
+</head><body><div class="wrap">
+<header>
+  <h1>Usage over time</h1>
+  <p class="sub">Who is using the cluster, and when</p>
+  <nav><a href="/admin">cluster</a> <a href="/admin/users">accounts</a>
+       <a href="/admin/quotas">quotas</a> <a href="/profile">your keys</a></nav>
+</header>
+
+{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
+
+{{/* One row of controls above the charts, which is where a reader looks for
+     them. Links rather than a script: the choice is in the URL, so a view worth
+     coming back to can be bookmarked or pasted to somebody else. */}}
+<div class="filters">
+  <span class="filter-label">Group by</span>
+  {{range .Groups}}<a class="chip{{if .Selected}} on{{end}}"
+     href="/admin/usage?by={{.Key}}&amp;window={{$.Window}}">{{.Label}}</a>{{end}}
+  <span class="filter-label">Window</span>
+  {{range .Windows}}<a class="chip{{if .Selected}} on{{end}}"
+     href="/admin/usage?by={{$.By}}&amp;window={{.Key}}">{{.Label}}</a>{{end}}
+</div>
+
+<div class="chart-row">
+  {{template "tschart" dict "t" .Requests "title" "Requests"
+     "note" "Requests served in each slice, counted where the key belongs now rather than where it was at the time."}}
+  {{template "tschart" dict "t" .Tokens "title" "Tokens"
+     "note" "Prompt and completion together, which is the figure a person recognises as what they consumed."}}
+</div>
+
+<div class="chart-row">
+  {{if .Priced}}
+  {{template "tschart" dict "t" .Spend "title" "Spend"
+     "note" "In whole units, the same ones a budget is written in."}}
+  {{else}}
+  {{/* Said rather than drawn. A spend chart flat along the bottom reads as a
+       quiet month, and this is not that. */}}
+  <div class="chart-wrap">
+    <h3 class="chart-title">Spend</h3>
+    <p class="empty">No model has a price, so every request costs zero and this
+    chart would be a flat line however busy the cluster is. Set prices in the
+    <code>pricing</code> block of the server configuration.</p>
+  </div>
+  {{end}}
+</div>
+
+{{if .Rows}}
+<table>
+  <tr><th>Group</th><th>Requests</th><th>Tokens</th><th>Spend</th><th>Errors</th></tr>
+  {{range .Rows}}
+  <tr>
+    <td>{{.Name}}</td>
+    <td>{{compact .Requests}}</td>
+    <td>{{compact .Tokens}}</td>
+    <td>{{.Spend}}</td>
+    <td>{{if .Errors}}{{.Errors}}{{else}}—{{end}}</td>
+  </tr>
+  {{end}}
+</table>
+{{else}}
+<p class="empty">Nothing recorded in this window.</p>
+{{end}}
+
+{{/* The charts fold their tail; the table does not. Both are here because they
+     answer different questions, and the one that is easy to get wrong is
+     thinking the four coloured lines are all of it. */}}
+<p class="note">The charts draw the four busiest groups by total and sum the
+rest into “other”, so they always add up to the whole cluster. The table below
+is every group, in the same order.</p>
+
+<p class="note">A person is not a level of the quota tree — one person holds
+keys in several projects — so grouping by person crosses the tree rather than
+naming a level of it. Quotas are written on the other three, on the
+<a href="/admin/quotas">quotas page</a>.</p>
+
+<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
+</div></body></html>{{end}}

internal/web/templates/admin_users.html

@@ -9,6 +9,7 @@
   <nav><a href="/admin">cluster</a>
 {{if .HasMirror}}       <a href="/admin/mirror">mirror</a>
 {{end}}{{if .HasQuotas}}       <a href="/admin/quotas">quotas</a>
+       <a href="/admin/usage">usage</a>
 {{end}}       <a href="/profile">your keys</a></nav>
 </header>
 

internal/web/timeseries.go

@@ -57,11 +57,30 @@ type LegendEntry struct {
 // One axis, always. Two measures of different scale belong in two charts: a
 // second y-axis lets any two lines be made to cross wherever the author likes,
 // and the reader has no way to know it happened.
+//
+// Ranked by peak, which is what a gauge asks: the model that hit the ceiling is
+// the one worth a colour. For a counter, see LineChartTotals.
 func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries {
+	return lineChart(series, unit, maxSeries, promq.Series.Max)
+}
+
+// LineChartTotals is LineChart ranked by the sum over the window.
+//
+// Which is the right question of a counter. "Who used the most" is answered by
+// the total, and ranking by peak would give a colour to the team that made one
+// large burst over the team that ran steadily all week — the opposite of what a
+// usage report is read for.
+func LineChartTotals(series []promq.Series, unit string, maxSeries int) TimeSeries {
+	return lineChart(series, unit, maxSeries, promq.Series.Sum)
+}
+
+func lineChart(series []promq.Series, unit string, maxSeries int,
+	weight func(promq.Series) float64,
+) TimeSeries {
 	if maxSeries <= 0 || maxSeries > len(vizColours) {
 		maxSeries = len(vizColours)
 	}
-	series = foldSeries(series, maxSeries)
+	series = foldSeries(series, maxSeries, weight)
 	if len(series) == 0 {
 		return TimeSeries{Empty: true}
 	}
@@ -107,7 +126,10 @@ const otherLabel = "other"
 // Rather than dropping the tail, which would make a chart that does not add up,
 // or colouring it, which needs hues the palette does not have. The fold is
 // labelled so nobody reads it as a single model.
-func foldSeries(in []promq.Series, keep int) []promq.Series {
+//
+// What "busiest" means is the caller's: a gauge is ranked by its peak and a
+// counter by its total, and the two disagree exactly where it matters.
+func foldSeries(in []promq.Series, keep int, weight func(promq.Series) float64) []promq.Series {
 	clean := make([]promq.Series, 0, len(in))
 	for _, s := range in {
 		if len(s.Points) > 0 {
@@ -117,7 +139,7 @@ func foldSeries(in []promq.Series, keep int) []promq.Series {
 	if len(clean) <= keep {
 		return clean
 	}
-	sort.SliceStable(clean, func(i, j int) bool { return clean[i].Max() > clean[j].Max() })
+	sort.SliceStable(clean, func(i, j int) bool { return weight(clean[i]) > weight(clean[j]) })
 
 	rest := clean[keep-1:]
 	folded := promq.Series{Name: otherLabel}

internal/web/web.go

@@ -130,6 +130,14 @@ var funcs = template.FuncMap{
 	},
 }
 
+// unitsOf renders micro-units as the whole units a budget is written in.
+//
+// A function rather than only a template helper, because the usage report
+// formats the same figure into a table row in Go and the two must not drift.
+func unitsOf(micros int64) string {
+	return fmt.Sprintf("%.2f", float64(micros)/1e6)
+}
+
 var tmpl = template.Must(template.New("").Funcs(funcs).ParseFS(files, "templates/*.html"))
 
 // Static serves the embedded assets, mounted on both listeners because both