diff --git a/internal/promq/promq.go b/internal/promq/promq.go index 41e9013..fd65fba 100644 --- a/internal/promq/promq.go +++ b/internal/promq/promq.go @@ -73,6 +73,19 @@ func (s Series) Max() float64 { return m } +// Sum is every value added together, and 0 for an empty series. +// +// Meaningful for a counter sampled into buckets, where each point is what +// happened during that bucket, and not for a gauge, where adding successive +// readings of the same thing produces a number that means nothing. +func (s Series) Sum() float64 { + total := 0.0 + for _, p := range s.Points { + total += p.Value + } + return total +} + // Last is the most recent value, and 0 for an empty series. func (s Series) Last() float64 { if len(s.Points) == 0 { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 6c2f52a..fb909d3 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -161,6 +161,8 @@ func (r *Router) Handler() http.Handler { // Quotas are written on the same tree the accounts page builds, so // they are mounted together. pages.RegisterAdminQuotas(mux, r.store, r.acct) + // And the report of what that tree actually used. + pages.RegisterAdminUsage(mux, r.store, r.acct) } // Independent of the cluster view: the mirror is a separate machine diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index bee55f8..bddd493 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -205,3 +205,50 @@ func TestPostgresUpsertsRatherThanDuplicating(t *testing.T) { t.Errorf("applying the tree twice produced %d orgs", len(orgs)) } } + +func TestPostgresBucketsUsageByEveryGrouping(t *testing.T) { + // The query that most deserves this. It selects a computed bucket + // expression and groups by ordinal, and three earlier reports shipped + // working on SQLite and failing here because PostgreSQL insists every + // non-aggregated column is grouped and matches grouped expressions to + // selected ones by their text. + ctx := context.Background() + s := openPG(t) + proj := tree(t, s) + _, k := mustKey(t, s, "k1", proj) + + now := time.Now() + err := s.WriteUsage(ctx, []Usage{ + {At: now.Add(-2 * time.Hour), KeyID: k.ID, ProjectID: proj, + Model: "m", ServedModel: "m", + PromptTokens: 100, CompletionTokens: 50, CostMicros: 900, Status: 200}, + {At: now, KeyID: k.ID, ProjectID: proj, Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 500}, + }) + if err != nil { + t.Fatalf("writing usage: %v", err) + } + + for _, g := range Groupings() { + rows, err := s.UsageBucketsBy(ctx, g, now.Add(-6*time.Hour), time.Hour) + if err != nil { + t.Fatalf("grouping by %s: %v", g, err) + } + if len(rows) != 1 { + t.Fatalf("grouping by %s returned %d groups, want 1", g, len(rows)) + } + if rows[0].CostMicros != 1000 || rows[0].Requests != 2 || rows[0].Errors != 1 { + t.Errorf("grouping by %s: %d micros, %d requests, %d errors", + g, rows[0].CostMicros, rows[0].Requests, rows[0].Errors) + } + busy := 0 + for _, b := range rows[0].Buckets { + if b.Requests > 0 { + busy++ + } + } + if busy != 2 { + t.Errorf("grouping by %s put the traffic in %d buckets, want 2", g, busy) + } + } +} diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go new file mode 100644 index 0000000..f24312a --- /dev/null +++ b/internal/store/usageseries.go @@ -0,0 +1,185 @@ +package store + +import ( + "context" + "fmt" + "time" +) + +// Grouping is how a usage report is broken down. +// +// Not store.Scope with one more value, although three of the four agree with +// it. Scope is where a limit hangs, and nothing hangs off a person: one person +// holds keys in several projects, so a per-user total crosses the tree rather +// than naming a level of it. Keeping the two apart means neither has to carry a +// case the other does not have. +type Grouping string + +// The groupings a usage report can be asked for. +const ( + GroupByOrg Grouping = "org" + GroupByTeam Grouping = "team" + GroupByProject Grouping = "project" + GroupByUser Grouping = "user" +) + +// Valid reports whether this is a grouping usage can be read by. +func (g Grouping) Valid() bool { + _, ok := groupColumns[g] + return ok +} + +// Groupings lists them in the order a page should offer them: widest first, +// then the one that is not a level at all. +func Groupings() []Grouping { + return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser} +} + +// UsageSeries is one group's usage over time. +// +// Buckets are contiguous and gap-filled: a group that stopped using the cluster +// has zeroes rather than an absence, because a line drawn straight across a gap +// says the traffic continued. It also means every series in a report shares one +// set of timestamps, which is what lets a chart sum the tail of them. +type UsageSeries struct { + Grouping Grouping + ID string + Name string + + Buckets []Bucket + + // Totals over the whole window, so a table beside the chart does not have + // to add the buckets up and possibly disagree with them. + Requests int64 + Prompt int64 + Completion int64 + CostMicros int64 + Errors int64 +} + +// Tokens is what the group consumed, prompt and completion together. +func (u UsageSeries) Tokens() int64 { return u.Prompt + u.Completion } + +// groupColumns is the id and display name for each grouping, and how usage +// reaches it. +// +// Written out per grouping rather than assembled from parts: these are table +// and column names going into SQL, and the one place they must not come from +// anywhere but here is the place they are interpolated. A grouping that is not +// in this map is refused before a query is built. +// +// Joined through api_keys rather than read from usage.project_id, for the same +// reason UsageByScope is: that column records where the key was when the +// request was served, and the tree is what a quota is written against. They +// agree until a key is moved, and a report that disagrees with the limit shown +// beside it is the useless one. +var groupColumns = map[Grouping]struct{ id, name, joins string }{ + GroupByOrg: {"o.id", "o.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id + JOIN teams t ON t.id = p.team_id + JOIN organizations o ON o.id = t.org_id`}, + GroupByTeam: {"t.id", "t.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id + JOIN teams t ON t.id = p.team_id`}, + GroupByProject: {"p.id", "p.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id`}, + // Left joins, and coalesced: a key issued by an operator before accounts + // existed has no owner, and its traffic is real. It groups under an empty + // id, which the page names rather than hides. + GroupByUser: {"COALESCE(k.user_id, '')", "COALESCE(us.email, '')", ` + LEFT JOIN api_keys k ON k.id = u.key_id + LEFT JOIN users us ON us.id = k.user_id`}, +} + +// UsageBucketsBy returns each group's usage over time. +// +// One query for every group rather than one per group: the page shows them +// together, and a query per organization is a loop whose cost grows with the +// thing being reported on. +func (s *DB) UsageBucketsBy( + ctx context.Context, g Grouping, since time.Time, width time.Duration, +) ([]UsageSeries, error) { + c, ok := groupColumns[g] + if !ok { + return nil, fmt.Errorf( + "usage cannot be grouped by %q; try org, team, project or user", g) + } + if width <= 0 { + width = time.Hour + } + ms := width.Milliseconds() + + // Grouped by ordinal. PostgreSQL matches a grouped expression to a selected + // one syntactically, so "(u.ts / ?) * ?" written twice is two expressions + // as far as it is concerned; the ordinal is the same one by construction, + // and both dialects accept it. + q := ` + SELECT ` + c.id + `, ` + c.name + `, (u.ts / ?) * ?, + COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), + SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END) + FROM usage u` + c.joins + ` + WHERE u.ts >= ? + GROUP BY 1, 2, 3 + ORDER BY 1, 3` + + rows, err := s.db.QueryContext(ctx, q, ms, ms, millis(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + type cell struct { + name string + at map[int64]Bucket + } + found := map[string]*cell{} + var order []string + + for rows.Next() { + var id, name string + var start int64 + var b Bucket + if err := rows.Scan(&id, &name, &start, + &b.Requests, &b.Prompt, &b.Completion, &b.CostMicros, &b.Errors); err != nil { + return nil, err + } + b.Start = fromMillis(start) + e, ok := found[id] + if !ok { + e = &cell{name: name, at: map[int64]Bucket{}} + found[id] = e + order = append(order, id) + } + e.at[start] = b + } + if err := rows.Err(); err != nil { + return nil, err + } + + first := (millis(since) / ms) * ms + last := (millis(time.Now()) / ms) * ms + + out := make([]UsageSeries, 0, len(order)) + for _, id := range order { + e := found[id] + u := UsageSeries{Grouping: g, ID: id, Name: e.name} + for t := first; t <= last; t += ms { + b, ok := e.at[t] + if !ok { + b = Bucket{Start: fromMillis(t)} + } + u.Buckets = append(u.Buckets, b) + u.Requests += b.Requests + u.Prompt += b.Prompt + u.Completion += b.Completion + u.CostMicros += b.CostMicros + u.Errors += b.Errors + } + out = append(out, u) + } + return out, nil +} diff --git a/internal/store/usageseries_test.go b/internal/store/usageseries_test.go new file mode 100644 index 0000000..6ac59da --- /dev/null +++ b/internal/store/usageseries_test.go @@ -0,0 +1,182 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +// spreadOverTime is one org, two teams, and traffic at two separate hours, so a +// bucketed report has something to put in more than one slice. +func spreadOverTime(t *testing.T) (*DB, time.Time) { + t.Helper() + ctx := context.Background() + s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "u.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.PutOrg(ctx, Org{ID: "o", Name: "Acme"})) + must(s.PutTeam(ctx, Team{ID: "t1", OrgID: "o", Name: "platform"})) + must(s.PutTeam(ctx, Team{ID: "t2", OrgID: "o", Name: "research"})) + must(s.PutProject(ctx, Project{ID: "p1", TeamID: "t1", Name: "gateway"})) + must(s.PutProject(ctx, Project{ID: "p2", TeamID: "t2", Name: "labs"})) + must(s.CreateUser(ctx, User{ID: "usr-1", Email: "ada@example.com", CreatedAt: time.Now()})) + for id, proj := range map[string]string{"k1": "p1", "k2": "p2"} { + _, k, _ := NewKey(id, "n", proj) + k.CreatedAt = time.Now() + must(s.CreateKey(ctx, k)) + } + + now := time.Now() + old := now.Add(-3 * time.Hour) + must(s.WriteUsage(ctx, []Usage{ + // platform, three hours ago + {At: old, KeyID: "k1", ProjectID: "p1", Model: "m", ServedModel: "m", + PromptTokens: 100, CompletionTokens: 50, CostMicros: 900, Status: 200}, + // platform, now + {At: now, KeyID: "k1", ProjectID: "p1", Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 200}, + // research, now only + {At: now, KeyID: "k2", ProjectID: "p2", Model: "m", ServedModel: "m", + PromptTokens: 1, CompletionTokens: 1, CostMicros: 50, Status: 500}, + })) + return s, now +} + +func seriesFor(rows []UsageSeries, id string) (UsageSeries, bool) { + for _, r := range rows { + if r.ID == id { + return r, true + } + } + return UsageSeries{}, false +} + +func TestUsageIsBucketedPerGroup(t *testing.T) { + s, _ := spreadOverTime(t) + + rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam, + time.Now().Add(-6*time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("expected two teams, got %d", len(rows)) + } + + platform, ok := seriesFor(rows, "t1") + if !ok { + t.Fatal("platform is missing") + } + // The totals are the window's, not one bucket's: a table beside the chart + // reads these rather than adding the buckets up itself. + if platform.CostMicros != 1000 || platform.Requests != 2 { + t.Errorf("platform totals wrong: %d micros over %d requests", + platform.CostMicros, platform.Requests) + } + + // Two of the seven hourly slices carry traffic, and they are not adjacent. + busy := 0 + for _, b := range platform.Buckets { + if b.Requests > 0 { + busy++ + } + } + if busy != 2 { + t.Errorf("expected traffic in two buckets, found %d", busy) + } +} + +func TestEveryGroupGetsTheSameTimestamps(t *testing.T) { + // Gap-filled, and the same fill for everyone. A chart sums the tail of its + // series by timestamp, so two groups whose points do not line up would add + // up to a line that is wrong everywhere they disagree. + s, _ := spreadOverTime(t) + + rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam, + time.Now().Add(-6*time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + if len(rows) < 2 { + t.Fatalf("expected two teams, got %d", len(rows)) + } + a, b := rows[0].Buckets, rows[1].Buckets + if len(a) != len(b) { + t.Fatalf("one group has %d buckets, the other %d", len(a), len(b)) + } + for i := range a { + if !a[i].Start.Equal(b[i].Start) { + t.Fatalf("bucket %d is %s for one group and %s for the other", + i, a[i].Start, b[i].Start) + } + } +} + +func TestAGroupThatWentQuietReadsAsZeroRatherThanAbsent(t *testing.T) { + // research used the cluster once, in the last hour. The earlier slices must + // be present and zero: a line drawn straight from three hours ago to now + // would claim it was busy the whole time. + s, _ := spreadOverTime(t) + + rows, err := s.UsageBucketsBy(t.Context(), GroupByTeam, + time.Now().Add(-6*time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + research, ok := seriesFor(rows, "t2") + if !ok { + t.Fatal("research is missing") + } + if len(research.Buckets) < 6 { + t.Fatalf("expected the window filled in, got %d buckets", len(research.Buckets)) + } + if research.Buckets[0].Requests != 0 { + t.Errorf("the first slice should be an explicit zero, got %d", + research.Buckets[0].Requests) + } + if research.Requests != 1 || research.Errors != 1 { + t.Errorf("research totals wrong: %d requests, %d errors", + research.Requests, research.Errors) + } +} + +func TestUsageCanBeGroupedByPerson(t *testing.T) { + // A person is not a level of the tenancy tree — they hold keys across + // projects — which is why this grouping exists beside the other three. + s, _ := spreadOverTime(t) + + rows, err := s.UsageBucketsBy(t.Context(), GroupByUser, + time.Now().Add(-6*time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + // Both keys were issued without an owner, so the traffic groups under the + // empty person rather than disappearing. + unowned, ok := seriesFor(rows, "") + if !ok { + t.Fatalf("keys with no owner were dropped: %+v", rows) + } + if unowned.Requests != 3 { + t.Errorf("expected all three requests, got %d", unowned.Requests) + } +} + +func TestAnUnknownGroupingIsRefusedRatherThanBuiltIntoSQL(t *testing.T) { + s, _ := spreadOverTime(t) + + _, err := s.UsageBucketsBy(t.Context(), Grouping("key_id; DROP TABLE usage"), + time.Now().Add(-time.Hour), time.Hour) + if err == nil { + t.Fatal("an unknown grouping was accepted") + } +} diff --git a/internal/web/admin_usage.go b/internal/web/admin_usage.go new file mode 100644 index 0000000..53bc277 --- /dev/null +++ b/internal/web/admin_usage.go @@ -0,0 +1,207 @@ +package web + +import ( + "context" + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// UsageAdmin is the store half of the usage report. +type UsageAdmin interface { + UsageBucketsBy(ctx context.Context, g store.Grouping, + since time.Time, width time.Duration) ([]store.UsageSeries, error) +} + +// usageWindow is one length of history the page offers, and the slice it is cut +// into. +// +// The bucket width is chosen with the window rather than separately. A day in +// five-minute slices is 288 points across 720 pixels, which is a texture; a +// month in five-minute slices is 8,640, which is a smear. Roughly fifty to a +// hundred points is what a line of this size can actually show. +type usageWindow struct { + Key string + Label string + Span time.Duration + Width time.Duration +} + +var usageWindows = []usageWindow{ + {"24h", "last 24 hours", 24 * time.Hour, 20 * time.Minute}, + {"7d", "last 7 days", 7 * 24 * time.Hour, 2 * time.Hour}, + {"30d", "last 30 days", 30 * 24 * time.Hour, 12 * time.Hour}, +} + +func windowFor(key string) usageWindow { + for _, w := range usageWindows { + if w.Key == key { + return w + } + } + return usageWindows[1] +} + +// usageGroup names one grouping for the chooser. +type usageGroup struct { + Key string + Label string + Selected bool +} + +// usageRow is one group's totals, shown under the charts. +type usageRow struct { + Name string + Requests int64 + Tokens int64 + Spend string + Errors int64 +} + +type adminUsageData struct { + Groups []usageGroup + Windows []struct { + Key, Label string + Selected bool + } + By string + Window string + + Requests TimeSeries + Tokens TimeSeries + Spend TimeSeries + + Rows []usageRow + // Priced is false when no model has a price, which makes the spend chart a + // flat zero however busy the cluster is. Said rather than drawn, because a + // chart along the bottom reads as "nobody spent anything". + Priced bool + + Error string + Version string + Now string +} + +// RegisterAdminUsage mounts the usage report under /admin/usage. +// +// Separate from the quotas page although both read the same tree. Quotas is a +// page for writing a number and seeing whether it is being approached; this one +// answers "who is using the cluster, and when" — including per person, which is +// not a level quotas can be written on at all. +func (p *AccountPages) RegisterAdminUsage(mux *http.ServeMux, s UsageAdmin, priced Priced) { + mux.HandleFunc("GET /admin/usage", p.AdminPage( + func(w http.ResponseWriter, r *http.Request) { + p.renderUsage(w, r, s, priced) + })) +} + +func (p *AccountPages) renderUsage(w http.ResponseWriter, r *http.Request, + s UsageAdmin, priced Priced, +) { + by := store.Grouping(r.URL.Query().Get("by")) + if !by.Valid() { + by = store.GroupByTeam + } + win := windowFor(r.URL.Query().Get("window")) + + d := adminUsageData{ + By: string(by), Window: win.Key, + Priced: priced != nil && priced.AnythingPriced(), + Version: p.version, + Now: time.Now().UTC().Format(time.RFC3339), + } + for _, g := range store.Groupings() { + d.Groups = append(d.Groups, usageGroup{ + Key: string(g), Label: groupLabel(g), Selected: g == by, + }) + } + for _, uw := range usageWindows { + d.Windows = append(d.Windows, struct { + Key, Label string + Selected bool + }{uw.Key, uw.Label, uw.Key == win.Key}) + } + + rows, err := s.UsageBucketsBy(r.Context(), by, time.Now().Add(-win.Span), win.Width) + if err != nil { + p.log.Error("could not read usage over time", "by", by, "err", err) + d.Error = "could not read usage" + render(w, "admin_usage", d) + return + } + + d.Requests = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 { + return float64(b.Requests) + }), "", 4) + d.Tokens = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 { + return float64(b.Prompt + b.Completion) + }), "", 4) + d.Spend = LineChartTotals(seriesOf(rows, func(b store.Bucket) float64 { + // Shown in whole units, as every other spend figure is: a budget is + // written in units and a chart in micros would not compare with it. + return float64(b.CostMicros) / 1e6 + }), "", 4) + + for _, u := range rows { + d.Rows = append(d.Rows, usageRow{ + Name: displayName(u), + Requests: u.Requests, + Tokens: u.Tokens(), + Spend: unitsOf(u.CostMicros), + Errors: u.Errors, + }) + } + render(w, "admin_usage", d) +} + +// seriesOf turns each group's buckets into a line of one measure. +// +// One pass per measure rather than one query per measure: the buckets are +// already in memory and already share a clock, and three queries that could +// disagree about the window is a worse answer than three loops. +func seriesOf(rows []store.UsageSeries, value func(store.Bucket) float64) []promq.Series { + out := make([]promq.Series, 0, len(rows)) + for _, u := range rows { + s := promq.Series{Name: displayName(u)} + for _, b := range u.Buckets { + s.Points = append(s.Points, promq.Point{At: b.Start, Value: value(b)}) + } + out = append(out, s) + } + return out +} + +// displayName is what a line and a row are labelled with. +// +// A person with no account is named rather than left blank: keys issued before +// accounts existed carry real traffic, and an unlabelled line in a legend is +// the one nobody can act on. +func displayName(u store.UsageSeries) string { + if u.Name != "" { + return u.Name + } + if u.Grouping == store.GroupByUser { + return "keys with no owner" + } + if u.ID != "" { + return u.ID + } + return "unknown" +} + +func groupLabel(g store.Grouping) string { + switch g { + case store.GroupByOrg: + return "organization" + case store.GroupByTeam: + return "team" + case store.GroupByProject: + return "project" + case store.GroupByUser: + return "person" + default: + return string(g) + } +} diff --git a/internal/web/admin_usage_test.go b/internal/web/admin_usage_test.go new file mode 100644 index 0000000..de8da22 --- /dev/null +++ b/internal/web/admin_usage_test.go @@ -0,0 +1,164 @@ +package web + +import ( + "context" + "io" + "log/slog" + "net/http" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type fakeScopeSeries struct { + byGroup map[store.Grouping][]store.UsageSeries + asked []store.Grouping + width time.Duration + since time.Time +} + +func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, + since time.Time, width time.Duration, +) ([]store.UsageSeries, error) { + f.asked = append(f.asked, g) + f.width, f.since = width, since + return f.byGroup[g], nil +} + +func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux { + t.Helper() + a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} + p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) + mux := http.NewServeMux() + p.RegisterAdminUsage(mux, s, priced) + return mux +} + +// hourly builds a group whose traffic rises over six slices, so a chart drawn +// from it has a shape rather than a dot. +func hourly(g store.Grouping, id, name string, scale int64) store.UsageSeries { + u := store.UsageSeries{Grouping: g, ID: id, Name: name} + base := time.Now().Add(-6 * time.Hour).Truncate(time.Hour) + for i := int64(1); i <= 6; i++ { + b := store.Bucket{ + Start: base.Add(time.Duration(i) * time.Hour), + Requests: i * scale, + Prompt: i * scale * 100, + Completion: i * scale * 20, + CostMicros: i * scale * 1000, + } + u.Buckets = append(u.Buckets, b) + u.Requests += b.Requests + u.Prompt += b.Prompt + u.Completion += b.Completion + u.CostMicros += b.CostMicros + } + return u +} + +func TestTheUsagePageDrawsAndTabulatesEachGroup(t *testing.T) { + s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByTeam: { + hourly(store.GroupByTeam, "t1", "platform", 3), + hourly(store.GroupByTeam, "t2", "research", 1), + }, + }} + body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage").Body.String() + + for _, want := range []string{"platform", "research"} { + if !strings.Contains(body, want) { + t.Errorf("%q is missing from the page", want) + } + } + // A chart is an SVG with lines in it, not a frame saying there is nothing. + if strings.Count(body, " s2.width) { + t.Errorf("a 30-day window bucketed at %s, a 24-hour one at %s", wide, s2.width) + } + if s2.width > time.Hour { + t.Errorf("a day in %s slices is too coarse to read", s2.width) + } +} + +func TestWithNoPricesTheSpendChartIsReplacedBySayingWhy(t *testing.T) { + s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByTeam: {hourly(store.GroupByTeam, "t1", "platform", 3)}, + }} + body := mirrorGet(t, usagePage(t, s, pricedNo{}), "/admin/usage").Body.String() + + // A spend chart flat along the bottom reads as a quiet month rather than as + // a cluster where nothing is priced. + if !strings.Contains(body, "No model has a price") { + t.Errorf("the page should say why spend is empty; got %s", body) + } + if strings.Count(body, " 2 { + t.Error("a spend chart was drawn although nothing is priced") + } +} + +func TestAPersonWithNoAccountIsNamedRatherThanBlank(t *testing.T) { + // Keys issued by an operator before accounts existed carry real traffic, + // and an unlabelled line in a legend is the one nobody can act on. + s := &fakeScopeSeries{byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByUser: {hourly(store.GroupByUser, "", "", 2)}, + }} + body := mirrorGet(t, usagePage(t, s, pricedYes{}), "/admin/usage?by=user").Body.String() + + if !strings.Contains(body, "keys with no owner") { + t.Errorf("unowned traffic was left unlabelled; got %s", body) + } +} diff --git a/internal/web/chart.go b/internal/web/chart.go index 592c051..386431f 100644 --- a/internal/web/chart.go +++ b/internal/web/chart.go @@ -3,6 +3,7 @@ package web import ( "fmt" "html/template" + "strconv" "strings" "time" ) @@ -120,6 +121,12 @@ func compact(v float64) string { return fmt.Sprintf("%.1fk", v/1e3) case v == float64(int64(v)): return fmt.Sprintf("%d", int64(v)) + case v > 0 && v < 1: + // Two significant digits rather than one decimal place. A spend of + // 0.03 printed as "0.0" says nothing was spent, which is the one thing + // a figure must not say when something was. Counts never reach this + // branch: a whole number is caught above, and zero is whole. + return strconv.FormatFloat(v, 'g', 2, 64) default: return fmt.Sprintf("%.1f", v) } diff --git a/internal/web/chart_test.go b/internal/web/chart_test.go index 9a41e4f..74ce4ac 100644 --- a/internal/web/chart_test.go +++ b/internal/web/chart_test.go @@ -104,3 +104,23 @@ func TestHourLabelsSayWhenAndHowMuch(t *testing.T) { t.Errorf("label = %q", got) } } + +func TestASmallFigureIsNotRenderedAsZero(t *testing.T) { + // A spend of 0.03 units shown as "0.0" says nothing was spent. On a cluster + // whose prices make a month's traffic a fraction of a unit, that is every + // figure on the page. + for _, c := range []struct { + in float64 + want string + }{ + {0.03, "0.03"}, + {0.5, "0.5"}, + {0, "0"}, + {1, "1"}, + {1500, "1.5k"}, + } { + if got := compact(c.in); got != c.want { + t.Errorf("compact(%v) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 5431edc..4cfa5e4 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -296,3 +296,24 @@ tr.editing form { margin: .4rem 0; } .legend i { width: 10px; height: 2px; border-radius: 1px; display: inline-block; } .legend b { font-weight: 600; color: var(--fg); font-variant-numeric: tabular-nums; } .chart-title { font-size: .9rem; font-weight: 600; margin: 0 0 .3rem; } + +/* The filter row above a report's charts. + One row, above the charts rather than beside them: the reader chooses what + they are looking at before they look, and a control column on the right is + found after the question has already been answered wrongly. Links rather + than a form, so the view is in the URL and can be shared. */ +.filters { display: flex; align-items: center; gap: .4rem; flex-wrap: wrap; + margin: 0 0 1rem; } +.filter-label { color: var(--muted); font-size: .78rem; text-transform: uppercase; + letter-spacing: .04em; margin-right: .2rem; } +.filter-label + .chip { margin-right: 0; } +.filters .filter-label:not(:first-child) { margin-left: 1rem; } +.chip { font-size: .85rem; text-decoration: none; color: var(--fg); + background: var(--card); border: 1px solid var(--line); + border-radius: 999px; padding: .25rem .7rem; } +.chip:hover { border-color: var(--accent); } +/* The selected one is filled, not merely outlined: an outline reads as + available rather than as chosen, and with four of them side by side the + difference has to survive a glance. */ +.chip.on { background: var(--accent); border-color: var(--accent); color: var(--bg); + font-weight: 600; } diff --git a/internal/web/templates/admin_quotas.html b/internal/web/templates/admin_quotas.html index 5f25620..e7aaf8b 100644 --- a/internal/web/templates/admin_quotas.html +++ b/internal/web/templates/admin_quotas.html @@ -7,7 +7,7 @@

Quotas

What each organization, team and project may use · spend over the last {{.Since}}

+ usage your keys {{if not .BudgetsBind}} diff --git a/internal/web/templates/admin_usage.html b/internal/web/templates/admin_usage.html new file mode 100644 index 0000000..7e344d9 --- /dev/null +++ b/internal/web/templates/admin_usage.html @@ -0,0 +1,80 @@ +{{define "admin_usage"}} + + +Usage · llmgw +
+
+

Usage over time

+

Who is using the cluster, and when

+ +
+ +{{if .Error}}

{{.Error}}

{{end}} + +{{/* One row of controls above the charts, which is where a reader looks for + them. Links rather than a script: the choice is in the URL, so a view worth + coming back to can be bookmarked or pasted to somebody else. */}} +
+ Group by + {{range .Groups}}{{.Label}}{{end}} + Window + {{range .Windows}}{{.Label}}{{end}} +
+ +
+ {{template "tschart" dict "t" .Requests "title" "Requests" + "note" "Requests served in each slice, counted where the key belongs now rather than where it was at the time."}} + {{template "tschart" dict "t" .Tokens "title" "Tokens" + "note" "Prompt and completion together, which is the figure a person recognises as what they consumed."}} +
+ +
+ {{if .Priced}} + {{template "tschart" dict "t" .Spend "title" "Spend" + "note" "In whole units, the same ones a budget is written in."}} + {{else}} + {{/* Said rather than drawn. A spend chart flat along the bottom reads as a + quiet month, and this is not that. */}} +
+

Spend

+

No model has a price, so every request costs zero and this + chart would be a flat line however busy the cluster is. Set prices in the + pricing block of the server configuration.

+
+ {{end}} +
+ +{{if .Rows}} + + + {{range .Rows}} + + + + + + + + {{end}} +
GroupRequestsTokensSpendErrors
{{.Name}}{{compact .Requests}}{{compact .Tokens}}{{.Spend}}{{if .Errors}}{{.Errors}}{{else}}—{{end}}
+{{else}} +

Nothing recorded in this window.

+{{end}} + +{{/* The charts fold their tail; the table does not. Both are here because they + answer different questions, and the one that is easy to get wrong is + thinking the four coloured lines are all of it. */}} +

The charts draw the four busiest groups by total and sum the +rest into “other”, so they always add up to the whole cluster. The table below +is every group, in the same order.

+ +

A person is not a level of the quota tree — one person holds +keys in several projects — so grouping by person crosses the tree rather than +naming a level of it. Quotas are written on the other three, on the +quotas page.

+ +
llmgw {{.Version}}{{.Now}}
+
{{end}} diff --git a/internal/web/templates/admin_users.html b/internal/web/templates/admin_users.html index 5371475..0c0385c 100644 --- a/internal/web/templates/admin_users.html +++ b/internal/web/templates/admin_users.html @@ -9,6 +9,7 @@ diff --git a/internal/web/timeseries.go b/internal/web/timeseries.go index 0c95c44..8d2062f 100644 --- a/internal/web/timeseries.go +++ b/internal/web/timeseries.go @@ -57,11 +57,30 @@ type LegendEntry struct { // One axis, always. Two measures of different scale belong in two charts: a // second y-axis lets any two lines be made to cross wherever the author likes, // and the reader has no way to know it happened. +// +// Ranked by peak, which is what a gauge asks: the model that hit the ceiling is +// the one worth a colour. For a counter, see LineChartTotals. func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries { + return lineChart(series, unit, maxSeries, promq.Series.Max) +} + +// LineChartTotals is LineChart ranked by the sum over the window. +// +// Which is the right question of a counter. "Who used the most" is answered by +// the total, and ranking by peak would give a colour to the team that made one +// large burst over the team that ran steadily all week — the opposite of what a +// usage report is read for. +func LineChartTotals(series []promq.Series, unit string, maxSeries int) TimeSeries { + return lineChart(series, unit, maxSeries, promq.Series.Sum) +} + +func lineChart(series []promq.Series, unit string, maxSeries int, + weight func(promq.Series) float64, +) TimeSeries { if maxSeries <= 0 || maxSeries > len(vizColours) { maxSeries = len(vizColours) } - series = foldSeries(series, maxSeries) + series = foldSeries(series, maxSeries, weight) if len(series) == 0 { return TimeSeries{Empty: true} } @@ -107,7 +126,10 @@ const otherLabel = "other" // Rather than dropping the tail, which would make a chart that does not add up, // or colouring it, which needs hues the palette does not have. The fold is // labelled so nobody reads it as a single model. -func foldSeries(in []promq.Series, keep int) []promq.Series { +// +// What "busiest" means is the caller's: a gauge is ranked by its peak and a +// counter by its total, and the two disagree exactly where it matters. +func foldSeries(in []promq.Series, keep int, weight func(promq.Series) float64) []promq.Series { clean := make([]promq.Series, 0, len(in)) for _, s := range in { if len(s.Points) > 0 { @@ -117,7 +139,7 @@ func foldSeries(in []promq.Series, keep int) []promq.Series { if len(clean) <= keep { return clean } - sort.SliceStable(clean, func(i, j int) bool { return clean[i].Max() > clean[j].Max() }) + sort.SliceStable(clean, func(i, j int) bool { return weight(clean[i]) > weight(clean[j]) }) rest := clean[keep-1:] folded := promq.Series{Name: otherLabel} diff --git a/internal/web/web.go b/internal/web/web.go index 6adc9fa..475b204 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -130,6 +130,14 @@ var funcs = template.FuncMap{ }, } +// unitsOf renders micro-units as the whole units a budget is written in. +// +// A function rather than only a template helper, because the usage report +// formats the same figure into a table row in Go and the two must not drift. +func unitsOf(micros int64) string { + return fmt.Sprintf("%.2f", float64(micros)/1e6) +} + var tmpl = template.Must(template.New("").Funcs(funcs).ParseFS(files, "templates/*.html")) // Static serves the embedded assets, mounted on both listeners because both