diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bee72f..d743467 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,10 @@ and a figure the driver does not give is left out rather than shown as zero. on the GPU map; a Grafana panel; and alerts — `GPUXidError`, `GPUOutOfSpareRows`, `GPUNeedsReset`, `GPUPCIeRetriesClimbing`, `GPUNVLinkErrorsClimbing`. +- **GPU time by owner** on the usage page: the cards each owner's models + held over the window, charted and summed into GPU-hours, whether or not the + models were asked anything — the time a GPU quota is about. From the + metrics store, as far back as it keeps history. - **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds each card and whose quota it counts against, for joining the GPU series to models and tenants. diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 7a3df40..b758f11 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -203,6 +203,9 @@ func (r *Router) Handler() http.Handler { } pages.RegisterAdminQuotas(mux, r.store, r.acct) // And the report of what that tree actually used. + if r.history != nil { + pages.SetHistory(r.history) + } pages.RegisterAdminUsage(mux, r.store, r.acct) } diff --git a/internal/web/account.go b/internal/web/account.go index 07925dc..c47a52f 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -64,6 +64,9 @@ type AccountPages struct { // hasQuotas is set when /admin/quotas is registered. hasQuotas bool // capacity sets capacity quotas, from the quotas page; nil leaves them off. + // history is the metrics store, for what only it knows: the GPU time + // each owner held. Nil without one. + history History capacity CapacityAdmin // personalOff refuses personal keys; see SetPersonalOff. personalOff bool @@ -651,3 +654,6 @@ func errorRef() string { _, _ = rand.Read(b[:]) return "err-" + hex.EncodeToString(b[:]) } + +// SetHistory gives the pages the metrics store. +func (p *AccountPages) SetHistory(h History) { p.history = h } diff --git a/internal/web/admin_usage.go b/internal/web/admin_usage.go index 29ed43e..2092508 100644 --- a/internal/web/admin_usage.go +++ b/internal/web/admin_usage.go @@ -3,6 +3,7 @@ package web import ( "context" "net/http" + "sort" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/promq" @@ -80,6 +81,12 @@ type adminUsageData struct { // chart along the bottom reads as "nobody spent anything". Priced bool + // GPUHeld and GPUHours are the cards each owner's models held over the + // window, from the metrics store; empty without one, or with no owned + // model on a GPU. + GPUHeld TimeSeries + GPUHours []gpuHoursRow + Error string Version string Now string @@ -135,9 +142,46 @@ func (p *AccountPages) renderUsage(w http.ResponseWriter, r *http.Request, } d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows) + d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win) render(w, "admin_usage", d) } +// gpuHoursRow is the GPU time one owner's models held over the window. +type gpuHoursRow struct { + Owner string + Hours float64 + // Now is how many cards they hold at the end of the window. + Now float64 +} + +// gpuHours is the cards held by each owner's models over the window, drawn +// and summed into GPU-hours. From the metrics store, because what a model +// holds is placement's, not the request log's: a model nobody asks still +// holds its cards. As far back as the store keeps history, which for a +// default Prometheus is fifteen days. +func (p *AccountPages) gpuHours(ctx context.Context, win usageWindow) (TimeSeries, []gpuHoursRow) { + if p.history == nil { + return TimeSeries{}, nil + } + series, err := p.history.RangeQuery(ctx, `sum by (owner) (llmgw_gpu_assigned)`, "owner", win.Span, win.Width) + if err != nil || len(series) == 0 { + return TimeSeries{}, nil + } + var rows []gpuHoursRow + for i := range series { + if series[i].Name == "" { + series[i].Name = "no owner" + } + row := gpuHoursRow{Owner: series[i].Name, Now: series[i].Last()} + // Each point is the cards held at that instant, standing for the + // step that follows it. + row.Hours = series[i].Sum() * win.Width.Hours() + rows = append(rows, row) + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Hours > rows[j].Hours }) + return LineChart(series, "", 6), rows +} + // seriesOf turns each group's buckets into a line of one measure. // // One pass per measure rather than one query per measure: the buckets are diff --git a/internal/web/admin_usage_test.go b/internal/web/admin_usage_test.go index de8da22..c157d15 100644 --- a/internal/web/admin_usage_test.go +++ b/internal/web/admin_usage_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "gitea.mynodes.xyz/ilker/llmgw/internal/promq" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -162,3 +163,32 @@ func TestAPersonWithNoAccountIsNamedRatherThanBlank(t *testing.T) { t.Errorf("unowned traffic was left unlabelled; got %s", body) } } + +func TestTheUsagePageCountsTheGPUTimeEachOwnerHeld(t *testing.T) { + a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} + p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil))) + // Over a day in 20-minute steps: team:research held 8 cards throughout, + // nobody's model 2 for the last three steps. + res, none := promq.Series{Name: "team:research"}, promq.Series{Name: ""} + for i := range 72 { + res.Points = append(res.Points, promq.Point{Value: 8}) + if i >= 69 { + none.Points = append(none.Points, promq.Point{Value: 2}) + } + } + h := &fakeHistory{byExpr: map[string][]promq.Series{`sum by (owner) (llmgw_gpu_assigned)`: {res, none}}} + p.SetHistory(h) + mux := http.NewServeMux() + p.RegisterAdminUsage(mux, &fakeScopeSeries{}, nil) + body := mirrorGet(t, mux, "/admin/usage?window=24h").Body.String() + for _, want := range []string{"GPU time, by owner", "
Nothing recorded in this window.
{{end}} +{{with .GPUHours}} +| Owner | GPU-hours | Held now |
|---|---|---|
| {{.Owner}} | {{printf "%.1f" .Hours}} | {{printf "%.0f" .Now}} |
From the metrics store, which keeps its history for as long as it is set to (fifteen days by default); a model with no owner is counted under “no owner”.
+{{end}} + {{/* The charts fold their tail; the table does not. Both are here because they answer different questions, and the one that is easy to get wrong is thinking the four coloured lines are all of it. */}}