Count the GPU time each owner's models held, on the usage page

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

CHANGELOG.md

@@ -67,6 +67,10 @@ and a figure the driver does not give is left out rather than shown as zero.
   on the GPU map; a Grafana panel; and alerts — `GPUXidError`,
   `GPUOutOfSpareRows`, `GPUNeedsReset`, `GPUPCIeRetriesClimbing`,
   `GPUNVLinkErrorsClimbing`.
+- **GPU time by owner** on the usage page: the cards each owner's models
+  held over the window, charted and summed into GPU-hours, whether or not the
+  models were asked anything — the time a GPU quota is about. From the
+  metrics store, as far back as it keeps history.
 - **`llmgw_gpu_assigned{node,gpu,model,replica,owner}`**: which replica holds
   each card and whose quota it counts against, for joining the GPU series to
   models and tenants.

internal/router/handlers.go

@@ -203,6 +203,9 @@ func (r *Router) Handler() http.Handler {
 			}
 			pages.RegisterAdminQuotas(mux, r.store, r.acct)
 			// And the report of what that tree actually used.
+			if r.history != nil {
+				pages.SetHistory(r.history)
+			}
 			pages.RegisterAdminUsage(mux, r.store, r.acct)
 		}
 

internal/web/account.go

@@ -64,6 +64,9 @@ type AccountPages struct {
 	// hasQuotas is set when /admin/quotas is registered.
 	hasQuotas bool
 	// capacity sets capacity quotas, from the quotas page; nil leaves them off.
+	// history is the metrics store, for what only it knows: the GPU time
+	// each owner held. Nil without one.
+	history  History
 	capacity CapacityAdmin
 	// personalOff refuses personal keys; see SetPersonalOff.
 	personalOff bool
@@ -651,3 +654,6 @@ func errorRef() string {
 	_, _ = rand.Read(b[:])
 	return "err-" + hex.EncodeToString(b[:])
 }
+
+// SetHistory gives the pages the metrics store.
+func (p *AccountPages) SetHistory(h History) { p.history = h }

internal/web/admin_usage.go

@@ -3,6 +3,7 @@ package web
 import (
 	"context"
 	"net/http"
+	"sort"
 	"time"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
@@ -80,6 +81,12 @@ type adminUsageData struct {
 	// chart along the bottom reads as "nobody spent anything".
 	Priced bool
 
+	// GPUHeld and GPUHours are the cards each owner's models held over the
+	// window, from the metrics store; empty without one, or with no owned
+	// model on a GPU.
+	GPUHeld  TimeSeries
+	GPUHours []gpuHoursRow
+
 	Error   string
 	Version string
 	Now     string
@@ -135,9 +142,46 @@ func (p *AccountPages) renderUsage(w http.ResponseWriter, r *http.Request,
 	}
 
 	d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows)
+	d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win)
 	render(w, "admin_usage", d)
 }
 
+// gpuHoursRow is the GPU time one owner's models held over the window.
+type gpuHoursRow struct {
+	Owner string
+	Hours float64
+	// Now is how many cards they hold at the end of the window.
+	Now float64
+}
+
+// gpuHours is the cards held by each owner's models over the window, drawn
+// and summed into GPU-hours. From the metrics store, because what a model
+// holds is placement's, not the request log's: a model nobody asks still
+// holds its cards. As far back as the store keeps history, which for a
+// default Prometheus is fifteen days.
+func (p *AccountPages) gpuHours(ctx context.Context, win usageWindow) (TimeSeries, []gpuHoursRow) {
+	if p.history == nil {
+		return TimeSeries{}, nil
+	}
+	series, err := p.history.RangeQuery(ctx, `sum by (owner) (llmgw_gpu_assigned)`, "owner", win.Span, win.Width)
+	if err != nil || len(series) == 0 {
+		return TimeSeries{}, nil
+	}
+	var rows []gpuHoursRow
+	for i := range series {
+		if series[i].Name == "" {
+			series[i].Name = "no owner"
+		}
+		row := gpuHoursRow{Owner: series[i].Name, Now: series[i].Last()}
+		// Each point is the cards held at that instant, standing for the
+		// step that follows it.
+		row.Hours = series[i].Sum() * win.Width.Hours()
+		rows = append(rows, row)
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].Hours > rows[j].Hours })
+	return LineChart(series, "", 6), rows
+}
+
 // seriesOf turns each group's buckets into a line of one measure.
 //
 // One pass per measure rather than one query per measure: the buckets are

internal/web/admin_usage_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 )
 
@@ -162,3 +163,32 @@ func TestAPersonWithNoAccountIsNamedRatherThanBlank(t *testing.T) {
 		t.Errorf("unowned traffic was left unlabelled; got %s", body)
 	}
 }
+
+func TestTheUsagePageCountsTheGPUTimeEachOwnerHeld(t *testing.T) {
+	a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}}
+	p := NewAccountPages(a, nil, nil, "v0", slog.New(slog.NewTextHandler(io.Discard, nil)))
+	// Over a day in 20-minute steps: team:research held 8 cards throughout,
+	// nobody's model 2 for the last three steps.
+	res, none := promq.Series{Name: "team:research"}, promq.Series{Name: ""}
+	for i := range 72 {
+		res.Points = append(res.Points, promq.Point{Value: 8})
+		if i >= 69 {
+			none.Points = append(none.Points, promq.Point{Value: 2})
+		}
+	}
+	h := &fakeHistory{byExpr: map[string][]promq.Series{`sum by (owner) (llmgw_gpu_assigned)`: {res, none}}}
+	p.SetHistory(h)
+	mux := http.NewServeMux()
+	p.RegisterAdminUsage(mux, &fakeScopeSeries{}, nil)
+	body := mirrorGet(t, mux, "/admin/usage?window=24h").Body.String()
+	for _, want := range []string{"GPU time, by owner", "<td>team:research</td><td class=\"num\">192.0</td>",
+		"<td>no owner</td><td class=\"num\">2.0</td>"} {
+		if !strings.Contains(body, want) {
+			t.Errorf("the page lacks %q", want)
+		}
+	}
+	// Without a metrics store, nothing is said about GPU time.
+	if body := mirrorGet(t, usagePage(t, &fakeScopeSeries{}, nil), "/admin/usage").Body.String(); strings.Contains(body, "GPU time") {
+		t.Error("GPU time shown with no metrics store to read it from")
+	}
+}

internal/web/templates/admin_usage.html

@@ -63,6 +63,21 @@
 <p class="empty">Nothing recorded in this window.</p>
 {{end}}
 
+{{with .GPUHours}}
+<h2>GPU time, by owner</h2>
+{{/* What each owner's models held, whether or not anybody asked them
+     anything: a model kept loaded holds its cards all the same, and that is
+     the time a GPU quota is about. */}}
+<div class="chart-row">
+  {{template "tschart" dict "t" $.GPUHeld "title" "GPUs held, by owner" "note" "The cards held by the replicas of each owner's models: models whose owner is a part of the quota tree."}}
+</div>
+<table class="rows">
+  <tr><th>Owner</th><th class="num">GPU-hours</th><th class="num">Held now</th></tr>
+  {{range .}}<tr><td>{{.Owner}}</td><td class="num">{{printf "%.1f" .Hours}}</td><td class="num">{{printf "%.0f" .Now}}</td></tr>{{end}}
+</table>
+<p class="note">From the metrics store, which keeps its history for as long as it is set to (fifteen days by default); a model with no owner is counted under “no owner”.</p>
+{{end}}
+
 {{/* The charts fold their tail; the table does not. Both are here because they
      answer different questions, and the one that is easy to get wrong is
      thinking the four coloured lines are all of it. */}}