Quotas can be written on an organization, a team or a project, and nothing could show what any of them had used: usage came back per key, per user and per model only. A quota whose consumption cannot be seen is one nobody reviews until the day it refuses somebody — and since the previous commit made a group quota a shared pool, it is now a number several people are spending against without being able to look at it. `llmgw ctl usage --by team` sums every key beneath each group, with the budget that level sets beside it. Beside it, and only when that level sets one. A team bounded by its organization's budget has none of its own, and printing the organization's against each of its teams would read as though each had that much — the exact misreading this work exists to remove. Joined through api_keys rather than read from usage.project_id: that column records where the key was when the request was served, while the quota is written against the tree. They agree until a key is moved, and a report that disagrees with the limit shown beside it is the useless one. Three existing reports were fixed on the way, all failing on PostgreSQL and all passing in tests, because the tests run on SQLite: column "k.name" must appear in the GROUP BY clause (UsageSince) column "us.email" must appear in the GROUP BY clause (UsageByUser) column "usage.ts" must appear in the GROUP BY clause (UsageBuckets) SQLite picks an arbitrary row for an ungrouped column and carries on; PostgreSQL refuses. The first two were selecting names they had not grouped. The third grouped "ts / ?" while selecting "(ts / ?) * ?", which PostgreSQL matches as text and so saw a bare ts — it is grouped by ordinal now, the same expression by construction. The dashboard's usage charts have been empty on the cluster this whole time, with the reason only in the log. All four queries were run against the cluster's own PostgreSQL before this was committed, because passing on SQLite is what they were already doing. The grouping level picks from a fixed table of levels rather than being interpolated, so an unknown one is an error and not a fragment of SQL. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
124 lines
4.2 KiB
Go
124 lines
4.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"time"
|
|
)
|
|
|
|
// ScopeUsage is what one group of the tenancy tree spent.
|
|
//
|
|
// The report that was missing. Quotas could be written on an organization, a
|
|
// team or a project, and nothing could show what any of them had used: usage
|
|
// came back per key, per user and per model only. A quota you cannot see the
|
|
// consumption of is a quota nobody reviews until it refuses somebody.
|
|
type ScopeUsage struct {
|
|
Scope Scope
|
|
ID string
|
|
Name string
|
|
|
|
Keys int64
|
|
Requests int64
|
|
Prompt int64
|
|
Completion int64
|
|
CostMicros int64
|
|
Errors int64
|
|
LastAt time.Time
|
|
|
|
// Budget is what was set at this scope, in micro-units, and nil when this
|
|
// level sets none.
|
|
//
|
|
// What is set here, not what binds. A team with no budget of its own may
|
|
// still be bounded by its organization's, and printing the inherited figure
|
|
// beside every team would read as though each had that much to itself —
|
|
// which is the misunderstanding this whole change exists to remove.
|
|
Budget *int64
|
|
BudgetPeriod Period
|
|
}
|
|
|
|
// Spent reports the fraction of this scope's budget used, and false when there
|
|
// is no budget here to be a fraction of.
|
|
func (s ScopeUsage) Spent() (float64, bool) {
|
|
if s.Budget == nil || *s.Budget <= 0 {
|
|
return 0, false
|
|
}
|
|
return float64(s.CostMicros) / float64(*s.Budget), true
|
|
}
|
|
|
|
// scopeColumns is the group id and name for each level, and how usage reaches
|
|
// it. Written out per level rather than built by string pasting: these are
|
|
// table and column names going into SQL, and the one place they must not come
|
|
// from anywhere but here is the place they are interpolated.
|
|
var scopeColumns = map[Scope]struct{ id, name, joins string }{
|
|
ScopeOrg: {"o.id", "o.name", `
|
|
JOIN api_keys k ON k.id = u.key_id
|
|
JOIN projects p ON p.id = k.project_id
|
|
JOIN teams t ON t.id = p.team_id
|
|
JOIN organizations o ON o.id = t.org_id`},
|
|
ScopeTeam: {"t.id", "t.name", `
|
|
JOIN api_keys k ON k.id = u.key_id
|
|
JOIN projects p ON p.id = k.project_id
|
|
JOIN teams t ON t.id = p.team_id`},
|
|
ScopeProject: {"p.id", "p.name", `
|
|
JOIN api_keys k ON k.id = u.key_id
|
|
JOIN projects p ON p.id = k.project_id`},
|
|
}
|
|
|
|
// UsageByScope totals usage for one level of the tenancy tree.
|
|
//
|
|
// Joined through api_keys rather than read from usage.project_id, because that
|
|
// column records where the key was when the request was served and the tree is
|
|
// what the quota is written against. They agree until a key is moved, and the
|
|
// report that disagrees with the limit it is shown beside is the useless one.
|
|
func (s *DB) UsageByScope(ctx context.Context, scope Scope, since time.Time) ([]ScopeUsage, error) {
|
|
c, ok := scopeColumns[scope]
|
|
if !ok {
|
|
return nil, fmt.Errorf("usage cannot be grouped by %q; try org, team or project", scope)
|
|
}
|
|
|
|
// Every non-aggregated column is grouped, and the budget is joined by the
|
|
// same expression it is grouped on. PostgreSQL requires both; SQLite
|
|
// silently picks a row for whatever is missing, which is how three of these
|
|
// queries shipped working in tests and failing on the cluster.
|
|
q := `
|
|
SELECT ` + c.id + `, ` + c.name + `,
|
|
COUNT(DISTINCT u.key_id),
|
|
COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens),
|
|
SUM(u.cost_micros),
|
|
SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END),
|
|
MAX(u.ts),
|
|
MAX(l.budget_micros), MAX(COALESCE(l.budget_period, ''))
|
|
FROM usage u` + c.joins + `
|
|
LEFT JOIN limits l ON l.scope_type = ? AND l.scope_id = ` + c.id + `
|
|
WHERE u.ts >= ?
|
|
GROUP BY ` + c.id + `, ` + c.name + `
|
|
ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`
|
|
|
|
rows, err := s.db.QueryContext(ctx, q, string(scope), millis(since))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var out []ScopeUsage
|
|
for rows.Next() {
|
|
su := ScopeUsage{Scope: scope}
|
|
var last int64
|
|
var budget sql.NullInt64
|
|
var period string
|
|
if err := rows.Scan(&su.ID, &su.Name, &su.Keys,
|
|
&su.Requests, &su.Prompt, &su.Completion, &su.CostMicros, &su.Errors,
|
|
&last, &budget, &period); err != nil {
|
|
return nil, err
|
|
}
|
|
su.LastAt = fromMillis(last)
|
|
if budget.Valid {
|
|
v := budget.Int64
|
|
su.Budget, su.BudgetPeriod = &v, Period(period)
|
|
}
|
|
out = append(out, su)
|
|
}
|
|
return out, rows.Err()
|
|
}
|