diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 9a14e3f..a8dd6fa 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -135,6 +135,7 @@ func newCtlCmd() *cobra.Command { newToolCheckCmd(), newWeightsCmd(client, &asJSON), newCapacityCmd(client, &asJSON), + newUsageCmd(client, &asJSON), newCertCmd(), ) return cmd diff --git a/internal/cli/server.go b/internal/cli/server.go index afdf7cd..37ae0af 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -162,6 +162,9 @@ func newServerCmd() *cobra.Command { // GPU time; without somewhere to put it the number lives in // whoever ran it's terminal and is gone by morning. api.SetCapacity(st) + // And what each group of the tenancy tree spent, which is + // the other half of a quota being reviewable. + api.SetUsage(st) } api.Register(adminMux) diff --git a/internal/cli/usage.go b/internal/cli/usage.go new file mode 100644 index 0000000..41311b9 --- /dev/null +++ b/internal/cli/usage.go @@ -0,0 +1,74 @@ +package cli + +import ( + "encoding/json" + "fmt" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// newUsageCmd shows what each group of the tenancy tree spent. +// +// Quotas could be written on an organization, a team or a project long before +// anything could show what any of them had used: usage came back per key, per +// user and per model only. A quota whose consumption cannot be seen is one +// nobody reviews until the day it refuses somebody. +func newUsageCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + var by string + var since time.Duration + + c := &cobra.Command{ + Use: "usage", + Short: "What each organization, team or project has spent", + Long: "Grouped by a level of the tenancy tree, which is the level a quota is\n" + + "written on. Spend is summed across every key beneath it, so two people\n" + + "sharing a team budget appear as one number — the number that will refuse\n" + + "them.\n\n" + + "The budget column is what that level sets, and is blank when it sets none.\n" + + "A team bounded only by its organization has no budget of its own, and\n" + + "printing the organization's beside each of its teams would read as though\n" + + "each had that much to spend.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var rows []adminapi.ScopeUsageView + path := fmt.Sprintf("/api/v1/usage?by=%s&since=%s", by, since) + if err := client().get(cmd.Context(), path, &rows); err != nil { + return err + } + if *asJSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(rows) + } + if len(rows) == 0 { + fmt.Fprintf(cmd.OutOrStdout(), "No %s used anything in the last %s.\n", by, since) + return nil + } + + w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "NAME\tID\tKEYS\tREQUESTS\tERRORS\tTOKENS\tSPEND\tBUDGET\tUSED\tLAST") + for _, u := range rows { + budget, used := "—", "—" + if u.Budget != nil { + budget = fmt.Sprintf("%.2f", float64(*u.Budget)/1e6) + if u.BudgetPeriod != "" { + budget += "/" + u.BudgetPeriod + } + } + if u.Fraction != nil { + used = fmt.Sprintf("%.0f%%", *u.Fraction*100) + } + fmt.Fprintf(w, "%s\t%s\t%d\t%d\t%d\t%d\t%.2f\t%s\t%s\t%s\n", + u.Name, u.ID, u.Keys, u.Requests, u.Errors, + u.Prompt+u.Completion, float64(u.CostMicros)/1e6, + budget, used, age(u.LastAt)) + } + return w.Flush() + }, + } + c.Flags().StringVar(&by, "by", "team", "level to group by: org, team or project") + c.Flags().DurationVar(&since, "since", 24*time.Hour, "how far back to total") + return c +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 740b16b..ad6c3c7 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -27,6 +27,7 @@ import ( type API struct { models Models capacity Capacity + usage Usage ctl *control.Controller } @@ -44,6 +45,7 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/models/{name}/scale", a.scaleModel) mux.HandleFunc("PUT /api/v1/models/{name}/capacity", a.putCapacity) mux.HandleFunc("GET /api/v1/capacity", a.listCapacity) + mux.HandleFunc("GET /api/v1/usage", a.usageByScope) mux.HandleFunc("GET /api/v1/nodes", a.listNodes) mux.HandleFunc("GET /api/v1/models", a.listModels) mux.HandleFunc("GET /api/v1/replicas", a.listReplicas) @@ -314,6 +316,80 @@ type Models interface { Scale(ctx context.Context, name string, s catalog.Scaling, by string) (catalog.Model, error) } +// Usage reports what each group of the tenancy tree spent. The store +// implements it. +type Usage interface { + UsageByScope(ctx context.Context, scope store.Scope, since time.Time) ([]store.ScopeUsage, error) +} + +// SetUsage supplies the usage reporting source. +func (a *API) SetUsage(u Usage) { a.usage = u } + +// ScopeUsageView is one group's spend, with its quota when it has one. +type ScopeUsageView struct { + Scope string `json:"scope"` + ID string `json:"id"` + Name string `json:"name"` + + Keys int64 `json:"keys"` + Requests int64 `json:"requests"` + Prompt int64 `json:"prompt_tokens"` + Completion int64 `json:"completion_tokens"` + CostMicros int64 `json:"cost_micros"` + Errors int64 `json:"errors"` + + LastAt time.Time `json:"last_at,omitzero"` + // Budget is what this level sets, absent when it sets none. A group bounded + // only by its parent has no budget of its own, and reporting the parent's + // against each child would read as though each had that much. + Budget *int64 `json:"budget_micros,omitempty"` + BudgetPeriod string `json:"budget_period,omitempty"` + // Fraction is spend over budget, absent for the same reason. + Fraction *float64 `json:"fraction,omitempty"` +} + +func (a *API) usageByScope(w http.ResponseWriter, r *http.Request) { + if a.usage == nil { + writeError(w, http.StatusNotImplemented, "this control plane has no store, so nothing is recorded") + return + } + scope := store.Scope(r.URL.Query().Get("by")) + if scope == "" { + scope = store.ScopeTeam + } + since := 24 * time.Hour + if v := r.URL.Query().Get("since"); v != "" { + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + writeError(w, http.StatusBadRequest, "since must be a positive duration, like 24h") + return + } + since = d + } + + rows, err := a.usage.UsageByScope(r.Context(), scope, time.Now().Add(-since)) + if err != nil { + // The only failure a caller can act on is naming a level that is not + // one, which is what this mostly is. + writeError(w, http.StatusBadRequest, err.Error()) + return + } + out := make([]ScopeUsageView, 0, len(rows)) + for _, u := range rows { + v := ScopeUsageView{ + Scope: string(u.Scope), ID: u.ID, Name: u.Name, + Keys: u.Keys, Requests: u.Requests, Prompt: u.Prompt, + Completion: u.Completion, CostMicros: u.CostMicros, Errors: u.Errors, + LastAt: u.LastAt, Budget: u.Budget, BudgetPeriod: string(u.BudgetPeriod), + } + if f, ok := u.Spent(); ok { + v.Fraction = &f + } + out = append(out, v) + } + writeJSON(w, http.StatusOK, out) +} + // Capacity is where measured capacity is kept. The store implements it. type Capacity interface { PutCapacity(ctx context.Context, c store.StoredCapacity) error diff --git a/internal/control/adminapi/api_test.go b/internal/control/adminapi/api_test.go index daf7c48..7be2ae9 100644 --- a/internal/control/adminapi/api_test.go +++ b/internal/control/adminapi/api_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control" @@ -533,3 +534,91 @@ func TestCapacityWithoutAStoreIsAnEmptyListRatherThanAnError(t *testing.T) { t.Errorf("expected an empty list, got %s", rec.Body) } } + +type memUsage struct { + rows []store.ScopeUsage + scope store.Scope + since time.Time + err error +} + +func (m *memUsage) UsageByScope(_ context.Context, s store.Scope, since time.Time) ([]store.ScopeUsage, error) { + m.scope, m.since = s, since + return m.rows, m.err +} + +func usageAPI(t *testing.T, u *memUsage) *http.ServeMux { + t.Helper() + a, _ := setup(t, testModels()) + a.SetUsage(u) + mux := http.NewServeMux() + a.Register(mux) + return mux +} + +func TestUsageDefaultsToTeamsOverTheLastDay(t *testing.T) { + u := &memUsage{} + rec := httptest.NewRecorder() + usageAPI(t, u).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/usage", nil)) + + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body) + } + // The team is the level a quota is usually written on, so it is the one + // somebody asking for "usage" with no argument means. + if u.scope != store.ScopeTeam { + t.Errorf("expected teams by default, got %q", u.scope) + } + if d := time.Since(u.since); d < 23*time.Hour || d > 25*time.Hour { + t.Errorf("expected roughly a day, got %v", d) + } +} + +func TestUsageReportsTheFractionOnlyWhenThereIsABudget(t *testing.T) { + budget := int64(2000) + u := &memUsage{rows: []store.ScopeUsage{ + {Scope: store.ScopeTeam, ID: "t1", Name: "platform", CostMicros: 1000, + Budget: &budget, BudgetPeriod: store.PeriodMonth}, + {Scope: store.ScopeTeam, ID: "t2", Name: "research", CostMicros: 100}, + }} + rec := httptest.NewRecorder() + usageAPI(t, u).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/v1/usage", nil)) + + var got []ScopeUsageView + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if got[0].Fraction == nil || *got[0].Fraction != 0.5 { + t.Errorf("expected half the budget used, got %v", got[0].Fraction) + } + // A team bounded only by its organization has no budget of its own; a + // fraction of nothing would have to invent a denominator. + if got[1].Fraction != nil || got[1].Budget != nil { + t.Errorf("a group with no budget should report neither, got %+v", got[1]) + } +} + +func TestAnUnknownGroupingIsTheCallersMistake(t *testing.T) { + u := &memUsage{err: errors.New(`usage cannot be grouped by "model"`)} + rec := httptest.NewRecorder() + usageAPI(t, u).ServeHTTP(rec, + httptest.NewRequest(http.MethodGet, "/api/v1/usage?by=model", nil)) + + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", rec.Code) + } +} + +func TestABadSinceIsRefusedBeforeTheQuery(t *testing.T) { + u := &memUsage{} + rec := httptest.NewRecorder() + usageAPI(t, u).ServeHTTP(rec, + httptest.NewRequest(http.MethodGet, "/api/v1/usage?since=lastweek", nil)) + + if rec.Code != http.StatusBadRequest { + t.Errorf("expected 400, got %d", rec.Code) + } + if !u.since.IsZero() { + t.Error("nothing should have reached the store") + } +} diff --git a/internal/store/usagescope.go b/internal/store/usagescope.go new file mode 100644 index 0000000..1a41e8c --- /dev/null +++ b/internal/store/usagescope.go @@ -0,0 +1,123 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// ScopeUsage is what one group of the tenancy tree spent. +// +// The report that was missing. Quotas could be written on an organization, a +// team or a project, and nothing could show what any of them had used: usage +// came back per key, per user and per model only. A quota you cannot see the +// consumption of is a quota nobody reviews until it refuses somebody. +type ScopeUsage struct { + Scope Scope + ID string + Name string + + Keys int64 + Requests int64 + Prompt int64 + Completion int64 + CostMicros int64 + Errors int64 + LastAt time.Time + + // Budget is what was set at this scope, in micro-units, and nil when this + // level sets none. + // + // What is set here, not what binds. A team with no budget of its own may + // still be bounded by its organization's, and printing the inherited figure + // beside every team would read as though each had that much to itself — + // which is the misunderstanding this whole change exists to remove. + Budget *int64 + BudgetPeriod Period +} + +// Spent reports the fraction of this scope's budget used, and false when there +// is no budget here to be a fraction of. +func (s ScopeUsage) Spent() (float64, bool) { + if s.Budget == nil || *s.Budget <= 0 { + return 0, false + } + return float64(s.CostMicros) / float64(*s.Budget), true +} + +// scopeColumns is the group id and name for each level, and how usage reaches +// it. Written out per level rather than built by string pasting: these are +// table and column names going into SQL, and the one place they must not come +// from anywhere but here is the place they are interpolated. +var scopeColumns = map[Scope]struct{ id, name, joins string }{ + ScopeOrg: {"o.id", "o.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id + JOIN teams t ON t.id = p.team_id + JOIN organizations o ON o.id = t.org_id`}, + ScopeTeam: {"t.id", "t.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id + JOIN teams t ON t.id = p.team_id`}, + ScopeProject: {"p.id", "p.name", ` + JOIN api_keys k ON k.id = u.key_id + JOIN projects p ON p.id = k.project_id`}, +} + +// UsageByScope totals usage for one level of the tenancy tree. +// +// Joined through api_keys rather than read from usage.project_id, because that +// column records where the key was when the request was served and the tree is +// what the quota is written against. They agree until a key is moved, and the +// report that disagrees with the limit it is shown beside is the useless one. +func (s *DB) UsageByScope(ctx context.Context, scope Scope, since time.Time) ([]ScopeUsage, error) { + c, ok := scopeColumns[scope] + if !ok { + return nil, fmt.Errorf("usage cannot be grouped by %q; try org, team or project", scope) + } + + // Every non-aggregated column is grouped, and the budget is joined by the + // same expression it is grouped on. PostgreSQL requires both; SQLite + // silently picks a row for whatever is missing, which is how three of these + // queries shipped working in tests and failing on the cluster. + q := ` + SELECT ` + c.id + `, ` + c.name + `, + COUNT(DISTINCT u.key_id), + COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), + SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), + MAX(u.ts), + MAX(l.budget_micros), MAX(COALESCE(l.budget_period, '')) + FROM usage u` + c.joins + ` + LEFT JOIN limits l ON l.scope_type = ? AND l.scope_id = ` + c.id + ` + WHERE u.ts >= ? + GROUP BY ` + c.id + `, ` + c.name + ` + ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC` + + rows, err := s.db.QueryContext(ctx, q, string(scope), millis(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []ScopeUsage + for rows.Next() { + su := ScopeUsage{Scope: scope} + var last int64 + var budget sql.NullInt64 + var period string + if err := rows.Scan(&su.ID, &su.Name, &su.Keys, + &su.Requests, &su.Prompt, &su.Completion, &su.CostMicros, &su.Errors, + &last, &budget, &period); err != nil { + return nil, err + } + su.LastAt = fromMillis(last) + if budget.Valid { + v := budget.Int64 + su.Budget, su.BudgetPeriod = &v, Period(period) + } + out = append(out, su) + } + return out, rows.Err() +} diff --git a/internal/store/usagescope_test.go b/internal/store/usagescope_test.go new file mode 100644 index 0000000..14f584e --- /dev/null +++ b/internal/store/usagescope_test.go @@ -0,0 +1,158 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +// twoTeams is one org, two teams, a project each, a key each. +func twoTeams(t *testing.T) *DB { + t.Helper() + ctx := context.Background() + s, err := OpenSQLite(ctx, filepath.Join(t.TempDir(), "u.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.PutOrg(ctx, Org{ID: "o", Name: "Acme"})) + must(s.PutTeam(ctx, Team{ID: "t1", OrgID: "o", Name: "platform"})) + must(s.PutTeam(ctx, Team{ID: "t2", OrgID: "o", Name: "research"})) + must(s.PutProject(ctx, Project{ID: "p1", TeamID: "t1", Name: "gateway"})) + must(s.PutProject(ctx, Project{ID: "p2", TeamID: "t2", Name: "labs"})) + for id, proj := range map[string]string{"k1": "p1", "k2": "p1", "k3": "p2"} { + _, k, _ := NewKey(id, "n", proj) + k.CreatedAt = time.Now() + must(s.CreateKey(ctx, k)) + } + must(s.WriteUsage(ctx, []Usage{ + {At: time.Now(), KeyID: "k1", ProjectID: "p1", Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 600, Status: 200}, + {At: time.Now(), KeyID: "k2", ProjectID: "p1", Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 400, Status: 500}, + {At: time.Now(), KeyID: "k3", ProjectID: "p2", Model: "m", ServedModel: "m", + PromptTokens: 1, CompletionTokens: 1, CostMicros: 100, Status: 200}, + })) + return s +} + +func byID(rows []ScopeUsage, id string) (ScopeUsage, bool) { + for _, r := range rows { + if r.ID == id { + return r, true + } + } + return ScopeUsage{}, false +} + +func TestATeamsSpendIsTheSumOfItsKeys(t *testing.T) { + s := twoTeams(t) + + rows, err := s.UsageByScope(t.Context(), ScopeTeam, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + // Two members of one team, spending separately, is one number for the team. + // Per-key totals could not answer this at all. + got, ok := byID(rows, "t1") + if !ok { + t.Fatalf("the team is missing from %+v", rows) + } + if got.CostMicros != 1000 { + t.Errorf("expected both members summed, got %d", got.CostMicros) + } + if got.Keys != 2 { + t.Errorf("expected two keys counted, got %d", got.Keys) + } + if got.Errors != 1 { + t.Errorf("expected the one failure counted, got %d", got.Errors) + } + if got.Name != "platform" { + t.Errorf("the group should be named, got %q", got.Name) + } +} + +func TestTeamsAreNotAddedTogether(t *testing.T) { + s := twoTeams(t) + rows, _ := s.UsageByScope(t.Context(), ScopeTeam, time.Now().Add(-time.Hour)) + + if len(rows) != 2 { + t.Fatalf("expected a row per team, got %+v", rows) + } + other, _ := byID(rows, "t2") + if other.CostMicros != 100 { + t.Errorf("the second team's spend leaked, got %d", other.CostMicros) + } +} + +func TestTheOrgRollsUpEveryTeamBeneathIt(t *testing.T) { + s := twoTeams(t) + rows, err := s.UsageByScope(t.Context(), ScopeOrg, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 || rows[0].CostMicros != 1100 { + t.Errorf("expected one org totalling 1100, got %+v", rows) + } +} + +func TestTheQuotaComesBackBesideTheSpend(t *testing.T) { + s := twoTeams(t) + budget := int64(2000) + if err := s.PutLimits(t.Context(), Limits{ + Scope: ScopeTeam, ScopeID: "t1", Budget: &budget, BudgetPeriod: PeriodMonth, + }); err != nil { + t.Fatal(err) + } + + rows, _ := s.UsageByScope(t.Context(), ScopeTeam, time.Now().Add(-time.Hour)) + got, _ := byID(rows, "t1") + // Spend without the quota beside it is a number nobody can act on. Half of + // a budget is the answer; 1000 micro-units on its own is trivia. + if got.Budget == nil || *got.Budget != 2000 { + t.Fatalf("the budget set on the team should come back, got %+v", got.Budget) + } + frac, ok := got.Spent() + if !ok || frac != 0.5 { + t.Errorf("expected half the budget used, got %v %v", frac, ok) + } +} + +func TestAGroupWithNoBudgetOfItsOwnShowsNone(t *testing.T) { + s := twoTeams(t) + budget := int64(9000) + if err := s.PutLimits(t.Context(), Limits{ + Scope: ScopeOrg, ScopeID: "o", Budget: &budget, BudgetPeriod: PeriodMonth, + }); err != nil { + t.Fatal(err) + } + + rows, _ := s.UsageByScope(t.Context(), ScopeTeam, time.Now().Add(-time.Hour)) + got, _ := byID(rows, "t1") + // The org's budget bounds this team but is not the team's to spend. + // Printing 9000 beside each of two teams would read as 18000 available, + // which is the exact misreading this work exists to remove. + if got.Budget != nil { + t.Errorf("a team with no budget of its own should show none, got %d", *got.Budget) + } + if _, ok := got.Spent(); ok { + t.Error("there is no fraction to report without a budget at this level") + } +} + +func TestGroupingBySomethingThatIsNotALevelIsRefused(t *testing.T) { + s := twoTeams(t) + // Never interpolated: the grouping picks from a fixed table of levels, so + // an unknown one is an error rather than a fragment of SQL. + if _, err := s.UsageByScope(t.Context(), Scope("key; DROP TABLE usage"), time.Now()); err == nil { + t.Error("expected an unknown grouping to be refused") + } +}