diff --git a/BACKLOG.md b/BACKLOG.md index d4b77c1..efeb600 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -397,7 +397,8 @@ portal is not done — the page shows; the console and ctl change. `keys rotate` with overlap, last-used shown, project-owned service keys listed apart. `internal/cli/keys.go`, `internal/store/keys.go`, `internal/auth/auth.go`. -4. **Usage export (S).** CSV/Parquet by org/team/project/model/period from +4. **Done (branch `features-c`):** CSV export from the rollups (API, ctl, + console). Left: Parquet, and a scheduled push to S3/SFTP. **Usage export (S).** CSV/Parquet by org/team/project/model/period from the rollups (`internal/store/usageseries.go`), `GET /api/v1/usage/export`, optional scheduled push to S3/SFTP. 5. **Alerting out of the box (S).** `deploy/monitoring/alertmanager.yml` diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e85809..317a5f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,15 @@ APIs change between minors without a deprecation period. - **Listing:** `keys list --all` shows each key's kind — service (a project's), member, personal — with service keys first, and when it expires. +- **Usage export.** `llmgw ctl usage export --from 2026-09-01 --to + 2026-10-01 --by project,model` writes CSV by hour, day or month and any of + organization, team, project, user, key and model. + - **Data:** read from the hourly and daily sums, so it reaches back past + the request records' retention. + - **Format:** cost is given in micro-units and units, and names that + look like a spreadsheet formula are quoted. + - **Other ways in:** `GET /api/v1/usage/export` (CSV, or `format=json`) + and **E** on the console's Usage view. ### Guardrails diff --git a/docs/console.md b/docs/console.md index ef24734..2bcbb92 100644 --- a/docs/console.md +++ b/docs/console.md @@ -169,6 +169,10 @@ its limits, or revoke it. The STAFF column shows each person's role. +On **Usage**, **E** exports usage as CSV to a file on the control plane: +by hour, day or month and any of organization, team, project, user, key and +model. + **Audit** lists every change of the last week — through `ctl`, this console or the /admin pages, and people's own keys and passwords — with who made it, through what, and how it ended. Enter shows the fields it was given, diff --git a/internal/cli/tui_audit.go b/internal/cli/tui_audit.go index e505d3a..950890a 100644 --- a/internal/cli/tui_audit.go +++ b/internal/cli/tui_audit.go @@ -6,6 +6,8 @@ import ( "strings" "time" + tea "github.com/charmbracelet/bubbletea" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -45,3 +47,25 @@ func auditDialog(e store.AuditEvent) (string, string) { b.WriteString("\n" + sDim.Render("any key")) return e.Action, b.String() } + +// usageExportForm writes usage to a CSV file on this machine, as `llmgw ctl +// usage export` does. +func (m *consoleModel) usageExportForm() { + now := time.Now().UTC() + first := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + m.openForm(&formDialog{title: "Export usage as CSV", + fields: []formField{ + textF("from", first.AddDate(0, -1, 0).Format("2006-01-02"), "first day"), + textF("to", first.Format("2006-01-02"), "the day after the last"), + textF("period", "day", "hour, day or month"), + textF("by", "project,model", "org, team, project, user, key, model"), + textF("file", "/tmp/llmgw-usage-"+first.AddDate(0, -1, 0).Format("2006-01")+".csv", "written on this machine"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + if v[4] == "" { + return nil, fmt.Errorf("a file to write") + } + return m.runCLI("Export usage", "ctl", "usage", "export", "--from", v[0], "--to", v[1], + "--period", v[2], "--by", v[3], "--output", v[4]), nil + }}) +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 2c1097c..86b3b32 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -536,6 +536,11 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cOrgs: m.createOrgForm() } + case "e": + if m.menu == cUsage { + m.usageExportForm() + return m, nil + } case "right", "l": if m.menu == cUsage { m.usageBy = (m.usageBy + 1) % len(usageGroupings) @@ -1092,7 +1097,7 @@ func (m consoleModel) panel() (title, body, keys string) { compactNum(u.Errors), fmt.Sprintf("%.2f", float64(u.CostMicros)/1e6)}) } if m.inPanel { - keys = "↑↓ row · → grouping · ← back · Q leave" + keys = "↑↓ row · → grouping · E export as CSV · ← back · Q leave" } return "Usage", head + table([]string{"WHO", "REQUESTS", "TOKENS", "ERRORS", "SPENT"}, rows, m.row, pw, m.inPanel), keys case cQuotas: diff --git a/internal/cli/usage.go b/internal/cli/usage.go index 41311b9..e5adfdc 100644 --- a/internal/cli/usage.go +++ b/internal/cli/usage.go @@ -3,12 +3,15 @@ package cli import ( "encoding/json" "fmt" + "net/url" + "os" "text/tabwriter" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // newUsageCmd shows what each group of the tenancy tree spent. @@ -17,6 +20,54 @@ import ( // anything could show what any of them had used: usage came back per key, per // user and per model only. A quota whose consumption cannot be seen is one // nobody reviews until the day it refuses somebody. +func newUsageExportCmd(client func() *ctlClient) *cobra.Command { + var from, to, period, by, output string + c := &cobra.Command{ + Use: "export", + Short: "Usage as CSV, by period and by organization, team, project, user, key or model", + Long: "Totals from the hourly and daily sums, so it reaches back past the request\n" + + "records' retention. Periods are UTC. Cost is given in micro-units and in\n" + + "units; names, not ids, except for keys.", + Example: " llmgw ctl usage export --from 2026-09-01 --to 2026-10-01 --by project,model > september.csv\n" + + " llmgw ctl usage export --period month --by org --output 2026.csv", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + q := url.Values{"period": {period}, "by": {by}, "format": {"json"}} + if from != "" { + q.Set("from", from) + } + if to != "" { + q.Set("to", to) + } + var rows []store.ExportRow + if err := client().get(cmd.Context(), "/api/v1/usage/export?"+q.Encode(), &rows); err != nil { + return err + } + out := cmd.OutOrStdout() + if output != "" { + f, err := os.Create(output) + if err != nil { + return err + } + defer f.Close() + out = f + } + adminapi.WriteExportCSV(out, rows) + if output != "" { + cmd.PrintErrf("%d rows written to %s\n", len(rows), output) + } + return nil + }, + } + f := c.Flags() + f.StringVar(&from, "from", "", "first day, 2026-09-01 (default: the first of this month)") + f.StringVar(&to, "to", "", "the day after the last, 2026-10-01 (default: now)") + f.StringVar(&period, "period", "day", "hour, day or month") + f.StringVar(&by, "by", "project,model", "any of org, team, project, user, key, model, comma-separated") + f.StringVarP(&output, "output", "o", "", "write to this file instead of standard output") + return c +} + func newUsageCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var by string var since time.Duration @@ -70,5 +121,6 @@ func newUsageCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } c.Flags().StringVar(&by, "by", "team", "level to group by: org, team or project") c.Flags().DurationVar(&since, "since", 24*time.Hour, "how far back to total") + c.AddCommand(newUsageExportCmd(client)) return c } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index f705223..34e0802 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -83,6 +83,7 @@ func (a *API) Register(mux *http.ServeMux) { a.registerQuotas(mux) a.registerGuard(mux) mux.HandleFunc("GET /api/v1/audit", a.auditEvents) + mux.HandleFunc("GET /api/v1/usage/export", a.usageExport) mux.HandleFunc("POST /api/v1/models/{name}/adapters", a.setAdapters) mux.HandleFunc("GET /api/v1/models/{name}/target", a.target) mux.HandleFunc("POST /api/v1/models/{name}/class", func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/control/adminapi/export.go b/internal/control/adminapi/export.go new file mode 100644 index 0000000..8098838 --- /dev/null +++ b/internal/control/adminapi/export.go @@ -0,0 +1,96 @@ +package adminapi + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Exporter totals usage for an export. +type Exporter interface { + UsageExport(ctx context.Context, from, to time.Time, period string, by []string) ([]store.ExportRow, error) +} + +// usageExport answers GET /api/v1/usage/export?from=2026-09-01&to=2026-10-01 +// &period=day&by=project,model as CSV, or JSON with format=json. +func (a *API) usageExport(w http.ResponseWriter, r *http.Request) { + ex, ok := a.usage.(Exporter) + if !ok { + writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records") + return + } + q := r.URL.Query() + day := func(s string, def time.Time) (time.Time, error) { + if s == "" { + return def, nil + } + if t, err := time.Parse("2006-01-02", s); err == nil { + return t, nil + } + return time.Parse(time.RFC3339, s) + } + now := time.Now().UTC() + from, err := day(q.Get("from"), time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)) + if err != nil { + writeError(w, http.StatusBadRequest, "from: a date, 2026-09-01") + return + } + to, err := day(q.Get("to"), now) + if err != nil { + writeError(w, http.StatusBadRequest, "to: a date, 2026-10-01") + return + } + period := q.Get("period") + if period == "" { + period = "day" + } + var by []string + if b := q.Get("by"); b != "" { + by = strings.Split(b, ",") + } + rows, err := ex.UsageExport(r.Context(), from, to, period, by) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + if q.Get("format") == "json" { + writeJSON(w, http.StatusOK, rows) + return + } + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="llmgw-usage-%s-%s.csv"`, + from.Format("20060102"), to.Format("20060102"))) + WriteExportCSV(w, rows) +} + +// WriteExportCSV writes rows with a header; cost in units as well as micros, +// so a spreadsheet needs no arithmetic. +func WriteExportCSV(w io.Writer, rows []store.ExportRow) { + c := csv.NewWriter(w) + _ = c.Write([]string{"period_start", "org", "team", "project", "user", "key", "model", + "requests", "errors", "limited", "prompt_tokens", "completion_tokens", "cost_micros", "cost"}) + for _, r := range rows { + _ = c.Write([]string{r.Period.Format(time.RFC3339), safe(r.Org), safe(r.Team), safe(r.Project), safe(r.User), + r.Key, safe(r.Model), i(r.Requests), i(r.Errors), i(r.Limited), i(r.Prompt), i(r.Completion), + i(r.CostMicros), strconv.FormatFloat(float64(r.CostMicros)/1e6, 'f', 6, 64)}) + } + c.Flush() +} + +func i(n int64) string { return strconv.FormatInt(n, 10) } + +// safe keeps a name from being read as a formula by a spreadsheet: a cell +// starting with = + - @ is one in Excel and LibreOffice. +func safe(s string) string { + if s != "" && strings.ContainsRune("=+-@\t\r", rune(s[0])) { + return "'" + s + } + return s +} diff --git a/internal/control/adminapi/guard_test.go b/internal/control/adminapi/guard_test.go index d441041..74f8b44 100644 --- a/internal/control/adminapi/guard_test.go +++ b/internal/control/adminapi/guard_test.go @@ -2,7 +2,10 @@ package adminapi import ( "context" + "net/http" + "net/http/httptest" "slices" + "strings" "testing" "time" @@ -82,3 +85,34 @@ func TestTheAuditLogIsReadWithFilters(t *testing.T) { t.Errorf("a bad since: %d", code) } } + +type memUsageExport struct { + memAudit + by []string + period string +} + +func (m *memUsageExport) UsageByScope(context.Context, store.Scope, time.Time) ([]store.ScopeUsage, error) { + return nil, nil +} + +func (m *memUsageExport) UsageExport(_ context.Context, from, to time.Time, period string, by []string) ([]store.ExportRow, error) { + m.by, m.period = by, period + return []store.ExportRow{{Period: from, Project: "=HYPERLINK(evil)", Model: "m", Requests: 3, CostMicros: 1500000}}, nil +} + +func TestUsageIsExportedAsCSVThatASpreadsheetCannotRun(t *testing.T) { + a, _ := setup(t, testModels()) + m := &memUsageExport{} + a.usage = m + mux := http.NewServeMux() + a.Register(mux) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, httptest.NewRequest("GET", "/api/v1/usage/export?from=2026-09-01&to=2026-10-01&period=month&by=project,model", nil)) + body := rec.Body.String() + if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/csv") || + !strings.Contains(body, "period_start,org,team,project") || !strings.Contains(body, "'=HYPERLINK") || + !strings.Contains(body, "1.500000") || m.period != "month" || !slices.Equal(m.by, []string{"project", "model"}) { + t.Errorf("%d %s %+v", rec.Code, body, m) + } +} diff --git a/internal/store/export.go b/internal/store/export.go new file mode 100644 index 0000000..06dc482 --- /dev/null +++ b/internal/store/export.go @@ -0,0 +1,152 @@ +package store + +import ( + "context" + "database/sql" + "fmt" + "slices" + "strings" + "time" +) + +// ExportRow is one line of a usage export: a period, the place, person, key +// and model it is about — those asked for, the rest empty — and what was used. +type ExportRow struct { + Period time.Time + Org string + Team string + Project string + User string + Key string + Model string + Requests int64 + Errors int64 + Limited int64 + Prompt int64 + Completion int64 + CostMicros int64 +} + +// ExportDims are what an export can be broken down by. +var ExportDims = []string{"org", "team", "project", "user", "key", "model"} + +// UsageExport totals usage between two moments, by period — hour, day or +// month, in UTC — and by the dimensions asked for. It reads the hourly and +// daily sums, so request records past their retention are still counted. +func (s *DB) UsageExport(ctx context.Context, from, to time.Time, period string, by []string) ([]ExportRow, error) { + for _, d := range by { + if !slices.Contains(ExportDims, d) { + return nil, fmt.Errorf("%q is not one of %s", d, strings.Join(ExportDims, ", ")) + } + } + table := "usage_1d" + switch period { + case "hour": + table = "usage_1h" + case "day", "month": + default: + return nil, fmt.Errorf("period %q is not hour, day or month", period) + } + type raw struct { + bucket int64 + key, user, project, model string + requests, errors, limited, prompt, completion, cost int64 + } + //nolint:gosec // table is one of two literals above + rows, err := queryAll(ctx, s.db, `SELECT bucket, key_id, user_id, project_id, served_model, + SUM(requests), SUM(errors), SUM(limited), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros) + FROM `+table+` WHERE bucket >= ? AND bucket < ? + GROUP BY bucket, key_id, user_id, project_id, served_model`, + []any{millis(from), millis(to)}, func(r *sql.Rows) (raw, error) { + var x raw + err := r.Scan(&x.bucket, &x.key, &x.user, &x.project, &x.model, + &x.requests, &x.errors, &x.limited, &x.prompt, &x.completion, &x.cost) + return x, err + }) + if err != nil { + return nil, err + } + + // Names, not ids: an export is read by people and spreadsheets. + type place struct{ org, team, project string } + places := map[string]place{"": {project: "personal"}} + tree, err := s.Tree(ctx) + if err != nil { + return nil, err + } + for _, o := range tree { + for _, t := range o.Teams { + for _, p := range t.Projects { + places[p.ID] = place{o.Name, t.Name, p.Name} + } + } + } + emails := map[string]string{} + users, err := s.Users(ctx, "") + if err != nil { + return nil, err + } + for _, u := range users { + emails[u.ID] = u.Email + } + has := func(d string) bool { return slices.Contains(by, d) } + + totals := map[ExportRow]*ExportRow{} + var order []ExportRow + for _, x := range rows { + at := fromMillis(x.bucket).UTC() + if period == "month" { + at = time.Date(at.Year(), at.Month(), 1, 0, 0, 0, 0, time.UTC) + } + pl := places[x.project] + k := ExportRow{Period: at} + if has("org") { + k.Org = pl.org + } + if has("team") { + k.Team = pl.team + } + if has("project") { + k.Project = pl.project + if pl.project == "" { + k.Project = x.project + } + } + if has("user") { + k.User = emails[x.user] + if k.User == "" { + k.User = x.user + } + } + if has("key") { + k.Key = x.key + } + if has("model") { + k.Model = x.model + } + t, ok := totals[k] + if !ok { + row := k + t = &row + totals[k] = t + order = append(order, k) + } + t.Requests += x.requests + t.Errors += x.errors + t.Limited += x.limited + t.Prompt += x.prompt + t.Completion += x.completion + t.CostMicros += x.cost + } + out := make([]ExportRow, 0, len(order)) + for _, k := range order { + out = append(out, *totals[k]) + } + slices.SortFunc(out, func(a, b ExportRow) int { + if c := a.Period.Compare(b.Period); c != 0 { + return c + } + return strings.Compare(a.Org+a.Team+a.Project+a.User+a.Key+a.Model, b.Org+b.Team+b.Project+b.User+b.Key+b.Model) + }) + return out, nil +} diff --git a/internal/store/export_test.go b/internal/store/export_test.go new file mode 100644 index 0000000..6b19bdf --- /dev/null +++ b/internal/store/export_test.go @@ -0,0 +1,48 @@ +package store + +import ( + "testing" + "time" +) + +func TestUsageIsExportedByPeriodAndWhatWasAskedFor(t *testing.T) { + db := capacityStore(t) + ctx := t.Context() + org, _ := db.CreateOrg(ctx, "acme") + team, _ := db.CreateTeam(ctx, org, "ml") + project, _ := db.CreateProject(ctx, team, "train") + day1 := time.Date(2026, 9, 30, 10, 0, 0, 0, time.UTC) + day2 := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC) + var rows []Usage + for _, r := range []struct { + at time.Time + model string + p int64 + }{{day1, "a", 10}, {day1, "b", 20}, {day2, "a", 5}, {day2.Add(time.Hour), "a", 5}} { + rows = append(rows, Usage{At: r.at, KeyID: "k", ProjectID: project, Model: r.model, ServedModel: r.model, + PromptTokens: r.p, CompletionTokens: 1, CostMicros: 100, Status: 200}) + } + if err := db.WriteUsage(ctx, rows); err != nil { + t.Fatal(err) + } + from, to := day1.Add(-24*time.Hour), day2.Add(48*time.Hour) + + byDay, err := db.UsageExport(ctx, from, to, "day", []string{"project", "model"}) + if err != nil || len(byDay) != 3 { + t.Fatalf("%+v %v", byDay, err) + } + last := byDay[2] + if last.Project != "train" || last.Model != "a" || last.Prompt != 10 || last.Requests != 2 || !last.Period.Equal(time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)) { + t.Errorf("%+v", last) + } + byMonth, _ := db.UsageExport(ctx, from, to, "month", []string{"org"}) + if len(byMonth) != 2 || byMonth[0].Org != "acme" || byMonth[0].Prompt != 30 || byMonth[1].CostMicros != 200 { + t.Errorf("by month: %+v", byMonth) + } + if _, err := db.UsageExport(ctx, from, to, "week", nil); err == nil { + t.Error("a week was taken") + } + if _, err := db.UsageExport(ctx, from, to, "day", []string{"planet"}); err == nil { + t.Error("an unknown dimension was taken") + } +}