From 4454e1322c75ce597e0fcad619a785773b686a6f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:43:01 +0200 Subject: [PATCH] Read usage from sums kept as it is written, not from every request Charts, totals and budgets summed the request table on every view. Usage is now also summed in the writing transaction into five-minute, hourly and daily tables per key, person, project and model, with latency and first-token histograms. Charts read the coarsest tier their slices allow; totals and budgets compose rows, slices, hours and days, and stay exact. Request records can be given a retention without moving any total. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 22 ++ deploy/examples/server.yaml | 5 + docs/reference.md | 12 + internal/config/config.go | 11 + internal/config/config_test.go | 14 ++ internal/router/router.go | 35 ++- internal/store/copy.go | 5 + .../migrations/postgres/0015_usage_rollup.sql | 204 ++++++++++++++++ .../migrations/sqlite/0015_usage_rollup.sql | 203 ++++++++++++++++ internal/store/postgres_test.go | 2 +- internal/store/rollup.go | 206 ++++++++++++++++ internal/store/rollup_test.go | 219 ++++++++++++++++++ internal/store/selfusage.go | 26 ++- internal/store/sqlite.go | 68 +++--- internal/store/usagescope.go | 14 +- internal/store/usageseries.go | 16 +- 16 files changed, 1013 insertions(+), 49 deletions(-) create mode 100644 internal/store/migrations/postgres/0015_usage_rollup.sql create mode 100644 internal/store/migrations/sqlite/0015_usage_rollup.sql create mode 100644 internal/store/rollup.go create mode 100644 internal/store/rollup_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 609bbcdb..f5d2de11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,28 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### Usage is read from sums kept as it is written + +Every chart and total — the usage tab, quotas, the dashboard, a person's own +page, and budgets restored at start — summed the request table again on each +view: a row per request, scanned whole. Usage is now also summed as it is +written, in the same transaction, into five-minute slices, hours and days per +key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration +0015 fills them from what is already recorded). + +- **Charts** read the coarsest of the three their slices are made of: a day in + 20-minute slices from the five-minute table, a week or a month from hours. +- **Totals and budgets** read the rows up to the next five-minute slice, then + slices, hours and days — exact, not rounded to a slice. On three million + requests over thirty days, a month's total per person went from 705 ms to + 18 ms on PostgreSQL. +- **Latency and time to first token** are kept as histograms in the sums, so a + percentile per team or model can be read after the rows are gone. +- **`auth.usage_retention`** (e.g. `2160h`) deletes request records older than + that, hourly. Totals, charts and budgets are unchanged by it — they read the + sums — but `ctl request` finds nothing older. Unset keeps every record, as + before. + ### People, limits and usage are one page Managing a team meant its members on the people page, its quota on the quotas diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 52ddcfb5..a24e4e7f 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -92,6 +92,11 @@ agent_listen: ":7443" # flush_interval: 200ms # batch_size: 500 # buffer_size: 8192 +# +# # How long the record of each request is kept (what `ctl request` reads). +# # Charts, totals and budgets come from five-minute sums that are kept, so +# # this only bounds the table that grows by a row a request. 0 keeps all. +# usage_retention: 2160h # 90 days router: scrape_interval: 2s diff --git a/docs/reference.md b/docs/reference.md index 45e780ff..aa1e3533 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -39,6 +39,7 @@ rather than repeated. | `flush_interval` | `time.Duration` | [Auth](#auth) | | `batch_size` | `int` | [Auth](#auth) | | `buffer_size` | `int` | [Auth](#auth) | +| `usage_retention` | `time.Duration` | [Auth](#auth) | | `name` | `string` | [Band](#band) | | `priority` | `int` | [Band](#band) | | `saturation_ceiling` | `float64` | [Band](#band) | @@ -367,6 +368,17 @@ BufferSize bounds the queue. Beyond it records are dropped rather than blocking a request: losing accounting is bad, turning a stalled database into an outage is worse. +### `usage_retention` + +`time.Duration` + +UsageRetention is how long the record of each request is kept: who sent +it, which replica answered, how long it queued — what `ctl request` +reads. Zero keeps it for ever. Totals, charts and budgets are read from +five-minute sums kept beside the records, which are not pruned, so this +bounds the table that grows by a row a request without changing any +figure a page shows. At least a day when set. + ## Band ### `name` diff --git a/internal/config/config.go b/internal/config/config.go index 24fefc24..5cd0a9ec 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -257,6 +257,14 @@ type Auth struct { // blocking a request: losing accounting is bad, turning a stalled database // into an outage is worse. BufferSize int `koanf:"buffer_size"` + + // UsageRetention is how long the record of each request is kept: who sent + // it, which replica answered, how long it queued — what `ctl request` + // reads. Zero keeps it for ever. Totals, charts and budgets are read from + // five-minute sums kept beside the records, which are not pruned, so this + // bounds the table that grows by a row a request without changing any + // figure a page shows. At least a day when set. + UsageRetention time.Duration `koanf:"usage_retention"` } // Personal configures use that belongs to a person rather than a project. @@ -573,6 +581,9 @@ func (s Server) Validate() error { if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") } + if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour { + return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention) + } if s.Threads.Store != "" && s.Threads.RetainHours <= 0 { return fmt.Errorf("threads.retain_hours must be positive when threads.store is set") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 47c1bb8b..91e022b5 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -206,3 +206,17 @@ func TestEnginesDefaultToTheCheckedVersions(t *testing.T) { t.Error("an emptied engine is still wanted") } } + +func TestAUsageRetentionUnderADayIsRefused(t *testing.T) { + // Shorter than a day is almost certainly a unit slip — 90m for 90 days — + // and would delete every request record within the hour. + s := Default() + s.Auth.UsageRetention = 90 * time.Minute + if err := s.Validate(); err == nil || !strings.Contains(err.Error(), "usage_retention") { + t.Errorf("a 90-minute retention was accepted: %v", err) + } + s.Auth.UsageRetention = 90 * 24 * time.Hour + if err := s.Validate(); err != nil { + t.Errorf("a 90-day retention was refused: %v", err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index bfa05038..3b62a572 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -83,7 +83,9 @@ type Router struct { auth *auth.Authenticator accounts *account.Service store *storepkg.DB - acct *accounting.Accountant + // usageRetention is how long request records are kept; zero is for ever. + usageRetention time.Duration + acct *accounting.Accountant // threads holds conversations and memory notes. It is store when auth is on, // its own file when only the threads block is configured, and nil when @@ -248,6 +250,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { return nil, fmt.Errorf("auth store: %w", err) } r.store = st + r.usageRetention = cfg.Auth.UsageRetention // Applied where no scope set one, so a project created without limits // is still metered. Zero leaves it unlimited, which was the old // behaviour and is now a choice rather than an omission. @@ -412,6 +415,36 @@ func (r *Router) Start(ctx context.Context) { if r.threads != nil { go r.pruneThreads(ctx) } + if r.store != nil && r.usageRetention > 0 { + go r.pruneUsage(ctx, r.usageRetention) + } +} + +// pruneUsage deletes request records older than the retention, hourly and +// once at startup, as pruneThreads does. The five-minute sums stay, so no +// total or chart moves; what goes is the trail of each old request. +func (r *Router) pruneUsage(ctx context.Context, keep time.Duration) { + prune := func() { + n, err := r.store.PruneUsage(ctx, time.Now().Add(-keep)) + if err != nil { + r.log.Warn("could not prune old request records", "err", err) + return + } + if n > 0 { + r.log.Info("removed request records past their retention", "rows", n, "retention", keep) + } + } + prune() + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + prune() + } + } } // pruneThreads enforces each project's retention window. diff --git a/internal/store/copy.go b/internal/store/copy.go index 1c72157b..98891392 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -33,6 +33,11 @@ var copyOrder = []string{ // quota on a scope that is later removed is inert, not an error. "capacity_quotas", "usage", + // Copied rather than rebuilt from the rows: once request rows have a + // retention, the sums are the only record of what is older. + "usage_5m", + "usage_1h", + "usage_1d", "models", // No foreign key to models, deliberately: a measurement outlives the // catalog entry it was taken for, and losing it when somebody removes a diff --git a/internal/store/migrations/postgres/0015_usage_rollup.sql b/internal/store/migrations/postgres/0015_usage_rollup.sql new file mode 100644 index 00000000..39abcf7e --- /dev/null +++ b/internal/store/migrations/postgres/0015_usage_rollup.sql @@ -0,0 +1,204 @@ +-- Ported from the SQLite migration of the same number. +-- Usage summed as it is written, per key, person, project and model: into +-- five-minute slices (usage_5m), hours (usage_1h) and days (usage_1d). +-- +-- Every chart and total read the raw table, one row per request, and summed it +-- again on every page view: fine for thousands of requests a day, and a full +-- scan of tens of millions on a cluster in real use. The sums are kept in the +-- transaction that writes the rows, so they never disagree with them; the raw +-- table stays for following one request, and can be given a retention of its +-- own (auth.usage_retention). +-- +-- Three widths because how well a slice compresses depends on traffic: many +-- keys each sending a little leave a five-minute slice with nearly a row per +-- request, which an hour or a day folds much further. A chart reads the +-- coarsest width its slices are made of; a total since a moment reads the +-- rows up to the next five-minute slice, those slices up to the hour, hours +-- up to the day and days from there — exact, and a month in about thirty +-- rows per key and model. +-- +-- Latency and time to first token are kept as histograms, a count per fixed +-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, +-- and above), which is what a percentile per team or model is read from after +-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose +-- first token was measured. + +CREATE TABLE usage_5m ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_5m_by_bucket ON usage_5m(bucket); +CREATE INDEX usage_5m_by_project ON usage_5m(project_id, bucket); +CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); +CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); + +INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1h ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1h_by_bucket ON usage_1h(bucket); +CREATE INDEX usage_1h_by_project ON usage_1h(project_id, bucket); +CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); +CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); + +INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1d ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1d_by_bucket ON usage_1d(bucket); +CREATE INDEX usage_1d_by_project ON usage_1d(project_id, bucket); +CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); +CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); + +INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; diff --git a/internal/store/migrations/sqlite/0015_usage_rollup.sql b/internal/store/migrations/sqlite/0015_usage_rollup.sql new file mode 100644 index 00000000..17d9ae95 --- /dev/null +++ b/internal/store/migrations/sqlite/0015_usage_rollup.sql @@ -0,0 +1,203 @@ +-- Usage summed as it is written, per key, person, project and model: into +-- five-minute slices (usage_5m), hours (usage_1h) and days (usage_1d). +-- +-- Every chart and total read the raw table, one row per request, and summed it +-- again on every page view: fine for thousands of requests a day, and a full +-- scan of tens of millions on a cluster in real use. The sums are kept in the +-- transaction that writes the rows, so they never disagree with them; the raw +-- table stays for following one request, and can be given a retention of its +-- own (auth.usage_retention). +-- +-- Three widths because how well a slice compresses depends on traffic: many +-- keys each sending a little leave a five-minute slice with nearly a row per +-- request, which an hour or a day folds much further. A chart reads the +-- coarsest width its slices are made of; a total since a moment reads the +-- rows up to the next five-minute slice, those slices up to the hour, hours +-- up to the day and days from there — exact, and a month in about thirty +-- rows per key and model. +-- +-- Latency and time to first token are kept as histograms, a count per fixed +-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, +-- and above), which is what a percentile per team or model is read from after +-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose +-- first token was measured. + +CREATE TABLE usage_5m ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_5m_by_bucket ON usage_5m(bucket); +CREATE INDEX usage_5m_by_project ON usage_5m(project_id, bucket); +CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); +CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); + +INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1h ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1h_by_bucket ON usage_1h(bucket); +CREATE INDEX usage_1h_by_project ON usage_1h(project_id, bucket); +CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); +CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); + +INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1d ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1d_by_bucket ON usage_1d(bucket); +CREATE INDEX usage_1d_by_project ON usage_1d(project_id, bucket); +CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); +CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); + +INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index 32465de3..746f13f5 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -41,7 +41,7 @@ func openPG(t *testing.T) *DB { // inherits the previous one's lease is refused for reasons that have // nothing to do with what it is testing. It passed on SQLite, where // every subtest gets a fresh file, and failed only here. - `TRUNCATE thread_messages, threads, memory_notes, usage, api_keys, memberships, + `TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships, key_claims, limits, models, sessions, users, projects, teams, organizations, leadership RESTART IDENTITY CASCADE`) if err != nil { diff --git a/internal/store/rollup.go b/internal/store/rollup.go new file mode 100644 index 00000000..4333ee2e --- /dev/null +++ b/internal/store/rollup.go @@ -0,0 +1,206 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// tier is one table usage is summed into as it is written: slices of a fixed +// width, per key, person, project and model. +type tier struct { + table string + ms int64 +} + +var ( + tier5m = tier{"usage_5m", int64(5 * time.Minute / time.Millisecond)} + tier1h = tier{"usage_1h", int64(time.Hour / time.Millisecond)} + tier1d = tier{"usage_1d", int64(24 * time.Hour / time.Millisecond)} + // tiers, finest first. + tiers = []tier{tier5m, tier1h, tier1d} +) + +// floor is the start of the slice a moment falls in. A chart read from a +// tier starts its first slice there, as a chart read from the rows always did. +func (t tier) floor(at time.Time) int64 { return (millis(at) / t.ms) * t.ms } + +// ceil is the start of the first slice wholly after a moment. +func (t tier) ceil(at time.Time) int64 { + return ((millis(at) + t.ms - 1) / t.ms) * t.ms +} + +// tierForSlices is the coarsest tier whose slices make up a chart slice of +// width exactly, and false when none does — a slice finer than five minutes, +// or not a whole number of them, is read from the rows. +func tierForSlices(width time.Duration) (tier, bool) { + ms := width.Milliseconds() + for i := len(tiers) - 1; i >= 0; i-- { + if ms >= tiers[i].ms && ms%tiers[i].ms == 0 { + return tiers[i], true + } + } + return tier{}, false +} + +// LatencyBounds are the upper edges, in milliseconds, of the histogram buckets +// latency and time to first token are counted in; one more bucket holds +// everything above the last. Fixed, because buckets are only summable across +// slices, keys and teams when every row has the same ones. +var LatencyBounds = []int64{100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000} + +// latencyBucket is the histogram bucket a duration is counted in. +func latencyBucket(ms int64) int { + for i, b := range LatencyBounds { + if ms <= b { + return i + } + } + return len(LatencyBounds) +} + +// rollupKey is one row of a tier: a slice, and who spent what on what. +type rollupKey struct { + bucket int64 + keyID, userID, projectID, model, servedModel string +} + +// rollupRow is what a batch adds to one row. +type rollupRow struct { + requests, errors, prompt, completion, cost int64 + latency, queue, ttft, ttftN, last int64 + lat, ttftH []int64 +} + +// rollupBatch sums a batch of records into the rows of one tier they add to. +func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey) { + rows := map[rollupKey]*rollupRow{} + var order []rollupKey + n := len(LatencyBounds) + 1 + for _, r := range records { + ts := millis(r.At) + k := rollupKey{(ts / t.ms) * t.ms, r.KeyID, r.UserID, r.ProjectID, r.Model, r.ServedModel} + row, ok := rows[k] + if !ok { + row = &rollupRow{lat: make([]int64, n), ttftH: make([]int64, n)} + rows[k] = row + order = append(order, k) + } + row.requests++ + if r.Status >= 400 { + row.errors++ + } + row.prompt += r.PromptTokens + row.completion += r.CompletionTokens + row.cost += r.CostMicros + row.latency += r.LatencyMs + row.queue += r.QueueMs + row.lat[latencyBucket(r.LatencyMs)]++ + if r.TTFTMs > 0 { + row.ttft += r.TTFTMs + row.ttftN++ + row.ttftH[latencyBucket(r.TTFTMs)]++ + } + row.last = max(row.last, ts) + } + return rows, order +} + +// rollupColumns are a tier's columns, in the order rollupUpsert takes them. +var rollupColumns = func() []string { + n := len(LatencyBounds) + 1 + cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model", + "requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros", + "latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"} + for i := range n { + cols = append(cols, fmt.Sprintf("lat_%d", i)) + } + for i := range n { + cols = append(cols, fmt.Sprintf("ttft_%d", i)) + } + return cols +}() + +// rollupUpsert is the statement that adds a batch's sums to a tier. +func rollupUpsert(table string) string { + var sets []string + for _, c := range rollupColumns[6:] { + if c == "last_ts" { + sets = append(sets, "last_ts = CASE WHEN excluded.last_ts > "+table+".last_ts "+ + "THEN excluded.last_ts ELSE "+table+".last_ts END") + continue + } + sets = append(sets, c+" = "+table+"."+c+" + excluded."+c) + } + return `INSERT INTO ` + table + ` (` + strings.Join(rollupColumns, ", ") + `) + VALUES (` + strings.TrimSuffix(strings.Repeat("?, ", len(rollupColumns)), ", ") + `) + ON CONFLICT (bucket, key_id, user_id, project_id, model, served_model) + DO UPDATE SET ` + strings.Join(sets, ", ") +} + +// writeRollup adds a batch to every tier, inside the transaction that writes +// its rows, so the sums never disagree with them. +func writeRollup(ctx context.Context, tx conn, records []Usage) error { + for _, t := range tiers { + rows, order := rollupBatch(t, records) + stmt, err := tx.PrepareContext(ctx, rollupUpsert(t.table)) + if err != nil { + return err + } + for _, k := range order { + r := rows[k] + args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel, + r.requests, r.errors, r.prompt, r.completion, r.cost, + r.latency, r.queue, r.ttft, r.ttftN, r.last} + for _, v := range r.lat { + args = append(args, v) + } + for _, v := range r.ttftH { + args = append(args, v) + } + if _, err := stmt.ExecContext(ctx, args...); err != nil { + _ = stmt.Close() + return fmt.Errorf("adding to %s: %w", t.table, err) + } + } + if err := stmt.Close(); err != nil { + return err + } + } + return nil +} + +// usageSince is every request since a moment, exactly, as a table with the +// columns a tier has (but the histograms): the rows up to the next +// five-minute slice, those slices up to the next hour, hours up to the next +// day, and days from there. Totals are read from this — a budget against it +// must be exact, and so is everything else, at the cost of a few rows read +// from each of four tables rather than a window rounded to a slice. +// +// Its placeholders come first in any query it is put into. +func usageSince(since time.Time) (string, []any) { + from := millis(since) + to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since) + const cols = `key_id, user_id, project_id, model, served_model, requests, errors, + prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts` + return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests, + CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors, + prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts + FROM usage WHERE ts >= ? AND ts < ? + UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ? + UNION ALL SELECT ` + cols + ` FROM usage_1h WHERE bucket >= ? AND bucket < ? + UNION ALL SELECT ` + cols + ` FROM usage_1d WHERE bucket >= ?)`, + []any{from, to5m, to5m, to1h, to1h, to1d, to1d} +} + +// PruneUsage deletes request rows older than a moment. The tiers keep their +// sums, so charts, totals and budgets are unchanged; what goes is the record +// of each request — `ctl request` on one that old finds nothing. +func (s *DB) PruneUsage(ctx context.Context, before time.Time) (int64, error) { + res, err := s.db.ExecContext(ctx, `DELETE FROM usage WHERE ts < ?`, millis(before)) + if err != nil { + return 0, err + } + return res.RowsAffected() +} diff --git a/internal/store/rollup_test.go b/internal/store/rollup_test.go new file mode 100644 index 00000000..d1984b16 --- /dev/null +++ b/internal/store/rollup_test.go @@ -0,0 +1,219 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" +) + +// onBoth runs a test on SQLite, and on PostgreSQL when LLMGW_TEST_POSTGRES +// names one: the rollup is written with an upsert, and the two dialects are +// where an upsert goes wrong. +func onBoth(t *testing.T, f func(t *testing.T, s *DB)) { + t.Run("sqlite", func(t *testing.T) { f(t, openDB(t)) }) + t.Run("postgres", func(t *testing.T) { f(t, openPG(t)) }) +} + +// slice is a moment a minute into an hour, a few hours back: inside a +// five-minute slice and an hour alike. +func slice() time.Time { + return time.UnixMilli(tier1h.floor(time.Now().Add(-3 * time.Hour))).Add(time.Minute) +} + +func TestTheRollupAddsUpToTheRows(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + // Two batches into the same slice, so the second has to add to the + // row the first made rather than replace it. + for _, batch := range [][]Usage{ + { + {At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 200, LatencyMs: 80, TTFTMs: 40}, + {At: at.Add(time.Second), KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 1, CompletionTokens: 1, CostMicros: 7, Status: 500, LatencyMs: 3000}, + }, + { + {At: at.Add(2 * time.Second), KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 2, CompletionTokens: 3, CostMicros: 50, Status: 200, LatencyMs: 400, TTFTMs: 200}, + }, + } { + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + } + + for _, tr := range tiers { + var req, errs, prompt, cost, lat, ttftN, last, lat0, lat2, lat5, ttft0, ttft1 int64 + err := s.db.QueryRowContext(ctx, `SELECT requests, errors, prompt_tokens, cost_micros, + latency_ms_sum, ttft_n, last_ts, lat_0, lat_2, lat_5, ttft_0, ttft_1 FROM `+tr.table). + Scan(&req, &errs, &prompt, &cost, &lat, &ttftN, &last, &lat0, &lat2, &lat5, &ttft0, &ttft1) + if err != nil { + t.Fatalf("%s: %v", tr.table, err) + } + if req != 3 || errs != 1 || prompt != 13 || cost != 157 || lat != 3480 { + t.Errorf("sums: %d requests, %d errors, %d prompt, %d cost, %d ms; want 3, 1, 13, 157, 3480", + req, errs, prompt, cost, lat) + } + // 80 ms in <=100, 400 in <=500, 3000 in <=5000; first tokens 40 and + // 200, and the failed request had none to count. + if lat0 != 1 || lat2 != 1 || lat5 != 1 || ttftN != 2 || ttft0 != 1 || ttft1 != 1 { + t.Errorf("histograms: lat %d/%d/%d, ttft n=%d %d/%d", lat0, lat2, lat5, ttftN, ttft0, ttft1) + } + if last != millis(at.Add(2*time.Second)) { + t.Errorf("last seen %d, want the latest request", last) + } + } + + b, err := s.UsageBuckets(ctx, at.Add(-time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + var total Bucket + for _, x := range b { + total.Requests += x.Requests + total.CostMicros += x.CostMicros + } + if total.Requests != 3 || total.CostMicros != 157 { + t.Errorf("the chart counts %d requests and %d spent, want 3 and 157", total.Requests, total.CostMicros) + } + }) +} + +func TestABudgetIsCountedFromTheMomentNotTheSlice(t *testing.T) { + // Charts start at the slice a window falls in; a budget must not. A + // request a minute before the period began is not the period's. + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + // Before the moment; after it in its own slice; in a later slice of + // the same hour; and in the next hour — one from each part a budget + // is read from. + if err := s.WriteUsage(ctx, []Usage{ + {At: at, KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 100, Status: 200}, + {At: at.Add(2 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 10, Status: 200}, + {At: at.Add(10 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 1, Status: 200}, + {At: at.Add(70 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 1000, Status: 200}, + }); err != nil { + t.Fatal(err) + } + spend, err := s.SpendSince(ctx, at.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if spend["k"] != 1011 { + t.Errorf("spent %d since mid-slice, want 1011", spend["k"]) + } + }) +} + +func TestPruningRequestsKeepsEveryTotal(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + if err := s.WriteUsage(ctx, []Usage{ + {At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + CostMicros: 100, Status: 200, RequestID: "req-1"}, + }); err != nil { + t.Fatal(err) + } + n, err := s.PruneUsage(ctx, time.Now()) + if err != nil || n != 1 { + t.Fatalf("pruned %d, %v", n, err) + } + if _, ok, _ := s.UsageByRequest(ctx, "req-1"); ok { + t.Error("a pruned request can still be looked up") + } + spend, _ := s.SpendSince(ctx, time.UnixMilli(tier5m.floor(at))) + users, _ := s.UsageByUser(ctx, at.Add(-time.Hour)) + if spend["k"] != 100 || len(users) != 1 || users[0].Requests != 1 { + t.Errorf("after pruning: spend %v, people %+v; the totals should not move", spend, users) + } + }) +} + +func TestTheMigrationRollsUpWhatIsAlreadyRecorded(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "old.db")+"?_pragma=foreign_keys(1)") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.ExecContext(ctx, + `CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + names, _ := migrationNames("sqlite") + for _, name := range names { + if name >= "0015" { + break + } + body, _ := migrationFS.ReadFile("migrations/sqlite/" + name) + if err := applyOne(ctx, db, "sqlite", name, string(body)); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + for _, ts := range []int64{600_001, 600_002, 900_001} { + if _, err := db.ExecContext(ctx, `INSERT INTO usage (ts, key_id, project_id, model, served_model, + prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id, ttft_ms) + VALUES (?, 'k', 'p', 'm', 'm', 1, 1, 5, 200, 700, 'u', 0)`, ts); err != nil { + t.Fatal(err) + } + } + if err := migrate(ctx, db, "sqlite"); err != nil { + t.Fatal(err) + } + // 600,001 and 600,002 share a five-minute slice and 900,001 starts the + // next; all three are in the first hour, and the first day. + for table, want := range map[string]int64{"usage_5m": 2, "usage_1h": 1, "usage_1d": 1} { + var slices, reqs, cost, lat3, ttftN int64 + if err := db.QueryRowContext(ctx, `SELECT COUNT(*), SUM(requests), SUM(cost_micros), SUM(lat_3), SUM(ttft_n) + FROM `+table).Scan(&slices, &reqs, &cost, &lat3, &ttftN); err != nil { + t.Fatal(err) + } + if slices != want || reqs != 3 || cost != 15 || lat3 != 3 || ttftN != 0 { + t.Errorf("%s: %d slices, %d requests, %d spent, %d in <=1000ms, %d first tokens; want %d, 3, 15, 3, 0", + table, slices, reqs, cost, lat3, ttftN, want) + } + } +} + +func TestATotalReadsEachTierForItsOwnStretchOnly(t *testing.T) { + // A total since a moment two days back, a few minutes into a slice, is + // read from the rows, then five-minute slices, then hours, then days. + // One request lands in each stretch, and one just before the moment; + // counting any of them twice, or the early one at all, shows here. + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + day := time.UnixMilli(tier1d.floor(time.Now().Add(-48 * time.Hour))) + since := day.Add(22*time.Hour + 52*time.Minute + 30*time.Second) + var batch []Usage + for i, at := range []time.Time{ + since.Add(-time.Second), // before: not counted + since.Add(time.Minute), // the rows + since.Add(5 * time.Minute), // five-minute slices + since.Add(20 * time.Minute), // hours + since.Add(2 * time.Hour), // days + } { + batch = append(batch, Usage{At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", + ServedModel: "m", CostMicros: int64(1) << i, Status: 200}) + } + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + users, err := s.UsageByUser(ctx, since) + if err != nil { + t.Fatal(err) + } + spend, err := s.SpendSince(ctx, since) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 || users[0].Requests != 4 || users[0].CostMicros != 2+4+8+16 || spend["k"] != 30 { + t.Errorf("people %+v, spend %d; want 4 requests and 30 spent", users, spend["k"]) + } + }) +} diff --git a/internal/store/selfusage.go b/internal/store/selfusage.go index 717d9fb6..371a6925 100644 --- a/internal/store/selfusage.go +++ b/internal/store/selfusage.go @@ -28,17 +28,18 @@ func (s *DB) UsageOfUser( if userID == "" { return nil, nil, fmt.Errorf("whose usage?") } + src, args := usageSince(since) places, err := queryAll(ctx, s.db, ` SELECT COALESCE(NULLIF(u.project_id, ''), ''), COALESCE(o.name || ' / ' || t.name || ' / ' || p.name, ''), - COUNT(*), SUM(u.prompt_tokens + u.completion_tokens), SUM(u.cost_micros) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens + u.completion_tokens), SUM(u.cost_micros) + FROM `+src+` u LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id LEFT JOIN organizations o ON o.id = t.org_id - WHERE u.user_id = ? AND u.ts >= ? + WHERE u.user_id = ? GROUP BY 1, 2 - ORDER BY 4 DESC`, []any{userID, millis(since)}, + ORDER BY 4 DESC`, append(args, userID), func(rows *sql.Rows) (PlaceUsage, error) { var pu PlaceUsage return pu, rows.Scan(&pu.ProjectID, &pu.Path, &pu.Requests, &pu.Tokens, &pu.CostMicros) @@ -51,12 +52,20 @@ func (s *DB) UsageOfUser( width = 24 * time.Hour } ms := width.Milliseconds() - got, err := queryAll(ctx, s.db, ` + q, from := ` SELECT (ts / ?) * ?, COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros) FROM usage WHERE user_id = ? AND ts >= ? - GROUP BY 1 ORDER BY 1`, []any{ms, ms, userID, millis(since)}, + GROUP BY 1 ORDER BY 1`, millis(since) + if t, ok := tierForSlices(width); ok { + q, from = ` + SELECT (bucket / ?) * ?, SUM(requests), SUM(errors), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros) + FROM `+t.table+` WHERE user_id = ? AND bucket >= ? + GROUP BY 1 ORDER BY 1`, t.floor(since) + } + got, err := queryAll(ctx, s.db, q, []any{ms, ms, userID, from}, func(rows *sql.Rows) (Bucket, error) { var b Bucket var start int64 @@ -131,8 +140,9 @@ func (s *DB) SpendOfPool(ctx context.Context, l Limits, since time.Time) (int64, return 0, nil } var total sql.NullInt64 + src, srcArgs := usageSince(since) err := s.db.QueryRowContext(ctx, - `SELECT SUM(u.cost_micros) FROM usage u WHERE `+where+` AND u.ts >= ?`, - append(args, millis(since))...).Scan(&total) + `SELECT SUM(u.cost_micros) FROM `+src+` u WHERE `+where, + append(srcArgs, args...)...).Scan(&total) return total.Int64, err } diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 3cb4c300..de4c72f8 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -348,14 +348,14 @@ func (s *DB) applyDefaults(e *Effective) { // the tree. Personal keys and the chat page's requests have no project and are // kept: dropping them is what would reset a person's budget on every restart. func (s *DB) SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, `SELECT u.key_id, u.user_id, COALESCE(p.id, ''), COALESCE(t.id, ''), COALESCE(t.org_id, ''), SUM(u.cost_micros) - FROM usage u + FROM `+src+` u LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id - WHERE u.ts >= ? - GROUP BY u.key_id, u.user_id, p.id, t.id, t.org_id`, millis(since)) + GROUP BY u.key_id, u.user_id, p.id, t.id, t.org_id`, args...) if err != nil { return nil, err } @@ -439,14 +439,15 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error { return err } } - return nil + return writeRollup(ctx, tx, records) }) } // SpendSince totals cost per key since a moment. func (s *DB) SpendSince(ctx context.Context, since time.Time) (map[string]int64, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, - `SELECT key_id, SUM(cost_micros) FROM usage WHERE ts >= ? GROUP BY key_id`, millis(since)) + `SELECT u.key_id, SUM(u.cost_micros) FROM `+src+` u GROUP BY u.key_id`, args...) if err != nil { return nil, err } @@ -620,16 +621,15 @@ var _ Store = (*DB)(nil) // to show: a row identified only by an opaque id tells an operator nothing. // Revoked keys still appear, because their usage still happened. func (s *DB) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) { + src, args := usageSince(since) return queryAll(ctx, s.db, ` SELECT u.key_id, COALESCE(k.name, ''), COALESCE(k.key_prefix, ''), u.project_id, COALESCE(us.email, ''), COALESCE(o.name || ' / ' || t.name || ' / ' || p.name, ''), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts) + FROM `+src+` u LEFT JOIN api_keys k ON k.id = u.key_id -- Who, as recorded with the spend; the owner of the key for rows -- written before the person was. @@ -637,12 +637,11 @@ func (s *DB) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id LEFT JOIN organizations o ON o.id = t.org_id - WHERE u.ts >= ? -- Everything selected beside the sums is grouped, which PostgreSQL -- wants; each is determined by the key and project already there. GROUP BY u.key_id, u.project_id, k.name, k.key_prefix, us.email, o.name, t.name, p.name - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, - []any{millis(since)}, func(rows *sql.Rows) (KeyUsage, error) { + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC`, + args, func(rows *sql.Rows) (KeyUsage, error) { var k KeyUsage var last int64 err := rows.Scan(&k.KeyID, &k.KeyName, &k.KeyPrefix, &k.ProjectID, @@ -664,7 +663,9 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat } ms := width.Milliseconds() - rows, err := s.db.QueryContext(ctx, ` + // From the rollup when the slices are whole rollup slices, which every + // page asks for; from the rows otherwise. + q, from := ` SELECT (ts / ?) * ?, COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), @@ -674,7 +675,16 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat -- selected one syntactically: "ts / ?" and "(ts / ?) * ?" are not the -- same text, so it saw a bare ts and refused. The ordinal is the same -- expression by construction, and both dialects accept it. - GROUP BY 1 ORDER BY 1`, ms, ms, millis(since)) + GROUP BY 1 ORDER BY 1`, millis(since) + if t, ok := tierForSlices(width); ok { + q, from = ` + SELECT (bucket / ?) * ?, SUM(requests), SUM(errors), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + 1.0 * SUM(latency_ms_sum) / SUM(requests) + FROM `+t.table+` WHERE bucket >= ? + GROUP BY 1 ORDER BY 1`, t.floor(since) + } + rows, err := s.db.QueryContext(ctx, q, ms, ms, from) if err != nil { return nil, err } @@ -684,13 +694,13 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat for rows.Next() { var start int64 var b Bucket - var latency float64 + var latency sql.NullFloat64 if err := rows.Scan(&start, &b.Requests, &b.Errors, &b.Prompt, &b.Completion, &b.CostMicros, &latency); err != nil { return nil, err } b.Start = fromMillis(start) - b.LatencyMs = int64(latency) + b.LatencyMs = int64(latency.Float64) found[start] = b } if err := rows.Err(); err != nil { @@ -736,23 +746,21 @@ func (u UserUsage) Tokens() int64 { return u.Prompt + u.Completion } // gets skipped. Keys with no owner — issued by an operator before accounts // existed — are grouped under an empty user, which is honest rather than tidy. func (s *DB) UsageByUser(ctx context.Context, since time.Time) ([]UserUsage, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, ` SELECT u.user_id, COALESCE(us.email, ''), COALESCE(us.name, ''), COUNT(DISTINCT u.key_id), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts) + FROM `+src+` u LEFT JOIN users us ON us.id = NULLIF(u.user_id, '') - WHERE u.ts >= ? -- Every selected column that is not an aggregate has to be grouped. -- SQLite picks an arbitrary row for the rest and carries on; PostgreSQL -- refuses, so this query worked in tests and returned nothing but an -- error on the cluster. The email and name are functionally dependent -- on the user id, so grouping by them changes no result. GROUP BY u.user_id, COALESCE(us.email, ''), COALESCE(us.name, '') - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, millis(since)) + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC`, args...) if err != nil { return nil, err } @@ -793,14 +801,14 @@ func (m ModelUsage) Tokens() int64 { return m.Prompt + m.Completion } // cluster. The requested name is kept in the same table for anyone who needs to // ask the other question. func (s *DB) UsageByModel(ctx context.Context, since time.Time) ([]ModelUsage, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, ` SELECT CASE WHEN served_model != '' THEN served_model ELSE model END, - COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - SUM(cost_micros), - SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), - AVG(latency_ms) - FROM usage WHERE ts >= ? - GROUP BY 1 ORDER BY COUNT(*) DESC`, millis(since)) + SUM(requests), SUM(prompt_tokens), SUM(completion_tokens), + SUM(cost_micros), SUM(errors), + 1.0 * SUM(latency_ms_sum) / SUM(requests) + FROM `+src+` u + GROUP BY 1 ORDER BY SUM(requests) DESC`, args...) if err != nil { return nil, err } diff --git a/internal/store/usagescope.go b/internal/store/usagescope.go index 24dcc1d2..8fa14b40 100644 --- a/internal/store/usagescope.go +++ b/internal/store/usagescope.go @@ -80,21 +80,19 @@ func (s *DB) UsageByScope(ctx context.Context, scope Scope, since time.Time) ([] // same expression it is grouped on. PostgreSQL requires both; SQLite // silently picks a row for whatever is missing, which is how three of these // queries shipped working in tests and failing on the cluster. + src, args := usageSince(since) q := ` SELECT ` + c.id + `, ` + c.name + `, COUNT(DISTINCT u.key_id), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts), + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts), MAX(l.budget_micros), MAX(COALESCE(l.budget_period, '')) - FROM usage u` + c.joins + ` + FROM ` + src + ` u` + c.joins + ` LEFT JOIN limits l ON l.scope_type = ? AND l.scope_id = ` + c.id + ` - WHERE u.ts >= ? GROUP BY ` + c.id + `, ` + c.name + ` - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC` + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC` - rows, err := s.db.QueryContext(ctx, q, string(scope), millis(since)) + rows, err := s.db.QueryContext(ctx, q, append(args, string(scope))...) if err != nil { return nil, err } diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go index 56fd71a5..398adf78 100644 --- a/internal/store/usageseries.go +++ b/internal/store/usageseries.go @@ -145,8 +145,22 @@ func (s *DB) usageBuckets( WHERE u.ts >= ?` + inOrg + ` GROUP BY 1, 2, 3 ORDER BY 1, 3` + from := millis(since) + // From the rollup for slices that are whole rollup slices — every + // window the pages offer — and from the rows for anything finer. + if t, ok := tierForSlices(width); ok { + q = ` + SELECT ` + c.id + `, ` + c.name + `, (u.bucket / ?) * ?, + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors) + FROM ` + t.table + ` u` + c.joins + ` + WHERE u.bucket >= ?` + inOrg + ` + GROUP BY 1, 2, 3 + ORDER BY 1, 3` + from = t.floor(since) + } - args := []any{ms, ms, millis(since)} + args := []any{ms, ms, from} if orgID != "" { args = append(args, orgID) }