From 4454e1322c75ce597e0fcad619a785773b686a6f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:43:01 +0200 Subject: [PATCH 1/4] Read usage from sums kept as it is written, not from every request Charts, totals and budgets summed the request table on every view. Usage is now also summed in the writing transaction into five-minute, hourly and daily tables per key, person, project and model, with latency and first-token histograms. Charts read the coarsest tier their slices allow; totals and budgets compose rows, slices, hours and days, and stay exact. Request records can be given a retention without moving any total. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 22 ++ deploy/examples/server.yaml | 5 + docs/reference.md | 12 + internal/config/config.go | 11 + internal/config/config_test.go | 14 ++ internal/router/router.go | 35 ++- internal/store/copy.go | 5 + .../migrations/postgres/0015_usage_rollup.sql | 204 ++++++++++++++++ .../migrations/sqlite/0015_usage_rollup.sql | 203 ++++++++++++++++ internal/store/postgres_test.go | 2 +- internal/store/rollup.go | 206 ++++++++++++++++ internal/store/rollup_test.go | 219 ++++++++++++++++++ internal/store/selfusage.go | 26 ++- internal/store/sqlite.go | 68 +++--- internal/store/usagescope.go | 14 +- internal/store/usageseries.go | 16 +- 16 files changed, 1013 insertions(+), 49 deletions(-) create mode 100644 internal/store/migrations/postgres/0015_usage_rollup.sql create mode 100644 internal/store/migrations/sqlite/0015_usage_rollup.sql create mode 100644 internal/store/rollup.go create mode 100644 internal/store/rollup_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 609bbcd..f5d2de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,28 @@ and a figure the driver does not give is left out rather than shown as zero. each card and whose quota it counts against, for joining the GPU series to models and tenants. +### Usage is read from sums kept as it is written + +Every chart and total — the usage tab, quotas, the dashboard, a person's own +page, and budgets restored at start — summed the request table again on each +view: a row per request, scanned whole. Usage is now also summed as it is +written, in the same transaction, into five-minute slices, hours and days per +key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration +0015 fills them from what is already recorded). + +- **Charts** read the coarsest of the three their slices are made of: a day in + 20-minute slices from the five-minute table, a week or a month from hours. +- **Totals and budgets** read the rows up to the next five-minute slice, then + slices, hours and days — exact, not rounded to a slice. On three million + requests over thirty days, a month's total per person went from 705 ms to + 18 ms on PostgreSQL. +- **Latency and time to first token** are kept as histograms in the sums, so a + percentile per team or model can be read after the rows are gone. +- **`auth.usage_retention`** (e.g. `2160h`) deletes request records older than + that, hourly. Totals, charts and budgets are unchanged by it — they read the + sums — but `ctl request` finds nothing older. Unset keeps every record, as + before. + ### People, limits and usage are one page Managing a team meant its members on the people page, its quota on the quotas diff --git a/deploy/examples/server.yaml b/deploy/examples/server.yaml index 52ddcfb..a24e4e7 100644 --- a/deploy/examples/server.yaml +++ b/deploy/examples/server.yaml @@ -92,6 +92,11 @@ agent_listen: ":7443" # flush_interval: 200ms # batch_size: 500 # buffer_size: 8192 +# +# # How long the record of each request is kept (what `ctl request` reads). +# # Charts, totals and budgets come from five-minute sums that are kept, so +# # this only bounds the table that grows by a row a request. 0 keeps all. +# usage_retention: 2160h # 90 days router: scrape_interval: 2s diff --git a/docs/reference.md b/docs/reference.md index 45e780f..aa1e353 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -39,6 +39,7 @@ rather than repeated. | `flush_interval` | `time.Duration` | [Auth](#auth) | | `batch_size` | `int` | [Auth](#auth) | | `buffer_size` | `int` | [Auth](#auth) | +| `usage_retention` | `time.Duration` | [Auth](#auth) | | `name` | `string` | [Band](#band) | | `priority` | `int` | [Band](#band) | | `saturation_ceiling` | `float64` | [Band](#band) | @@ -367,6 +368,17 @@ BufferSize bounds the queue. Beyond it records are dropped rather than blocking a request: losing accounting is bad, turning a stalled database into an outage is worse. +### `usage_retention` + +`time.Duration` + +UsageRetention is how long the record of each request is kept: who sent +it, which replica answered, how long it queued — what `ctl request` +reads. Zero keeps it for ever. Totals, charts and budgets are read from +five-minute sums kept beside the records, which are not pruned, so this +bounds the table that grows by a row a request without changing any +figure a page shows. At least a day when set. + ## Band ### `name` diff --git a/internal/config/config.go b/internal/config/config.go index 24fefc2..5cd0a9e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -257,6 +257,14 @@ type Auth struct { // blocking a request: losing accounting is bad, turning a stalled database // into an outage is worse. BufferSize int `koanf:"buffer_size"` + + // UsageRetention is how long the record of each request is kept: who sent + // it, which replica answered, how long it queued — what `ctl request` + // reads. Zero keeps it for ever. Totals, charts and budgets are read from + // five-minute sums kept beside the records, which are not pruned, so this + // bounds the table that grows by a row a request without changing any + // figure a page shows. At least a day when set. + UsageRetention time.Duration `koanf:"usage_retention"` } // Personal configures use that belongs to a person rather than a project. @@ -573,6 +581,9 @@ func (s Server) Validate() error { if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") } + if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour { + return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention) + } if s.Threads.Store != "" && s.Threads.RetainHours <= 0 { return fmt.Errorf("threads.retain_hours must be positive when threads.store is set") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 47c1bb8..91e022b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -206,3 +206,17 @@ func TestEnginesDefaultToTheCheckedVersions(t *testing.T) { t.Error("an emptied engine is still wanted") } } + +func TestAUsageRetentionUnderADayIsRefused(t *testing.T) { + // Shorter than a day is almost certainly a unit slip — 90m for 90 days — + // and would delete every request record within the hour. + s := Default() + s.Auth.UsageRetention = 90 * time.Minute + if err := s.Validate(); err == nil || !strings.Contains(err.Error(), "usage_retention") { + t.Errorf("a 90-minute retention was accepted: %v", err) + } + s.Auth.UsageRetention = 90 * 24 * time.Hour + if err := s.Validate(); err != nil { + t.Errorf("a 90-day retention was refused: %v", err) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index bfa0503..3b62a57 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -83,7 +83,9 @@ type Router struct { auth *auth.Authenticator accounts *account.Service store *storepkg.DB - acct *accounting.Accountant + // usageRetention is how long request records are kept; zero is for ever. + usageRetention time.Duration + acct *accounting.Accountant // threads holds conversations and memory notes. It is store when auth is on, // its own file when only the threads block is configured, and nil when @@ -248,6 +250,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { return nil, fmt.Errorf("auth store: %w", err) } r.store = st + r.usageRetention = cfg.Auth.UsageRetention // Applied where no scope set one, so a project created without limits // is still metered. Zero leaves it unlimited, which was the old // behaviour and is now a choice rather than an omission. @@ -412,6 +415,36 @@ func (r *Router) Start(ctx context.Context) { if r.threads != nil { go r.pruneThreads(ctx) } + if r.store != nil && r.usageRetention > 0 { + go r.pruneUsage(ctx, r.usageRetention) + } +} + +// pruneUsage deletes request records older than the retention, hourly and +// once at startup, as pruneThreads does. The five-minute sums stay, so no +// total or chart moves; what goes is the trail of each old request. +func (r *Router) pruneUsage(ctx context.Context, keep time.Duration) { + prune := func() { + n, err := r.store.PruneUsage(ctx, time.Now().Add(-keep)) + if err != nil { + r.log.Warn("could not prune old request records", "err", err) + return + } + if n > 0 { + r.log.Info("removed request records past their retention", "rows", n, "retention", keep) + } + } + prune() + t := time.NewTicker(time.Hour) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + prune() + } + } } // pruneThreads enforces each project's retention window. diff --git a/internal/store/copy.go b/internal/store/copy.go index 1c72157..9889139 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -33,6 +33,11 @@ var copyOrder = []string{ // quota on a scope that is later removed is inert, not an error. "capacity_quotas", "usage", + // Copied rather than rebuilt from the rows: once request rows have a + // retention, the sums are the only record of what is older. + "usage_5m", + "usage_1h", + "usage_1d", "models", // No foreign key to models, deliberately: a measurement outlives the // catalog entry it was taken for, and losing it when somebody removes a diff --git a/internal/store/migrations/postgres/0015_usage_rollup.sql b/internal/store/migrations/postgres/0015_usage_rollup.sql new file mode 100644 index 0000000..39abcf7 --- /dev/null +++ b/internal/store/migrations/postgres/0015_usage_rollup.sql @@ -0,0 +1,204 @@ +-- Ported from the SQLite migration of the same number. +-- Usage summed as it is written, per key, person, project and model: into +-- five-minute slices (usage_5m), hours (usage_1h) and days (usage_1d). +-- +-- Every chart and total read the raw table, one row per request, and summed it +-- again on every page view: fine for thousands of requests a day, and a full +-- scan of tens of millions on a cluster in real use. The sums are kept in the +-- transaction that writes the rows, so they never disagree with them; the raw +-- table stays for following one request, and can be given a retention of its +-- own (auth.usage_retention). +-- +-- Three widths because how well a slice compresses depends on traffic: many +-- keys each sending a little leave a five-minute slice with nearly a row per +-- request, which an hour or a day folds much further. A chart reads the +-- coarsest width its slices are made of; a total since a moment reads the +-- rows up to the next five-minute slice, those slices up to the hour, hours +-- up to the day and days from there — exact, and a month in about thirty +-- rows per key and model. +-- +-- Latency and time to first token are kept as histograms, a count per fixed +-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, +-- and above), which is what a percentile per team or model is read from after +-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose +-- first token was measured. + +CREATE TABLE usage_5m ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_5m_by_bucket ON usage_5m(bucket); +CREATE INDEX usage_5m_by_project ON usage_5m(project_id, bucket); +CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); +CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); + +INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1h ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1h_by_bucket ON usage_1h(bucket); +CREATE INDEX usage_1h_by_project ON usage_1h(project_id, bucket); +CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); +CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); + +INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1d ( + bucket BIGINT NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests BIGINT NOT NULL DEFAULT 0, + errors BIGINT NOT NULL DEFAULT 0, + prompt_tokens BIGINT NOT NULL DEFAULT 0, + completion_tokens BIGINT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + latency_ms_sum BIGINT NOT NULL DEFAULT 0, + queue_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_ms_sum BIGINT NOT NULL DEFAULT 0, + ttft_n BIGINT NOT NULL DEFAULT 0, + last_ts BIGINT NOT NULL DEFAULT 0, + lat_0 BIGINT NOT NULL DEFAULT 0, + lat_1 BIGINT NOT NULL DEFAULT 0, + lat_2 BIGINT NOT NULL DEFAULT 0, + lat_3 BIGINT NOT NULL DEFAULT 0, + lat_4 BIGINT NOT NULL DEFAULT 0, + lat_5 BIGINT NOT NULL DEFAULT 0, + lat_6 BIGINT NOT NULL DEFAULT 0, + lat_7 BIGINT NOT NULL DEFAULT 0, + lat_8 BIGINT NOT NULL DEFAULT 0, + lat_9 BIGINT NOT NULL DEFAULT 0, + lat_10 BIGINT NOT NULL DEFAULT 0, + ttft_0 BIGINT NOT NULL DEFAULT 0, + ttft_1 BIGINT NOT NULL DEFAULT 0, + ttft_2 BIGINT NOT NULL DEFAULT 0, + ttft_3 BIGINT NOT NULL DEFAULT 0, + ttft_4 BIGINT NOT NULL DEFAULT 0, + ttft_5 BIGINT NOT NULL DEFAULT 0, + ttft_6 BIGINT NOT NULL DEFAULT 0, + ttft_7 BIGINT NOT NULL DEFAULT 0, + ttft_8 BIGINT NOT NULL DEFAULT 0, + ttft_9 BIGINT NOT NULL DEFAULT 0, + ttft_10 BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1d_by_bucket ON usage_1d(bucket); +CREATE INDEX usage_1d_by_project ON usage_1d(project_id, bucket); +CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); +CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); + +INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; diff --git a/internal/store/migrations/sqlite/0015_usage_rollup.sql b/internal/store/migrations/sqlite/0015_usage_rollup.sql new file mode 100644 index 0000000..17d9ae9 --- /dev/null +++ b/internal/store/migrations/sqlite/0015_usage_rollup.sql @@ -0,0 +1,203 @@ +-- Usage summed as it is written, per key, person, project and model: into +-- five-minute slices (usage_5m), hours (usage_1h) and days (usage_1d). +-- +-- Every chart and total read the raw table, one row per request, and summed it +-- again on every page view: fine for thousands of requests a day, and a full +-- scan of tens of millions on a cluster in real use. The sums are kept in the +-- transaction that writes the rows, so they never disagree with them; the raw +-- table stays for following one request, and can be given a retention of its +-- own (auth.usage_retention). +-- +-- Three widths because how well a slice compresses depends on traffic: many +-- keys each sending a little leave a five-minute slice with nearly a row per +-- request, which an hour or a day folds much further. A chart reads the +-- coarsest width its slices are made of; a total since a moment reads the +-- rows up to the next five-minute slice, those slices up to the hour, hours +-- up to the day and days from there — exact, and a month in about thirty +-- rows per key and model. +-- +-- Latency and time to first token are kept as histograms, a count per fixed +-- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, +-- and above), which is what a percentile per team or model is read from after +-- the rows are gone; a mean cannot do that. ttft_* counts only requests whose +-- first token was measured. + +CREATE TABLE usage_5m ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_5m_by_bucket ON usage_5m(bucket); +CREATE INDEX usage_5m_by_project ON usage_5m(project_id, bucket); +CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); +CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); + +INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1h ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1h_by_bucket ON usage_1h(bucket); +CREATE INDEX usage_1h_by_project ON usage_1h(project_id, bucket); +CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); +CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); + +INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; + +CREATE TABLE usage_1d ( + bucket INTEGER NOT NULL, + key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + project_id TEXT NOT NULL, + model TEXT NOT NULL, + served_model TEXT NOT NULL, + requests INTEGER NOT NULL DEFAULT 0, + errors INTEGER NOT NULL DEFAULT 0, + prompt_tokens INTEGER NOT NULL DEFAULT 0, + completion_tokens INTEGER NOT NULL DEFAULT 0, + cost_micros INTEGER NOT NULL DEFAULT 0, + latency_ms_sum INTEGER NOT NULL DEFAULT 0, + queue_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_ms_sum INTEGER NOT NULL DEFAULT 0, + ttft_n INTEGER NOT NULL DEFAULT 0, + last_ts INTEGER NOT NULL DEFAULT 0, + lat_0 INTEGER NOT NULL DEFAULT 0, + lat_1 INTEGER NOT NULL DEFAULT 0, + lat_2 INTEGER NOT NULL DEFAULT 0, + lat_3 INTEGER NOT NULL DEFAULT 0, + lat_4 INTEGER NOT NULL DEFAULT 0, + lat_5 INTEGER NOT NULL DEFAULT 0, + lat_6 INTEGER NOT NULL DEFAULT 0, + lat_7 INTEGER NOT NULL DEFAULT 0, + lat_8 INTEGER NOT NULL DEFAULT 0, + lat_9 INTEGER NOT NULL DEFAULT 0, + lat_10 INTEGER NOT NULL DEFAULT 0, + ttft_0 INTEGER NOT NULL DEFAULT 0, + ttft_1 INTEGER NOT NULL DEFAULT 0, + ttft_2 INTEGER NOT NULL DEFAULT 0, + ttft_3 INTEGER NOT NULL DEFAULT 0, + ttft_4 INTEGER NOT NULL DEFAULT 0, + ttft_5 INTEGER NOT NULL DEFAULT 0, + ttft_6 INTEGER NOT NULL DEFAULT 0, + ttft_7 INTEGER NOT NULL DEFAULT 0, + ttft_8 INTEGER NOT NULL DEFAULT 0, + ttft_9 INTEGER NOT NULL DEFAULT 0, + ttft_10 INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bucket, key_id, user_id, project_id, model, served_model) +); +CREATE INDEX usage_1d_by_bucket ON usage_1d(bucket); +CREATE INDEX usage_1d_by_project ON usage_1d(project_id, bucket); +CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); +CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); + +INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, + requests, errors, prompt_tokens, completion_tokens, cost_micros, + latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, + lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, + ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) +SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), + SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), + SUM(CASE WHEN ttft_ms > 0 AND ttft_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 100 AND ttft_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 250 AND ttft_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 500 AND ttft_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 1000 AND ttft_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 2500 AND ttft_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 5000 AND ttft_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 10000 AND ttft_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 30000 AND ttft_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 60000 AND ttft_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN ttft_ms > 0 AND ttft_ms > 120000 THEN 1 ELSE 0 END) +FROM usage +GROUP BY 1, 2, 3, 4, 5, 6; diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index 32465de..746f13f 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -41,7 +41,7 @@ func openPG(t *testing.T) *DB { // inherits the previous one's lease is refused for reasons that have // nothing to do with what it is testing. It passed on SQLite, where // every subtest gets a fresh file, and failed only here. - `TRUNCATE thread_messages, threads, memory_notes, usage, api_keys, memberships, + `TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships, key_claims, limits, models, sessions, users, projects, teams, organizations, leadership RESTART IDENTITY CASCADE`) if err != nil { diff --git a/internal/store/rollup.go b/internal/store/rollup.go new file mode 100644 index 0000000..4333ee2 --- /dev/null +++ b/internal/store/rollup.go @@ -0,0 +1,206 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// tier is one table usage is summed into as it is written: slices of a fixed +// width, per key, person, project and model. +type tier struct { + table string + ms int64 +} + +var ( + tier5m = tier{"usage_5m", int64(5 * time.Minute / time.Millisecond)} + tier1h = tier{"usage_1h", int64(time.Hour / time.Millisecond)} + tier1d = tier{"usage_1d", int64(24 * time.Hour / time.Millisecond)} + // tiers, finest first. + tiers = []tier{tier5m, tier1h, tier1d} +) + +// floor is the start of the slice a moment falls in. A chart read from a +// tier starts its first slice there, as a chart read from the rows always did. +func (t tier) floor(at time.Time) int64 { return (millis(at) / t.ms) * t.ms } + +// ceil is the start of the first slice wholly after a moment. +func (t tier) ceil(at time.Time) int64 { + return ((millis(at) + t.ms - 1) / t.ms) * t.ms +} + +// tierForSlices is the coarsest tier whose slices make up a chart slice of +// width exactly, and false when none does — a slice finer than five minutes, +// or not a whole number of them, is read from the rows. +func tierForSlices(width time.Duration) (tier, bool) { + ms := width.Milliseconds() + for i := len(tiers) - 1; i >= 0; i-- { + if ms >= tiers[i].ms && ms%tiers[i].ms == 0 { + return tiers[i], true + } + } + return tier{}, false +} + +// LatencyBounds are the upper edges, in milliseconds, of the histogram buckets +// latency and time to first token are counted in; one more bucket holds +// everything above the last. Fixed, because buckets are only summable across +// slices, keys and teams when every row has the same ones. +var LatencyBounds = []int64{100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000} + +// latencyBucket is the histogram bucket a duration is counted in. +func latencyBucket(ms int64) int { + for i, b := range LatencyBounds { + if ms <= b { + return i + } + } + return len(LatencyBounds) +} + +// rollupKey is one row of a tier: a slice, and who spent what on what. +type rollupKey struct { + bucket int64 + keyID, userID, projectID, model, servedModel string +} + +// rollupRow is what a batch adds to one row. +type rollupRow struct { + requests, errors, prompt, completion, cost int64 + latency, queue, ttft, ttftN, last int64 + lat, ttftH []int64 +} + +// rollupBatch sums a batch of records into the rows of one tier they add to. +func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey) { + rows := map[rollupKey]*rollupRow{} + var order []rollupKey + n := len(LatencyBounds) + 1 + for _, r := range records { + ts := millis(r.At) + k := rollupKey{(ts / t.ms) * t.ms, r.KeyID, r.UserID, r.ProjectID, r.Model, r.ServedModel} + row, ok := rows[k] + if !ok { + row = &rollupRow{lat: make([]int64, n), ttftH: make([]int64, n)} + rows[k] = row + order = append(order, k) + } + row.requests++ + if r.Status >= 400 { + row.errors++ + } + row.prompt += r.PromptTokens + row.completion += r.CompletionTokens + row.cost += r.CostMicros + row.latency += r.LatencyMs + row.queue += r.QueueMs + row.lat[latencyBucket(r.LatencyMs)]++ + if r.TTFTMs > 0 { + row.ttft += r.TTFTMs + row.ttftN++ + row.ttftH[latencyBucket(r.TTFTMs)]++ + } + row.last = max(row.last, ts) + } + return rows, order +} + +// rollupColumns are a tier's columns, in the order rollupUpsert takes them. +var rollupColumns = func() []string { + n := len(LatencyBounds) + 1 + cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model", + "requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros", + "latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"} + for i := range n { + cols = append(cols, fmt.Sprintf("lat_%d", i)) + } + for i := range n { + cols = append(cols, fmt.Sprintf("ttft_%d", i)) + } + return cols +}() + +// rollupUpsert is the statement that adds a batch's sums to a tier. +func rollupUpsert(table string) string { + var sets []string + for _, c := range rollupColumns[6:] { + if c == "last_ts" { + sets = append(sets, "last_ts = CASE WHEN excluded.last_ts > "+table+".last_ts "+ + "THEN excluded.last_ts ELSE "+table+".last_ts END") + continue + } + sets = append(sets, c+" = "+table+"."+c+" + excluded."+c) + } + return `INSERT INTO ` + table + ` (` + strings.Join(rollupColumns, ", ") + `) + VALUES (` + strings.TrimSuffix(strings.Repeat("?, ", len(rollupColumns)), ", ") + `) + ON CONFLICT (bucket, key_id, user_id, project_id, model, served_model) + DO UPDATE SET ` + strings.Join(sets, ", ") +} + +// writeRollup adds a batch to every tier, inside the transaction that writes +// its rows, so the sums never disagree with them. +func writeRollup(ctx context.Context, tx conn, records []Usage) error { + for _, t := range tiers { + rows, order := rollupBatch(t, records) + stmt, err := tx.PrepareContext(ctx, rollupUpsert(t.table)) + if err != nil { + return err + } + for _, k := range order { + r := rows[k] + args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel, + r.requests, r.errors, r.prompt, r.completion, r.cost, + r.latency, r.queue, r.ttft, r.ttftN, r.last} + for _, v := range r.lat { + args = append(args, v) + } + for _, v := range r.ttftH { + args = append(args, v) + } + if _, err := stmt.ExecContext(ctx, args...); err != nil { + _ = stmt.Close() + return fmt.Errorf("adding to %s: %w", t.table, err) + } + } + if err := stmt.Close(); err != nil { + return err + } + } + return nil +} + +// usageSince is every request since a moment, exactly, as a table with the +// columns a tier has (but the histograms): the rows up to the next +// five-minute slice, those slices up to the next hour, hours up to the next +// day, and days from there. Totals are read from this — a budget against it +// must be exact, and so is everything else, at the cost of a few rows read +// from each of four tables rather than a window rounded to a slice. +// +// Its placeholders come first in any query it is put into. +func usageSince(since time.Time) (string, []any) { + from := millis(since) + to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since) + const cols = `key_id, user_id, project_id, model, served_model, requests, errors, + prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts` + return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests, + CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors, + prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts + FROM usage WHERE ts >= ? AND ts < ? + UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ? + UNION ALL SELECT ` + cols + ` FROM usage_1h WHERE bucket >= ? AND bucket < ? + UNION ALL SELECT ` + cols + ` FROM usage_1d WHERE bucket >= ?)`, + []any{from, to5m, to5m, to1h, to1h, to1d, to1d} +} + +// PruneUsage deletes request rows older than a moment. The tiers keep their +// sums, so charts, totals and budgets are unchanged; what goes is the record +// of each request — `ctl request` on one that old finds nothing. +func (s *DB) PruneUsage(ctx context.Context, before time.Time) (int64, error) { + res, err := s.db.ExecContext(ctx, `DELETE FROM usage WHERE ts < ?`, millis(before)) + if err != nil { + return 0, err + } + return res.RowsAffected() +} diff --git a/internal/store/rollup_test.go b/internal/store/rollup_test.go new file mode 100644 index 0000000..d1984b1 --- /dev/null +++ b/internal/store/rollup_test.go @@ -0,0 +1,219 @@ +package store + +import ( + "context" + "database/sql" + "path/filepath" + "testing" + "time" +) + +// onBoth runs a test on SQLite, and on PostgreSQL when LLMGW_TEST_POSTGRES +// names one: the rollup is written with an upsert, and the two dialects are +// where an upsert goes wrong. +func onBoth(t *testing.T, f func(t *testing.T, s *DB)) { + t.Run("sqlite", func(t *testing.T) { f(t, openDB(t)) }) + t.Run("postgres", func(t *testing.T) { f(t, openPG(t)) }) +} + +// slice is a moment a minute into an hour, a few hours back: inside a +// five-minute slice and an hour alike. +func slice() time.Time { + return time.UnixMilli(tier1h.floor(time.Now().Add(-3 * time.Hour))).Add(time.Minute) +} + +func TestTheRollupAddsUpToTheRows(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + // Two batches into the same slice, so the second has to add to the + // row the first made rather than replace it. + for _, batch := range [][]Usage{ + { + {At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 10, CompletionTokens: 5, CostMicros: 100, Status: 200, LatencyMs: 80, TTFTMs: 40}, + {At: at.Add(time.Second), KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 1, CompletionTokens: 1, CostMicros: 7, Status: 500, LatencyMs: 3000}, + }, + { + {At: at.Add(2 * time.Second), KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + PromptTokens: 2, CompletionTokens: 3, CostMicros: 50, Status: 200, LatencyMs: 400, TTFTMs: 200}, + }, + } { + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + } + + for _, tr := range tiers { + var req, errs, prompt, cost, lat, ttftN, last, lat0, lat2, lat5, ttft0, ttft1 int64 + err := s.db.QueryRowContext(ctx, `SELECT requests, errors, prompt_tokens, cost_micros, + latency_ms_sum, ttft_n, last_ts, lat_0, lat_2, lat_5, ttft_0, ttft_1 FROM `+tr.table). + Scan(&req, &errs, &prompt, &cost, &lat, &ttftN, &last, &lat0, &lat2, &lat5, &ttft0, &ttft1) + if err != nil { + t.Fatalf("%s: %v", tr.table, err) + } + if req != 3 || errs != 1 || prompt != 13 || cost != 157 || lat != 3480 { + t.Errorf("sums: %d requests, %d errors, %d prompt, %d cost, %d ms; want 3, 1, 13, 157, 3480", + req, errs, prompt, cost, lat) + } + // 80 ms in <=100, 400 in <=500, 3000 in <=5000; first tokens 40 and + // 200, and the failed request had none to count. + if lat0 != 1 || lat2 != 1 || lat5 != 1 || ttftN != 2 || ttft0 != 1 || ttft1 != 1 { + t.Errorf("histograms: lat %d/%d/%d, ttft n=%d %d/%d", lat0, lat2, lat5, ttftN, ttft0, ttft1) + } + if last != millis(at.Add(2*time.Second)) { + t.Errorf("last seen %d, want the latest request", last) + } + } + + b, err := s.UsageBuckets(ctx, at.Add(-time.Hour), time.Hour) + if err != nil { + t.Fatal(err) + } + var total Bucket + for _, x := range b { + total.Requests += x.Requests + total.CostMicros += x.CostMicros + } + if total.Requests != 3 || total.CostMicros != 157 { + t.Errorf("the chart counts %d requests and %d spent, want 3 and 157", total.Requests, total.CostMicros) + } + }) +} + +func TestABudgetIsCountedFromTheMomentNotTheSlice(t *testing.T) { + // Charts start at the slice a window falls in; a budget must not. A + // request a minute before the period began is not the period's. + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + // Before the moment; after it in its own slice; in a later slice of + // the same hour; and in the next hour — one from each part a budget + // is read from. + if err := s.WriteUsage(ctx, []Usage{ + {At: at, KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 100, Status: 200}, + {At: at.Add(2 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 10, Status: 200}, + {At: at.Add(10 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 1, Status: 200}, + {At: at.Add(70 * time.Minute), KeyID: "k", ProjectID: "p", Model: "m", CostMicros: 1000, Status: 200}, + }); err != nil { + t.Fatal(err) + } + spend, err := s.SpendSince(ctx, at.Add(time.Minute)) + if err != nil { + t.Fatal(err) + } + if spend["k"] != 1011 { + t.Errorf("spent %d since mid-slice, want 1011", spend["k"]) + } + }) +} + +func TestPruningRequestsKeepsEveryTotal(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + if err := s.WriteUsage(ctx, []Usage{ + {At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", ServedModel: "m", + CostMicros: 100, Status: 200, RequestID: "req-1"}, + }); err != nil { + t.Fatal(err) + } + n, err := s.PruneUsage(ctx, time.Now()) + if err != nil || n != 1 { + t.Fatalf("pruned %d, %v", n, err) + } + if _, ok, _ := s.UsageByRequest(ctx, "req-1"); ok { + t.Error("a pruned request can still be looked up") + } + spend, _ := s.SpendSince(ctx, time.UnixMilli(tier5m.floor(at))) + users, _ := s.UsageByUser(ctx, at.Add(-time.Hour)) + if spend["k"] != 100 || len(users) != 1 || users[0].Requests != 1 { + t.Errorf("after pruning: spend %v, people %+v; the totals should not move", spend, users) + } + }) +} + +func TestTheMigrationRollsUpWhatIsAlreadyRecorded(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "old.db")+"?_pragma=foreign_keys(1)") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + if _, err := db.ExecContext(ctx, + `CREATE TABLE schema_migrations (version TEXT PRIMARY KEY, applied_at INTEGER NOT NULL)`); err != nil { + t.Fatal(err) + } + names, _ := migrationNames("sqlite") + for _, name := range names { + if name >= "0015" { + break + } + body, _ := migrationFS.ReadFile("migrations/sqlite/" + name) + if err := applyOne(ctx, db, "sqlite", name, string(body)); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + for _, ts := range []int64{600_001, 600_002, 900_001} { + if _, err := db.ExecContext(ctx, `INSERT INTO usage (ts, key_id, project_id, model, served_model, + prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id, ttft_ms) + VALUES (?, 'k', 'p', 'm', 'm', 1, 1, 5, 200, 700, 'u', 0)`, ts); err != nil { + t.Fatal(err) + } + } + if err := migrate(ctx, db, "sqlite"); err != nil { + t.Fatal(err) + } + // 600,001 and 600,002 share a five-minute slice and 900,001 starts the + // next; all three are in the first hour, and the first day. + for table, want := range map[string]int64{"usage_5m": 2, "usage_1h": 1, "usage_1d": 1} { + var slices, reqs, cost, lat3, ttftN int64 + if err := db.QueryRowContext(ctx, `SELECT COUNT(*), SUM(requests), SUM(cost_micros), SUM(lat_3), SUM(ttft_n) + FROM `+table).Scan(&slices, &reqs, &cost, &lat3, &ttftN); err != nil { + t.Fatal(err) + } + if slices != want || reqs != 3 || cost != 15 || lat3 != 3 || ttftN != 0 { + t.Errorf("%s: %d slices, %d requests, %d spent, %d in <=1000ms, %d first tokens; want %d, 3, 15, 3, 0", + table, slices, reqs, cost, lat3, ttftN, want) + } + } +} + +func TestATotalReadsEachTierForItsOwnStretchOnly(t *testing.T) { + // A total since a moment two days back, a few minutes into a slice, is + // read from the rows, then five-minute slices, then hours, then days. + // One request lands in each stretch, and one just before the moment; + // counting any of them twice, or the early one at all, shows here. + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + day := time.UnixMilli(tier1d.floor(time.Now().Add(-48 * time.Hour))) + since := day.Add(22*time.Hour + 52*time.Minute + 30*time.Second) + var batch []Usage + for i, at := range []time.Time{ + since.Add(-time.Second), // before: not counted + since.Add(time.Minute), // the rows + since.Add(5 * time.Minute), // five-minute slices + since.Add(20 * time.Minute), // hours + since.Add(2 * time.Hour), // days + } { + batch = append(batch, Usage{At: at, KeyID: "k", UserID: "u", ProjectID: "p", Model: "m", + ServedModel: "m", CostMicros: int64(1) << i, Status: 200}) + } + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + users, err := s.UsageByUser(ctx, since) + if err != nil { + t.Fatal(err) + } + spend, err := s.SpendSince(ctx, since) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 || users[0].Requests != 4 || users[0].CostMicros != 2+4+8+16 || spend["k"] != 30 { + t.Errorf("people %+v, spend %d; want 4 requests and 30 spent", users, spend["k"]) + } + }) +} diff --git a/internal/store/selfusage.go b/internal/store/selfusage.go index 717d9fb..371a692 100644 --- a/internal/store/selfusage.go +++ b/internal/store/selfusage.go @@ -28,17 +28,18 @@ func (s *DB) UsageOfUser( if userID == "" { return nil, nil, fmt.Errorf("whose usage?") } + src, args := usageSince(since) places, err := queryAll(ctx, s.db, ` SELECT COALESCE(NULLIF(u.project_id, ''), ''), COALESCE(o.name || ' / ' || t.name || ' / ' || p.name, ''), - COUNT(*), SUM(u.prompt_tokens + u.completion_tokens), SUM(u.cost_micros) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens + u.completion_tokens), SUM(u.cost_micros) + FROM `+src+` u LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id LEFT JOIN organizations o ON o.id = t.org_id - WHERE u.user_id = ? AND u.ts >= ? + WHERE u.user_id = ? GROUP BY 1, 2 - ORDER BY 4 DESC`, []any{userID, millis(since)}, + ORDER BY 4 DESC`, append(args, userID), func(rows *sql.Rows) (PlaceUsage, error) { var pu PlaceUsage return pu, rows.Scan(&pu.ProjectID, &pu.Path, &pu.Requests, &pu.Tokens, &pu.CostMicros) @@ -51,12 +52,20 @@ func (s *DB) UsageOfUser( width = 24 * time.Hour } ms := width.Milliseconds() - got, err := queryAll(ctx, s.db, ` + q, from := ` SELECT (ts / ?) * ?, COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros) FROM usage WHERE user_id = ? AND ts >= ? - GROUP BY 1 ORDER BY 1`, []any{ms, ms, userID, millis(since)}, + GROUP BY 1 ORDER BY 1`, millis(since) + if t, ok := tierForSlices(width); ok { + q, from = ` + SELECT (bucket / ?) * ?, SUM(requests), SUM(errors), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros) + FROM `+t.table+` WHERE user_id = ? AND bucket >= ? + GROUP BY 1 ORDER BY 1`, t.floor(since) + } + got, err := queryAll(ctx, s.db, q, []any{ms, ms, userID, from}, func(rows *sql.Rows) (Bucket, error) { var b Bucket var start int64 @@ -131,8 +140,9 @@ func (s *DB) SpendOfPool(ctx context.Context, l Limits, since time.Time) (int64, return 0, nil } var total sql.NullInt64 + src, srcArgs := usageSince(since) err := s.db.QueryRowContext(ctx, - `SELECT SUM(u.cost_micros) FROM usage u WHERE `+where+` AND u.ts >= ?`, - append(args, millis(since))...).Scan(&total) + `SELECT SUM(u.cost_micros) FROM `+src+` u WHERE `+where, + append(srcArgs, args...)...).Scan(&total) return total.Int64, err } diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 3cb4c30..de4c72f 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -348,14 +348,14 @@ func (s *DB) applyDefaults(e *Effective) { // the tree. Personal keys and the chat page's requests have no project and are // kept: dropping them is what would reset a person's budget on every restart. func (s *DB) SpendSinceByScope(ctx context.Context, since time.Time) ([]ScopeSpend, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, `SELECT u.key_id, u.user_id, COALESCE(p.id, ''), COALESCE(t.id, ''), COALESCE(t.org_id, ''), SUM(u.cost_micros) - FROM usage u + FROM `+src+` u LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id - WHERE u.ts >= ? - GROUP BY u.key_id, u.user_id, p.id, t.id, t.org_id`, millis(since)) + GROUP BY u.key_id, u.user_id, p.id, t.id, t.org_id`, args...) if err != nil { return nil, err } @@ -439,14 +439,15 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error { return err } } - return nil + return writeRollup(ctx, tx, records) }) } // SpendSince totals cost per key since a moment. func (s *DB) SpendSince(ctx context.Context, since time.Time) (map[string]int64, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, - `SELECT key_id, SUM(cost_micros) FROM usage WHERE ts >= ? GROUP BY key_id`, millis(since)) + `SELECT u.key_id, SUM(u.cost_micros) FROM `+src+` u GROUP BY u.key_id`, args...) if err != nil { return nil, err } @@ -620,16 +621,15 @@ var _ Store = (*DB)(nil) // to show: a row identified only by an opaque id tells an operator nothing. // Revoked keys still appear, because their usage still happened. func (s *DB) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error) { + src, args := usageSince(since) return queryAll(ctx, s.db, ` SELECT u.key_id, COALESCE(k.name, ''), COALESCE(k.key_prefix, ''), u.project_id, COALESCE(us.email, ''), COALESCE(o.name || ' / ' || t.name || ' / ' || p.name, ''), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts) + FROM `+src+` u LEFT JOIN api_keys k ON k.id = u.key_id -- Who, as recorded with the spend; the owner of the key for rows -- written before the person was. @@ -637,12 +637,11 @@ func (s *DB) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') LEFT JOIN teams t ON t.id = p.team_id LEFT JOIN organizations o ON o.id = t.org_id - WHERE u.ts >= ? -- Everything selected beside the sums is grouped, which PostgreSQL -- wants; each is determined by the key and project already there. GROUP BY u.key_id, u.project_id, k.name, k.key_prefix, us.email, o.name, t.name, p.name - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, - []any{millis(since)}, func(rows *sql.Rows) (KeyUsage, error) { + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC`, + args, func(rows *sql.Rows) (KeyUsage, error) { var k KeyUsage var last int64 err := rows.Scan(&k.KeyID, &k.KeyName, &k.KeyPrefix, &k.ProjectID, @@ -664,7 +663,9 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat } ms := width.Milliseconds() - rows, err := s.db.QueryContext(ctx, ` + // From the rollup when the slices are whole rollup slices, which every + // page asks for; from the rows otherwise. + q, from := ` SELECT (ts / ?) * ?, COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), @@ -674,7 +675,16 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat -- selected one syntactically: "ts / ?" and "(ts / ?) * ?" are not the -- same text, so it saw a bare ts and refused. The ordinal is the same -- expression by construction, and both dialects accept it. - GROUP BY 1 ORDER BY 1`, ms, ms, millis(since)) + GROUP BY 1 ORDER BY 1`, millis(since) + if t, ok := tierForSlices(width); ok { + q, from = ` + SELECT (bucket / ?) * ?, SUM(requests), SUM(errors), + SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), + 1.0 * SUM(latency_ms_sum) / SUM(requests) + FROM `+t.table+` WHERE bucket >= ? + GROUP BY 1 ORDER BY 1`, t.floor(since) + } + rows, err := s.db.QueryContext(ctx, q, ms, ms, from) if err != nil { return nil, err } @@ -684,13 +694,13 @@ func (s *DB) UsageBuckets(ctx context.Context, since time.Time, width time.Durat for rows.Next() { var start int64 var b Bucket - var latency float64 + var latency sql.NullFloat64 if err := rows.Scan(&start, &b.Requests, &b.Errors, &b.Prompt, &b.Completion, &b.CostMicros, &latency); err != nil { return nil, err } b.Start = fromMillis(start) - b.LatencyMs = int64(latency) + b.LatencyMs = int64(latency.Float64) found[start] = b } if err := rows.Err(); err != nil { @@ -736,23 +746,21 @@ func (u UserUsage) Tokens() int64 { return u.Prompt + u.Completion } // gets skipped. Keys with no owner — issued by an operator before accounts // existed — are grouped under an empty user, which is honest rather than tidy. func (s *DB) UsageByUser(ctx context.Context, since time.Time) ([]UserUsage, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, ` SELECT u.user_id, COALESCE(us.email, ''), COALESCE(us.name, ''), COUNT(DISTINCT u.key_id), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts) - FROM usage u + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts) + FROM `+src+` u LEFT JOIN users us ON us.id = NULLIF(u.user_id, '') - WHERE u.ts >= ? -- Every selected column that is not an aggregate has to be grouped. -- SQLite picks an arbitrary row for the rest and carries on; PostgreSQL -- refuses, so this query worked in tests and returned nothing but an -- error on the cluster. The email and name are functionally dependent -- on the user id, so grouping by them changes no result. GROUP BY u.user_id, COALESCE(us.email, ''), COALESCE(us.name, '') - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, millis(since)) + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC`, args...) if err != nil { return nil, err } @@ -793,14 +801,14 @@ func (m ModelUsage) Tokens() int64 { return m.Prompt + m.Completion } // cluster. The requested name is kept in the same table for anyone who needs to // ask the other question. func (s *DB) UsageByModel(ctx context.Context, since time.Time) ([]ModelUsage, error) { + src, args := usageSince(since) rows, err := s.db.QueryContext(ctx, ` SELECT CASE WHEN served_model != '' THEN served_model ELSE model END, - COUNT(*), SUM(prompt_tokens), SUM(completion_tokens), - SUM(cost_micros), - SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), - AVG(latency_ms) - FROM usage WHERE ts >= ? - GROUP BY 1 ORDER BY COUNT(*) DESC`, millis(since)) + SUM(requests), SUM(prompt_tokens), SUM(completion_tokens), + SUM(cost_micros), SUM(errors), + 1.0 * SUM(latency_ms_sum) / SUM(requests) + FROM `+src+` u + GROUP BY 1 ORDER BY SUM(requests) DESC`, args...) if err != nil { return nil, err } diff --git a/internal/store/usagescope.go b/internal/store/usagescope.go index 24dcc1d..8fa14b4 100644 --- a/internal/store/usagescope.go +++ b/internal/store/usagescope.go @@ -80,21 +80,19 @@ func (s *DB) UsageByScope(ctx context.Context, scope Scope, since time.Time) ([] // same expression it is grouped on. PostgreSQL requires both; SQLite // silently picks a row for whatever is missing, which is how three of these // queries shipped working in tests and failing on the cluster. + src, args := usageSince(since) q := ` SELECT ` + c.id + `, ` + c.name + `, COUNT(DISTINCT u.key_id), - COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), - SUM(u.cost_micros), - SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), - MAX(u.ts), + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors), MAX(u.last_ts), MAX(l.budget_micros), MAX(COALESCE(l.budget_period, '')) - FROM usage u` + c.joins + ` + FROM ` + src + ` u` + c.joins + ` LEFT JOIN limits l ON l.scope_type = ? AND l.scope_id = ` + c.id + ` - WHERE u.ts >= ? GROUP BY ` + c.id + `, ` + c.name + ` - ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC` + ORDER BY SUM(u.cost_micros) DESC, SUM(u.requests) DESC` - rows, err := s.db.QueryContext(ctx, q, string(scope), millis(since)) + rows, err := s.db.QueryContext(ctx, q, append(args, string(scope))...) if err != nil { return nil, err } diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go index 56fd71a..398adf7 100644 --- a/internal/store/usageseries.go +++ b/internal/store/usageseries.go @@ -145,8 +145,22 @@ func (s *DB) usageBuckets( WHERE u.ts >= ?` + inOrg + ` GROUP BY 1, 2, 3 ORDER BY 1, 3` + from := millis(since) + // From the rollup for slices that are whole rollup slices — every + // window the pages offer — and from the rows for anything finer. + if t, ok := tierForSlices(width); ok { + q = ` + SELECT ` + c.id + `, ` + c.name + `, (u.bucket / ?) * ?, + SUM(u.requests), SUM(u.prompt_tokens), SUM(u.completion_tokens), + SUM(u.cost_micros), SUM(u.errors) + FROM ` + t.table + ` u` + c.joins + ` + WHERE u.bucket >= ?` + inOrg + ` + GROUP BY 1, 2, 3 + ORDER BY 1, 3` + from = t.floor(since) + } - args := []any{ms, ms, millis(since)} + args := []any{ms, ms, from} if orgID != "" { args = append(args, orgID) } From 9f479b9a702f640450a2a1469a96b999e8577d2a Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:50:16 +0200 Subject: [PATCH 2/4] Show each team's first-token and latency percentiles, errors and refusals Percentiles are read from the histograms kept with the usage sums and shown per group on the usage tab and in the people page's panels. Requests refused by a limit are recorded as 429s against their owner, free of charge, so a team hitting its limits shows as such. Usage can be grouped by model. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 15 ++ internal/accounting/accounting.go | 11 ++ internal/router/handlers.go | 5 + internal/router/pricing_test.go | 36 +++++ internal/store/latency.go | 132 ++++++++++++++++++ internal/store/latency_test.go | 61 ++++++++ .../migrations/postgres/0015_usage_rollup.sql | 18 ++- .../migrations/sqlite/0015_usage_rollup.sql | 18 ++- internal/store/rollup.go | 17 ++- internal/store/usageseries.go | 10 +- internal/web/admin_people.go | 17 ++- internal/web/admin_people_test.go | 24 ++++ internal/web/admin_usage.go | 50 +++++++ internal/web/admin_usage_test.go | 12 +- internal/web/templates/admin_people.html | 7 + internal/web/templates/admin_usage.html | 11 +- internal/web/web.go | 3 + 17 files changed, 418 insertions(+), 29 deletions(-) create mode 100644 internal/store/latency.go create mode 100644 internal/store/latency_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d2de1..f39601e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,21 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration sums — but `ctl request` finds nothing older. Unset keeps every record, as before. +### How each team's requests fared, not only what they cost + +- **Percentiles from the sums.** First token and latency, p50, p95 and p99, + per organization, team, project, person or model, read from the histograms + kept with the usage sums: on the usage tab's table, and in a branch's or + person's panel on the people page, beside the error rate and requests + refused by a limit. +- **Refused requests are counted against their owner.** A request a rate, + token or budget limit refused was counted only in Prometheus, by model; a + team running into its limit looked like a team that had gone quiet. It is + now recorded as a 429 against its key, person and project — no tokens, no + cost — and shown as "Limited". +- **Usage by model** is a grouping of the usage tab, by the model that + answered. + ### People, limits and usage are one page Managing a team meant its members on the people page, its quota on the quotas diff --git a/internal/accounting/accounting.go b/internal/accounting/accounting.go index 66c42e3..a6ada98 100644 --- a/internal/accounting/accounting.go +++ b/internal/accounting/accounting.go @@ -316,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) { } } +// RecordRefused queues the row of a request a limit refused: nothing is +// charged — it used nothing — but it is counted, so a team running into its +// limits shows as one rather than as a team that went quiet. +func (a *Accountant) RecordRefused(u store.Usage) { + if u.KeyID == "" || a.recorder == nil { + return + } + u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0 + a.recorder.Record(u) +} + // Status returns what a key has used, for headers and `ctl`. func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) { if p.KeyID == "" { diff --git a/internal/router/handlers.go b/internal/router/handlers.go index b758f11..e48a60c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -537,6 +537,11 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.acct.RecordRefused(store.Usage{ + At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, + Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests, + LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, + }) r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start)) } diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 40d06ae..408f681 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -188,3 +188,39 @@ func TestAnIDThatIsNotOneIsReplaced(t *testing.T) { t.Error("a plain id was not kept") } } + +func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) { + // Refused requests were counted only in Prometheus, by model: a team + // running into its limit looked, on every usage page, like a team that + // had gone quiet. + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + rpm := 1 + if err := rt.store.PutLimits(t.Context(), store.Limits{Scope: store.ScopeKey, ScopeID: "key-1", RPM: &rpm}); err != nil { + t.Fatal(err) + } + body := map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": "hi"}}} + auth := map[string]string{"Authorization": "Bearer " + secret} + codes := []int{post(t, rt.Handler(), body, auth).Code, post(t, rt.Handler(), body, auth).Code} + if codes[1] != 429 { + t.Fatalf("the second request in the minute answered %v, want 429", codes) + } + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + stats, err := rt.store.StatsBy(t.Context(), store.GroupByProject, time.Now().Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Limited == 1 && s.Requests == 2 { + spend, _ := rt.store.SpendSince(t.Context(), time.Now().Add(-time.Hour)) + if spend["key-1"] == 0 { + t.Error("the request that was served cost nothing") + } + return + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the refused request was never counted against its project") +} diff --git a/internal/store/latency.go b/internal/store/latency.go new file mode 100644 index 0000000..e3699a3 --- /dev/null +++ b/internal/store/latency.go @@ -0,0 +1,132 @@ +package store + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Quantiles are a duration's percentiles, in milliseconds, read from the +// histogram kept with the usage sums. Within a bucket the value is +// interpolated; above the last bucket's edge it is that edge, and Over says +// so. N is how many durations they are read from — zero is no reading. +type Quantiles struct { + P50, P95, P99 int64 + N int64 + Over bool +} + +// GroupStats is one group's requests, how they ended, and how long they took. +type GroupStats struct { + ID string + Requests int64 + Errors int64 + // Limited is requests refused by a rate, token or budget limit. + Limited int64 + Latency Quantiles + // TTFT is time to first token, for requests that streamed one. + TTFT Quantiles +} + +// ErrorRate is the share of requests that failed, limits included, in percent. +func (g GroupStats) ErrorRate() float64 { + if g.Requests == 0 { + return 0 + } + return 100 * float64(g.Errors) / float64(g.Requests) +} + +// StatsBy is each group's request outcomes and latency percentiles since a +// moment, keyed by the group's id. +// +// Read from the hourly sums for a window of two days or more and from the +// five-minute ones otherwise, starting at the slice the moment falls in: +// percentiles are a shape, and a window starting up to one slice early does +// not change one. Budgets, which must be exact, are not read this way. +func (s *DB) StatsBy(ctx context.Context, g Grouping, since time.Time) (map[string]GroupStats, error) { + c, ok := groupColumns[g] + if !ok { + return nil, fmt.Errorf("usage cannot be grouped by %q", g) + } + t := tier5m + if time.Since(since) >= 48*time.Hour { + t = tier1h + } + n := len(LatencyBounds) + 1 + var sums []string + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.lat_%d)", i)) + } + for i := range n { + sums = append(sums, fmt.Sprintf("SUM(u.ttft_%d)", i)) + } + rows, err := s.db.QueryContext(ctx, ` + SELECT `+c.id+`, SUM(u.requests), SUM(u.errors), SUM(u.limited), `+strings.Join(sums, ", ")+` + FROM `+t.table+` u`+c.joins+` + WHERE u.bucket >= ? + GROUP BY 1`, t.floor(since)) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[string]GroupStats{} + for rows.Next() { + var gs GroupStats + lat, ttft := make([]int64, n), make([]int64, n) + dst := []any{&gs.ID, &gs.Requests, &gs.Errors, &gs.Limited} + for i := range lat { + dst = append(dst, &lat[i]) + } + for i := range ttft { + dst = append(dst, &ttft[i]) + } + if err := rows.Scan(dst...); err != nil { + return nil, err + } + gs.Latency, gs.TTFT = quantiles(lat), quantiles(ttft) + out[gs.ID] = gs + } + return out, rows.Err() +} + +// quantiles reads p50, p95 and p99 from counts per LatencyBounds bucket. +func quantiles(counts []int64) Quantiles { + var q Quantiles + for _, c := range counts { + q.N += c + } + if q.N == 0 { + return q + } + at := func(p float64) (int64, bool) { + rank := p * float64(q.N) + var seen float64 + for i, c := range counts { + if c == 0 { + continue + } + if seen+float64(c) >= rank { + if i >= len(LatencyBounds) { + return LatencyBounds[len(LatencyBounds)-1], true + } + lo := int64(0) + if i > 0 { + lo = LatencyBounds[i-1] + } + hi := LatencyBounds[i] + frac := (rank - seen) / float64(c) + return lo + int64(frac*float64(hi-lo)), false + } + seen += float64(c) + } + return LatencyBounds[len(LatencyBounds)-1], true + } + var over bool + q.P50, _ = at(0.50) + q.P95, _ = at(0.95) + q.P99, over = at(0.99) + q.Over = over + return q +} diff --git a/internal/store/latency_test.go b/internal/store/latency_test.go new file mode 100644 index 0000000..1fabc66 --- /dev/null +++ b/internal/store/latency_test.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "testing" + "time" +) + +func TestQuantilesAreReadFromTheHistogram(t *testing.T) { + // 100 durations: 90 at <=100 ms, 8 in (1000, 2500], 2 above two minutes. + counts := make([]int64, len(LatencyBounds)+1) + counts[0], counts[4], counts[len(counts)-1] = 90, 8, 2 + q := quantiles(counts) + if q.N != 100 || q.P50 > 100 || q.P95 <= 1000 || q.P95 > 2500 || !q.Over { + t.Errorf("%+v: want p50 within 100 ms, p95 in (1000, 2500], p99 past the last edge", q) + } + if (quantiles(make([]int64, len(LatencyBounds)+1)) != Quantiles{}) { + t.Error("no durations read as some") + } +} + +func TestStatsSayHowEachGroupFaredAndHowLongItTook(t *testing.T) { + onBoth(t, func(t *testing.T, s *DB) { + ctx := context.Background() + at := slice() + var batch []Usage + for i := range 20 { + batch = append(batch, Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", + ServedModel: "big", Status: 200, LatencyMs: 4000, TTFTMs: int64(200 + i)}) + } + batch = append(batch, + Usage{At: at, KeyID: "k", UserID: "ayse", ProjectID: "p", Model: "big", ServedModel: "big", Status: 502, LatencyMs: 50}, + Usage{At: at, KeyID: "k2", UserID: "bora", ProjectID: "p", Model: "small", ServedModel: "small", Status: 429}, + ) + if err := s.WriteUsage(ctx, batch); err != nil { + t.Fatal(err) + } + byUser, err := s.StatsBy(ctx, GroupByUser, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + a, b := byUser["ayse"], byUser["bora"] + if a.Requests != 21 || a.Errors != 1 || a.Limited != 0 || b.Limited != 1 || b.Errors != 1 { + t.Errorf("outcomes: ayse %+v, bora %+v", a, b) + } + if a.TTFT.N != 20 || a.TTFT.P95 <= 100 || a.TTFT.P95 > 250 || a.Latency.P95 <= 2500 || a.Latency.P95 > 5000 { + t.Errorf("ayse's first token %+v and latency %+v", a.TTFT, a.Latency) + } + byModel, err := s.StatsBy(ctx, GroupByModel, at.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if byModel["big"].Requests != 21 || byModel["small"].Requests != 1 { + t.Errorf("by model: %+v", byModel) + } + series, err := s.UsageBucketsBy(ctx, GroupByModel, at.Add(-time.Hour), time.Hour) + if err != nil || len(series) != 2 { + t.Fatalf("usage over time by model: %v, %+v", err, series) + } + }) +} diff --git a/internal/store/migrations/postgres/0015_usage_rollup.sql b/internal/store/migrations/postgres/0015_usage_rollup.sql index 39abcf7..9121268 100644 --- a/internal/store/migrations/postgres/0015_usage_rollup.sql +++ b/internal/store/migrations/postgres/0015_usage_rollup.sql @@ -21,7 +21,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket BIGINT NOT NULL, @@ -32,6 +33,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -70,12 +72,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -92,6 +94,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -130,12 +133,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -152,6 +155,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests BIGINT NOT NULL DEFAULT 0, errors BIGINT NOT NULL DEFAULT 0, + limited BIGINT NOT NULL DEFAULT 0, prompt_tokens BIGINT NOT NULL DEFAULT 0, completion_tokens BIGINT NOT NULL DEFAULT 0, cost_micros BIGINT NOT NULL DEFAULT 0, @@ -190,12 +194,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/migrations/sqlite/0015_usage_rollup.sql b/internal/store/migrations/sqlite/0015_usage_rollup.sql index 17d9ae9..abe08f8 100644 --- a/internal/store/migrations/sqlite/0015_usage_rollup.sql +++ b/internal/store/migrations/sqlite/0015_usage_rollup.sql @@ -20,7 +20,8 @@ -- bucket (ms: <=100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000, 120000, -- and above), which is what a percentile per team or model is read from after -- the rows are gone; a mean cannot do that. ttft_* counts only requests whose --- first token was measured. +-- first token was measured. limited counts requests refused with 429 — a rate, +-- token or budget limit — which are recorded with no tokens and no cost. CREATE TABLE usage_5m ( bucket INTEGER NOT NULL, @@ -31,6 +32,7 @@ CREATE TABLE usage_5m ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -69,12 +71,12 @@ CREATE INDEX usage_5m_by_user ON usage_5m(user_id, bucket); CREATE INDEX usage_5m_by_key ON usage_5m(key_id, bucket); INSERT INTO usage_5m (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 300000) * 300000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -91,6 +93,7 @@ CREATE TABLE usage_1h ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -129,12 +132,12 @@ CREATE INDEX usage_1h_by_user ON usage_1h(user_id, bucket); CREATE INDEX usage_1h_by_key ON usage_1h(key_id, bucket); INSERT INTO usage_1h (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 3600000) * 3600000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), @@ -151,6 +154,7 @@ CREATE TABLE usage_1d ( served_model TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0, errors INTEGER NOT NULL DEFAULT 0, + limited INTEGER NOT NULL DEFAULT 0, prompt_tokens INTEGER NOT NULL DEFAULT 0, completion_tokens INTEGER NOT NULL DEFAULT 0, cost_micros INTEGER NOT NULL DEFAULT 0, @@ -189,12 +193,12 @@ CREATE INDEX usage_1d_by_user ON usage_1d(user_id, bucket); CREATE INDEX usage_1d_by_key ON usage_1d(key_id, bucket); INSERT INTO usage_1d (bucket, key_id, user_id, project_id, model, served_model, - requests, errors, prompt_tokens, completion_tokens, cost_micros, + requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, queue_ms_sum, ttft_ms_sum, ttft_n, last_ts, lat_0, lat_1, lat_2, lat_3, lat_4, lat_5, lat_6, lat_7, lat_8, lat_9, lat_10, ttft_0, ttft_1, ttft_2, ttft_3, ttft_4, ttft_5, ttft_6, ttft_7, ttft_8, ttft_9, ttft_10) SELECT (ts / 86400000) * 86400000, key_id, COALESCE(user_id, ''), project_id, model, served_model, - COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), + COUNT(*), SUM(CASE WHEN status >= 400 THEN 1 ELSE 0 END), SUM(CASE WHEN status = 429 THEN 1 ELSE 0 END), SUM(prompt_tokens), SUM(completion_tokens), SUM(cost_micros), SUM(latency_ms), SUM(queue_ms), SUM(ttft_ms), SUM(CASE WHEN ttft_ms > 0 THEN 1 ELSE 0 END), MAX(ts), SUM(CASE WHEN latency_ms <= 100 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 100 AND latency_ms <= 250 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 250 AND latency_ms <= 500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 500 AND latency_ms <= 1000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 1000 AND latency_ms <= 2500 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 2500 AND latency_ms <= 5000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 5000 AND latency_ms <= 10000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 10000 AND latency_ms <= 30000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 30000 AND latency_ms <= 60000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 60000 AND latency_ms <= 120000 THEN 1 ELSE 0 END), SUM(CASE WHEN latency_ms > 120000 THEN 1 ELSE 0 END), diff --git a/internal/store/rollup.go b/internal/store/rollup.go index 4333ee2..fa0e296 100644 --- a/internal/store/rollup.go +++ b/internal/store/rollup.go @@ -68,9 +68,10 @@ type rollupKey struct { // rollupRow is what a batch adds to one row. type rollupRow struct { - requests, errors, prompt, completion, cost int64 - latency, queue, ttft, ttftN, last int64 - lat, ttftH []int64 + requests, errors, limited, prompt int64 + completion, cost int64 + latency, queue, ttft, ttftN, last int64 + lat, ttftH []int64 } // rollupBatch sums a batch of records into the rows of one tier they add to. @@ -91,6 +92,9 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey if r.Status >= 400 { row.errors++ } + if r.Status == 429 { + row.limited++ + } row.prompt += r.PromptTokens row.completion += r.CompletionTokens row.cost += r.CostMicros @@ -111,7 +115,7 @@ func rollupBatch(t tier, records []Usage) (map[rollupKey]*rollupRow, []rollupKey var rollupColumns = func() []string { n := len(LatencyBounds) + 1 cols := []string{"bucket", "key_id", "user_id", "project_id", "model", "served_model", - "requests", "errors", "prompt_tokens", "completion_tokens", "cost_micros", + "requests", "errors", "limited", "prompt_tokens", "completion_tokens", "cost_micros", "latency_ms_sum", "queue_ms_sum", "ttft_ms_sum", "ttft_n", "last_ts"} for i := range n { cols = append(cols, fmt.Sprintf("lat_%d", i)) @@ -151,7 +155,7 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { for _, k := range order { r := rows[k] args := []any{k.bucket, k.keyID, k.userID, k.projectID, k.model, k.servedModel, - r.requests, r.errors, r.prompt, r.completion, r.cost, + r.requests, r.errors, r.limited, r.prompt, r.completion, r.cost, r.latency, r.queue, r.ttft, r.ttftN, r.last} for _, v := range r.lat { args = append(args, v) @@ -182,10 +186,11 @@ func writeRollup(ctx context.Context, tx conn, records []Usage) error { func usageSince(since time.Time) (string, []any) { from := millis(since) to5m, to1h, to1d := tier5m.ceil(since), tier1h.ceil(since), tier1d.ceil(since) - const cols = `key_id, user_id, project_id, model, served_model, requests, errors, + const cols = `key_id, user_id, project_id, model, served_model, requests, errors, limited, prompt_tokens, completion_tokens, cost_micros, latency_ms_sum, last_ts` return `(SELECT key_id, user_id, project_id, model, served_model, 1 AS requests, CASE WHEN status >= 400 THEN 1 ELSE 0 END AS errors, + CASE WHEN status = 429 THEN 1 ELSE 0 END AS limited, prompt_tokens, completion_tokens, cost_micros, latency_ms AS latency_ms_sum, ts AS last_ts FROM usage WHERE ts >= ? AND ts < ? UNION ALL SELECT ` + cols + ` FROM usage_5m WHERE bucket >= ? AND bucket < ? diff --git a/internal/store/usageseries.go b/internal/store/usageseries.go index 398adf7..d5e6eee 100644 --- a/internal/store/usageseries.go +++ b/internal/store/usageseries.go @@ -21,6 +21,9 @@ const ( GroupByTeam Grouping = "team" GroupByProject Grouping = "project" GroupByUser Grouping = "user" + // GroupByModel is by the model that answered, which differs from the one + // asked for when a fallback served it. + GroupByModel Grouping = "model" ) // Valid reports whether this is a grouping usage can be read by. @@ -32,7 +35,7 @@ func (g Grouping) Valid() bool { // Groupings lists them in the order a page should offer them: widest first, // then the one that is not a level at all. func Groupings() []Grouping { - return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser} + return []Grouping{GroupByOrg, GroupByTeam, GroupByProject, GroupByUser, GroupByModel} } // UsageSeries is one group's usage over time. @@ -87,8 +90,13 @@ var groupColumns = map[Grouping]struct{ id, name, joins string }{ // rather than hides. GroupByUser: {"u.user_id", "COALESCE(us.email, '')", ` LEFT JOIN users us ON us.id = NULLIF(u.user_id, '')`}, + GroupByModel: {servedModel, servedModel, ""}, } +// servedModel is what answered: the served model, or the one asked for on a +// row from before the two were told apart. +const servedModel = "CASE WHEN u.served_model != '' THEN u.served_model ELSE u.model END" + // UsageBucketsBy returns each group's usage over time. // // One query for every group rather than one per group: the page shows them diff --git a/internal/web/admin_people.go b/internal/web/admin_people.go index 7cfe35d..5eae074 100644 --- a/internal/web/admin_people.go +++ b/internal/web/admin_people.go @@ -66,10 +66,14 @@ type peopleDetail struct { BudgetsBind bool // Its use over the last week, and the totals beside it. - HasUsage bool - Requests, Tokens, Spend TimeSeries - Totals usageRow - Priced bool + HasUsage bool + Requests, Tokens, Spend TimeSeries + Totals usageRow + Priced bool + // Stats is how its requests ended and how long they took, over the + // same week; HasStats is false where the store does not say. + Stats store.GroupStats + HasStats bool UsageWindow, QuotaWindow string CSRF string @@ -248,6 +252,11 @@ func (p *AccountPages) usageDetail(ctx context.Context, d *peopleDetail) { } d.HasUsage = true d.Priced = p.usagePriced != nil && p.usagePriced.AnythingPriced() + if sr, ok := p.usageStore.(StatsReader); ok { + if stats, err := sr.StatsBy(ctx, g, time.Now().Add(-win.Span)); err == nil { + d.Stats, d.HasStats = stats[d.ID], true + } + } for _, s := range rows { if s.ID == d.ID { var table []usageRow diff --git a/internal/web/admin_people_test.go b/internal/web/admin_people_test.go index c9e1fed..4039f03 100644 --- a/internal/web/admin_people_test.go +++ b/internal/web/admin_people_test.go @@ -85,3 +85,27 @@ func TestSomethingNotInTheTreeOpensNoPanel(t *testing.T) { t.Error("a panel opened for an organization that does not exist") } } + +func TestHowRequestsFaredIsShownBesideWhatTheyCost(t *testing.T) { + stats := map[string]store.GroupStats{"org-1": {ID: "org-1", Requests: 200, Errors: 30, Limited: 25, + TTFT: store.Quantiles{P50: 180, P95: 900, P99: 2400, N: 150}, + Latency: store.Quantiles{P50: 3000, P95: 8000, P99: 120000, N: 200, Over: true}}} + u := &fakeScopeSeries{stats: stats, byGroup: map[store.Grouping][]store.UsageSeries{ + store.GroupByOrg: {{Grouping: store.GroupByOrg, ID: "org-1", Name: "Scania", Requests: 200, + Buckets: []store.Bucket{{Start: time.Now().Add(-time.Hour), Requests: 200}}}}, + }} + mux := peoplePage(t, oneTeam(), u) + + table := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/usage?by=org").Body.String()), " ") + for _, want := range []string{"15.0%", ">25<", "180 ms / 900 ms", "8.0 s"} { + if !strings.Contains(table, want) { + t.Errorf("the usage table lacks %q", want) + } + } + panel := strings.Join(strings.Fields(mirrorGet(t, mux, "/admin/users?sel=org:org-1").Body.String()), " ") + for _, want := range []string{"
Refused by a limit
", "p95 900 ms", "p99 > 120 s"} { + if !strings.Contains(panel, want) { + t.Errorf("the panel lacks %q", want) + } + } +} diff --git a/internal/web/admin_usage.go b/internal/web/admin_usage.go index 4f0d0b2..ebcf0a7 100644 --- a/internal/web/admin_usage.go +++ b/internal/web/admin_usage.go @@ -2,6 +2,7 @@ package web import ( "context" + "fmt" "net/http" "sort" "time" @@ -16,6 +17,35 @@ type UsageAdmin interface { since time.Time, width time.Duration) ([]store.UsageSeries, error) } +// StatsReader is how each group fared and how long its requests took; the +// store implements it. Optional: without it the usage table has no such +// columns. +type StatsReader interface { + StatsBy(ctx context.Context, g store.Grouping, since time.Time) (map[string]store.GroupStats, error) +} + +// pctile is a percentile from the usage histograms as a person reads it: +// "180 ms", "4.2 s", "> 120 s" past the last bucket, "—" with nothing to read. +func pctile(q store.Quantiles, which int) string { + if q.N == 0 { + return "—" + } + v := q.P50 + switch which { + case 95: + v = q.P95 + case 99: + v = q.P99 + if q.Over { + return fmt.Sprintf("> %d s", v/1000) + } + } + if v < 1000 { + return fmt.Sprintf("%d ms", v) + } + return fmt.Sprintf("%.1f s", float64(v)/1000) +} + // usageWindow is one length of history the page offers, and the slice it is cut // into. // @@ -54,11 +84,16 @@ type usageGroup struct { // usageRow is one group's totals, shown under the charts. type usageRow struct { + ID string Name string Requests int64 Tokens int64 Spend string Errors int64 + // Stats is how its requests ended and how long they took; HasStats is + // false where the store does not say. + Stats store.GroupStats + HasStats bool } type adminUsageData struct { @@ -86,6 +121,8 @@ type adminUsageData struct { // model on a GPU. GPUHeld TimeSeries GPUHours []gpuHoursRow + // HasStats adds outcomes and latency percentiles to the table. + HasStats bool Error string Version string @@ -141,6 +178,16 @@ func (p *AccountPages) buildUsage(r *http.Request, s UsageAdmin, priced Priced) } d.Requests, d.Tokens, d.Spend, d.Rows = usageCharts(rows) + if sr, ok := s.(StatsReader); ok { + stats, err := sr.StatsBy(r.Context(), by, time.Now().Add(-win.Span)) + if err != nil { + p.log.Warn("could not read latency and outcomes for the usage tab", "by", by, "err", err) + } + for i := range d.Rows { + d.Rows[i].Stats, d.Rows[i].HasStats = stats[d.Rows[i].ID], stats != nil + } + d.HasStats = stats != nil + } d.GPUHeld, d.GPUHours = p.gpuHours(r.Context(), win) return d } @@ -226,6 +273,8 @@ func groupLabel(g store.Grouping) string { return "project" case store.GroupByUser: return "person" + case store.GroupByModel: + return "model" default: return string(g) } @@ -246,6 +295,7 @@ func usageCharts(rows []store.UsageSeries) (req, tok, spend TimeSeries, table [] }), "", 4) for _, u := range rows { table = append(table, usageRow{ + ID: u.ID, Name: displayName(u), Requests: u.Requests, Tokens: u.Tokens(), diff --git a/internal/web/admin_usage_test.go b/internal/web/admin_usage_test.go index c157d15..72de07c 100644 --- a/internal/web/admin_usage_test.go +++ b/internal/web/admin_usage_test.go @@ -15,9 +15,11 @@ import ( type fakeScopeSeries struct { byGroup map[store.Grouping][]store.UsageSeries - asked []store.Grouping - width time.Duration - since time.Time + // stats, when set, is what StatsBy answers for every grouping. + stats map[string]store.GroupStats + asked []store.Grouping + width time.Duration + since time.Time } func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, @@ -28,6 +30,10 @@ func (f *fakeScopeSeries) UsageBucketsBy(_ context.Context, g store.Grouping, return f.byGroup[g], nil } +func (f *fakeScopeSeries) StatsBy(context.Context, store.Grouping, time.Time) (map[string]store.GroupStats, error) { + return f.stats, nil +} + func usagePage(t *testing.T, s *fakeScopeSeries, priced Priced) *http.ServeMux { t.Helper() a := fakeAccounts{user: store.User{ID: "u1", Email: "a@example.com", IsAdmin: true}} diff --git a/internal/web/templates/admin_people.html b/internal/web/templates/admin_people.html index bf53062..51357b6 100644 --- a/internal/web/templates/admin_people.html +++ b/internal/web/templates/admin_people.html @@ -140,6 +140,13 @@

Use, {{.UsageWindow}}

{{if .Totals.Name}}

{{compact .Totals.Requests}} requests · {{compact .Totals.Tokens}} tokens{{if .Priced}} · {{.Totals.Spend}} spent{{end}}{{if .Totals.Errors}} · {{.Totals.Errors}} errors{{end}}

+{{if and .HasStats .Stats.Requests}}{{with .Stats}} +
+
Error rate
{{pct1 .ErrorRate}}% +
Refused by a limit
{{if .Limited}}{{compact .Limited}}{{else}}none{{end}} +
First token
p50 {{pctile .TTFT 50}} · p95 {{pctile .TTFT 95}} · p99 {{pctile .TTFT 99}}
+
Latency
p50 {{pctile .Latency 50}} · p95 {{pctile .Latency 95}} · p99 {{pctile .Latency 99}}
+
{{end}}{{end}} {{template "tschart" dict "t" .Requests "title" "Requests" "note" ""}} {{template "tschart" dict "t" .Tokens "title" "Tokens" "note" ""}} {{if .Priced}}{{template "tschart" dict "t" .Spend "title" "Spend" "note" ""}}{{end}} diff --git a/internal/web/templates/admin_usage.html b/internal/web/templates/admin_usage.html index 456960b..ad6d637 100644 --- a/internal/web/templates/admin_usage.html +++ b/internal/web/templates/admin_usage.html @@ -40,7 +40,11 @@ {{if .Rows}} - + + {{if .HasStats}} + + + {{end}} {{range .Rows}} @@ -48,6 +52,11 @@ + {{if $.HasStats}}{{with .Stats}} + + + + {{end}}{{end}} {{end}}
GroupRequestsTokensSpendErrors
GroupRequestsTokensSpendErrorsError rateLimitedFirst token p50 / p95Latency p95
{{.Name}}{{compact .Tokens}} {{.Spend}} {{if .Errors}}{{.Errors}}{{else}}—{{end}}{{if .Requests}}{{pct1 .ErrorRate}}%{{else}}—{{end}}{{if .Limited}}{{compact .Limited}}{{else}}—{{end}}{{pctile .TTFT 50}} / {{pctile .TTFT 95}}{{pctile .Latency 95}}
diff --git a/internal/web/web.go b/internal/web/web.go index 8f44fe9..e870014 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -22,6 +22,7 @@ import ( "io/fs" "net/http" "slices" + "strconv" "strings" "sync" "time" @@ -90,6 +91,8 @@ var funcs = template.FuncMap{ return strings.Join(v, ", ") }, "latency": humanLatency, + "pctile": pctile, + "pct1": func(v float64) string { return strconv.FormatFloat(v, 'f', 1, 64) }, "duration": func(sec int64) string { return time.Duration(sec * int64(time.Second)).Round(time.Second).String() }, From ce006a6e6f63bc1b45da87371d6f2abeeeaa5584 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:55:24 +0200 Subject: [PATCH 3/4] Publish per-tenant request and budget series, and alert on them Requests by outcome, tokens and first-token time per tenant, at the level router.tenant_label names (team by default), and every budget's share spent this period. Rules alert on budgets at 80% and 100%, tenants hitting limits, failing, spiking past their weekly average, or waiting on first tokens. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 17 +++ deploy/monitoring/llmgw-rules.yml | 80 +++++++++++++ docs/reference.md | 11 ++ go.mod | 1 + internal/config/config.go | 13 +++ internal/router/handlers.go | 4 + internal/router/metrics.go | 7 +- internal/router/pricing_test.go | 38 +++++++ internal/router/router.go | 25 ++++ internal/router/tenantmetrics.go | 182 ++++++++++++++++++++++++++++++ 10 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 internal/router/tenantmetrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f39601e..9897e48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,23 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration sums — but `ctl request` finds nothing older. Unset keeps every record, as before. +### Alerts per tenant and per budget + +- **`llmgw_tenant_requests_total{tenant,outcome}`**, `llmgw_tenant_tokens_total` + and `llmgw_tenant_ttft_seconds`: requests by outcome (ok, error, limited), + tokens and time to first token per tenant. The tenant is the level + `router.tenant_label` names — `team` by default, or `org`, `project`, or + `none` — named by its path, "acme/platform". One level rather than every + project, so a cluster of thousands of projects does not become a Prometheus + of millions of series. +- **`llmgw_budget_spent_ratio{scope,id,name,period}`**: this period's spend + over the budget for every organization, team, project and person given one, + refreshed every minute. +- **Alerts** in `deploy/monitoring/llmgw-rules.yml`: `BudgetMostlySpent` (80%), + `BudgetSpent`, `TenantHittingLimits` (a tenth of requests refused), + `TenantErrors`, `TenantTrafficSpike` (three times the weekly average) and + `TenantSlowFirstToken`. + ### How each team's requests fared, not only what they cost - **Percentiles from the sums.** First token and latency, p50, p95 and p99, diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index e877019..2ecf061 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -350,3 +350,83 @@ groups: Working as designed when the cluster is full: the lowest band is held first. Worth knowing about because the callers being refused do not know why. + + # Per tenant: the level router.tenant_label names (a team by default), and + # every organization, team, project and person given a budget. + - name: llmgw-tenants + rules: + - alert: BudgetMostlySpent + expr: llmgw_budget_spent_ratio >= 0.8 and llmgw_budget_spent_ratio < 1 + for: 5m + labels: { severity: warning } + annotations: + summary: "{{ $labels.scope }} {{ $labels.name }} has spent {{ $value | humanizePercentage }} of its {{ $labels.period }}ly budget" + description: >- + Its requests are refused once it reaches all of it, until the + period turns. Raise it on the people page, or tell them now + rather than when they are refused. + + - alert: BudgetSpent + expr: llmgw_budget_spent_ratio >= 1 + for: 1m + labels: { severity: critical } + annotations: + summary: "{{ $labels.scope }} {{ $labels.name }} has spent its {{ $labels.period }}ly budget" + description: >- + Every request under it is refused with 429 until the period + turns, unless a cheaper model is named as its fallback. + + - alert: TenantHittingLimits + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="limited"}[10m])) + / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.1 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests are refused by a limit" + description: >- + A rate, token or budget limit is turning their traffic away. Either + the limit is too low for what they now do, or something of theirs + is looping. + + - alert: TenantErrors + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="error"}[10m])) + / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05 + and sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests fail" + description: >- + Not counting refusals by a limit. If every tenant fails alike it + is a model (ModelFailing); one alone is usually what they send — + prompts past the context, a model they may not use. + + - alert: TenantTrafficSpike + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) + > 3 * sum by (tenant) (rate(llmgw_tenant_requests_total[7d])) + and sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) > 0.2 + for: 15m + labels: { severity: info } + annotations: + summary: "{{ $labels.tenant }} is sending three times its weekly average" + description: >- + A new workload, a batch job, or a client retrying in a loop. Only + for a tenant sending more than one request every five seconds, so + a quiet team starting work in the morning does not page anybody. + + - alert: TenantSlowFirstToken + expr: >- + histogram_quantile(0.95, + sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket[15m]))) > 10 + and sum by (tenant) (rate(llmgw_tenant_ttft_seconds_count[15m])) > 0.02 + for: 15m + labels: { severity: warning } + annotations: + summary: "{{ $labels.tenant }} waits {{ $value | humanizeDuration }} for a first token at p95" + description: >- + What their people feel. Compare with SlowFirstToken per model: if + only this tenant is slow, it is the models it uses or its queue + band, not the cluster. diff --git a/docs/reference.md b/docs/reference.md index aa1e353..46674a3 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -162,6 +162,7 @@ rather than repeated. | `prefix` | `Prefix` | [Router](#router) | | `session_header` | `string` | [Router](#router) | | `inject_usage` | `bool` | [Router](#router) | +| `tenant_label` | `string` | [Router](#router) | | `max_body_bytes` | `int64` | [Router](#router) | | `timeouts` | `Timeouts` | [Router](#router) | | `scoring` | `Scoring` | [Router](#router) | @@ -1254,6 +1255,16 @@ SessionHeader carries the session id for affinity (plan §7.5 tier 0). InjectUsage asks engines for usage even when the client did not, so token accounting works regardless of client behaviour. +### `tenant_label` + +`string` + +TenantLabel is the level of the tree the per-tenant request series are +labelled by — llmgw_tenant_requests_total and its siblings: "org", +"team" or "project", or "none" for no such series. A label per project +on a cluster of thousands is more series than Prometheus wants, and a +team is what an alert is usually about; "team" is the default. + ### `max_body_bytes` `int64` diff --git a/go.mod b/go.mod index 5c59dba..4a3c016 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 5cd0a9e..a01980b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -371,6 +371,13 @@ type Router struct { // accounting works regardless of client behaviour. InjectUsage bool `koanf:"inject_usage"` + // TenantLabel is the level of the tree the per-tenant request series are + // labelled by — llmgw_tenant_requests_total and its siblings: "org", + // "team" or "project", or "none" for no such series. A label per project + // on a cluster of thousands is more series than Prometheus wants, and a + // team is what an alert is usually about; "team" is the default. + TenantLabel string `koanf:"tenant_label"` + // MaxBodyBytes bounds the request body the router will read. MaxBodyBytes int64 `koanf:"max_body_bytes"` @@ -491,6 +498,7 @@ func Default() Server { Cooldown: 15 * time.Second, FailureThreshold: 3, DefaultProfile: "chat", + TenantLabel: "team", SessionHeader: "x-session-id", InjectUsage: true, MaxBodyBytes: 32 << 20, @@ -581,6 +589,11 @@ func (s Server) Validate() error { if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") } + switch s.Router.TenantLabel { + case "", "none", "org", "team", "project": + default: + return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel) + } if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour { return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention) } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index e48a60c..5345a51 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -261,6 +261,7 @@ type parsedRequest struct { KeyID string UserID string ProjectID string + TeamID, OrgID string Tenant string ProfileOverride string PromptPrefix string @@ -519,6 +520,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too // the truth than charging nothing. u.PromptTokens = int64(req.PromptTokens) } + r.m.tenant.observe(req, status, u.PromptTokens, u.CompletionTokens, res.outcome.TTFT) r.acct.Record(req.reservation, u) } @@ -537,6 +539,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.m.tenant.observe(req, http.StatusTooManyRequests, 0, 0, 0) r.acct.RecordRefused(store.Usage{ At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests, @@ -584,6 +587,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { p.KeyID = principal.KeyID p.UserID = principal.UserID p.ProjectID = principal.ProjectID + p.TeamID, p.OrgID = principal.TeamID, principal.OrgID // Tenant fairness within a band is per key rather than per session, so // one key cannot take a whole band by opening many sessions. p.Tenant = principal.KeyID diff --git a/internal/router/metrics.go b/internal/router/metrics.go index 79ef1e1..ddc5d38 100644 --- a/internal/router/metrics.go +++ b/internal/router/metrics.go @@ -29,6 +29,8 @@ type metrics struct { admissionRejected *prometheus.CounterVec prefixDecisions *prometheus.CounterVec + + tenant *tenantMetrics } func newMetrics() *metrics { @@ -86,16 +88,17 @@ func newMetrics() *metrics { Name: "llmgw_admission_rejected_total", Help: "Requests refused by flow control, by reason.", }, []string{"band", "reason"}), + tenant: newTenantMetrics(), } } // collectors returns everything to register, so registration lives in one place. func (m *metrics) collectors() []prometheus.Collector { - return []prometheus.Collector{ + return append([]prometheus.Collector{ m.requests, m.duration, m.ttft, m.tokens, m.scorerValue, m.filtered, m.considered, m.admissionWait, m.admissionRejected, m.prefixDecisions, - } + }, m.tenant.collectors()...) } // observePrefix records which branch a prefix decision took. diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 408f681..e6f836f 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -224,3 +226,39 @@ func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) { } t.Fatal("the refused request was never counted against its project") } + +func TestTenantsAreCountedAndTheirBudgetsGauged(t *testing.T) { + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + ctx := t.Context() + orgs, _ := rt.store.AllOrgs(ctx) + teams, _ := rt.store.AllTeams(ctx) + budget := int64(1_000_000) + if err := rt.store.PutLimits(ctx, store.Limits{Scope: store.ScopeTeam, ScopeID: teams[0].ID, + Budget: &budget, BudgetPeriod: store.PeriodMonth}); err != nil { + t.Fatal(err) + } + if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil { + t.Fatal(err) + } + rec := post(t, rt.Handler(), map[string]any{"model": "q14", "messages": []any{ + map[string]any{"role": "user", "content": "hello"}}}, + map[string]string{"Authorization": "Bearer " + secret}) + if rec.Code != 200 { + t.Fatalf("status %d", rec.Code) + } + tenant := orgs[0].Name + "/" + teams[0].Name + if got := testutil.ToFloat64(rt.m.tenant.requests.WithLabelValues(tenant, "ok")); got != 1 { + t.Errorf("%s has %v ok requests, want 1", tenant, got) + } + + u := waitForUsage(t, rt) + if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil { + t.Fatal(err) + } + want := float64(u.CostMicros) / float64(budget) + got := testutil.ToFloat64(rt.m.tenant.budget.WithLabelValues("team", teams[0].ID, tenant, "month")) + if want == 0 || got != want { + t.Errorf("the team's budget gauge reads %v, want %v", got, want) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 3b62a57..e5f2954 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -191,6 +191,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { // the decision metric is the whole difference between tuning this and // guessing at it. metrics := newMetrics() + metrics.tenant.level = cfg.Router.TenantLabel var prefixScorer *scheduler.PrefixScorer if cfg.Router.Prefix.Enabled { prefixScorer = scheduler.NewPrefixScorer( @@ -418,6 +419,30 @@ func (r *Router) Start(ctx context.Context) { if r.store != nil && r.usageRetention > 0 { go r.pruneUsage(ctx, r.usageRetention) } + if r.store != nil { + go r.refreshTenants(ctx) + } +} + +// refreshTenants keeps the tenant names the request series are labelled with, +// and the budget gauges, a minute fresh. +func (r *Router) refreshTenants(ctx context.Context) { + refresh := func() { + if err := r.m.tenant.refresh(ctx, r.store, time.Now()); err != nil && ctx.Err() == nil { + r.log.Warn("could not refresh tenant names and budget gauges", "err", err) + } + } + refresh() + t := time.NewTicker(time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + refresh() + } + } } // pruneUsage deletes request records older than the retention, hourly and diff --git a/internal/router/tenantmetrics.go b/internal/router/tenantmetrics.go new file mode 100644 index 0000000..7202b7b --- /dev/null +++ b/internal/router/tenantmetrics.go @@ -0,0 +1,182 @@ +package router + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// tenantMetrics are the router's series per tenant: requests by how they +// ended, tokens, time to first token, and how much of each budget is spent. +// +// The request series carry one level of the tree, chosen by +// router.tenant_label, rather than every key or project: a series per project +// on a cluster of thousands is a Prometheus that falls over, and alerting on a +// team is what an operator does. Budgets are a series per limit that has one, +// which is as many as somebody wrote. +type tenantMetrics struct { + level string + + requests *prometheus.CounterVec + tokens *prometheus.CounterVec + ttft *prometheus.HistogramVec + budget *prometheus.GaugeVec + + mu sync.RWMutex + names map[string]string // "team:" -> "acme/platform" +} + +func newTenantMetrics() *tenantMetrics { + return &tenantMetrics{ + level: "team", + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_tenant_requests_total", + Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.", + }, []string{"tenant", "outcome"}), + tokens: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_tenant_tokens_total", + Help: "Tokens by tenant and direction.", + }, []string{"tenant", "direction"}), + ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "llmgw_tenant_ttft_seconds", + Help: "Time to first token by tenant, for streamed requests.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, + }, []string{"tenant"}), + budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "llmgw_budget_spent_ratio", + Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.", + }, []string{"scope", "id", "name", "period"}), + names: map[string]string{}, + } +} + +func (t *tenantMetrics) collectors() []prometheus.Collector { + return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget} +} + +// tenantOf is the label a request is counted under: the name of its +// organization, team or project at the configured level, "personal" for use +// outside any project, or "" when tenants are not labelled at all. +func (t *tenantMetrics) tenantOf(req *parsedRequest) string { + var scope, id string + switch t.level { + case "org": + scope, id = "org", req.OrgID + case "team": + scope, id = "team", req.TeamID + case "project": + scope, id = "project", req.ProjectID + default: + return "" + } + if id == "" { + return "personal" + } + t.mu.RLock() + name, ok := t.names[scope+":"+id] + t.mu.RUnlock() + if !ok { + // Not yet in the cache — made since the last refresh. The id is + // stable, which is what a label has to be; the name follows. + return id + } + return name +} + +// observe counts one finished or refused request. +func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) { + tenant := t.tenantOf(req) + if tenant == "" { + return + } + outcome := "ok" + switch { + case status == 429: + outcome = "limited" + case status >= 400: + outcome = "error" + } + t.requests.WithLabelValues(tenant, outcome).Inc() + if prompt > 0 { + t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt)) + } + if completion > 0 { + t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion)) + } + if ttft > 0 { + t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds()) + } +} + +// tenantStore is what the refresh reads. +type tenantStore interface { + Tree(ctx context.Context) ([]storepkg.TreeOrg, error) + AllLimits(ctx context.Context) (map[string]storepkg.Limits, error) + SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error) + Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error) +} + +// refresh reads the tree's names and every budget's spend. A minute old at +// most, which is plenty for an alert that waits five before firing. +func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error { + tree, err := s.Tree(ctx) + if err != nil { + return err + } + names := map[string]string{} + for _, o := range tree { + names["org:"+o.ID] = o.Name + for _, tm := range o.Teams { + names["team:"+tm.ID] = o.Name + "/" + tm.Name + for _, p := range tm.Projects { + names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name + } + } + } + t.mu.Lock() + t.names = names + t.mu.Unlock() + + limits, err := s.AllLimits(ctx) + if err != nil { + return err + } + emails := map[string]string{} + if users, err := s.Users(ctx, ""); err == nil { + for _, u := range users { + emails[u.ID] = u.Email + } + } + // Rebuilt whole, so a budget removed since the last refresh stops being + // published rather than staying at its last value for ever. + t.budget.Reset() + for _, l := range limits { + if l.Budget == nil || *l.Budget <= 0 { + continue + } + name := "" + switch l.Scope { + case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject: + name = names[string(l.Scope)+":"+l.ScopeID] + case storepkg.ScopeUser: + name = emails[l.ScopeID] + default: + // Per key and per member are too many series and too little + // to act on; their pools are bounded by the levels above. + continue + } + spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now)) + if err != nil { + return err + } + t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)). + Set(float64(spent) / float64(*l.Budget)) + } + return nil +} From 17cc2e379bb265a374f1eaed69fd3514be674a60 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:58:39 +0200 Subject: [PATCH 4/4] Add a Grafana dashboard per tenant Requests and tokens by tenant, the shares failing and refused by a limit, first-token p95, budgets spent this period and the day's heaviest users, filtered by a tenant chooser. Installed beside the cluster dashboard. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 4 + deploy/monitoring/README.md | 2 + deploy/monitoring/grafana/llmgw-tenants.json | 650 +++++++++++++++++++ 3 files changed, 656 insertions(+) create mode 100644 deploy/monitoring/grafana/llmgw-tenants.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 9897e48..5a3a973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -113,6 +113,10 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration `BudgetSpent`, `TenantHittingLimits` (a tenth of requests refused), `TenantErrors`, `TenantTrafficSpike` (three times the weekly average) and `TenantSlowFirstToken`. +- **A Grafana dashboard per tenant**, `deploy/monitoring/grafana/llmgw-tenants.json` + (uid `llmgw-tenants`): requests and tokens by tenant, the share failing and + refused by a limit, first-token p95, every budget's share spent, and who used + most in the last day — with a tenant chooser at the top. ### How each team's requests fared, not only what they cost diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md index 2483daa..b81a5f7 100644 --- a/deploy/monitoring/README.md +++ b/deploy/monitoring/README.md @@ -66,6 +66,8 @@ install -m 0644 grafana/datasource.yaml /etc/grafana/provisioning/datasources/ install -m 0644 grafana/dashboards.yaml /etc/grafana/provisioning/dashboards/llmgw.yaml install -m 0644 -o grafana -g grafana grafana/llmgw-overview.json \ /var/lib/grafana/dashboards/llmgw.json +install -m 0644 -o grafana -g grafana grafana/llmgw-tenants.json \ + /var/lib/grafana/dashboards/llmgw-tenants.json systemctl restart grafana-server ``` diff --git a/deploy/monitoring/grafana/llmgw-tenants.json b/deploy/monitoring/grafana/llmgw-tenants.json new file mode 100644 index 0000000..f9b39c5 --- /dev/null +++ b/deploy/monitoring/grafana/llmgw-tenants.json @@ -0,0 +1,650 @@ +{ + "uid": "llmgw-tenants", + "title": "llmgw \u2014 tenants", + "schemaVersion": 39, + "version": 1, + "editable": true, + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "1m", + "tags": [ + "llmgw" + ], + "description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.", + "templating": { + "list": [ + { + "name": "tenant", + "label": "Tenant", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "query": { + "query": "label_values(llmgw_tenant_requests_total, tenant)", + "refId": "tenant" + }, + "definition": "label_values(llmgw_tenant_requests_total, tenant)", + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Requests a second", + "type": "stat", + "description": "Across the tenants chosen above.", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 2, + "title": "Failing", + "type": "stat", + "description": "Requests that failed, not counting refusals by a limit.", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.02 + }, + { + "color": "red", + "value": 0.05 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 3, + "title": "Refused by a limit", + "type": "stat", + "description": "Rate, token and budget limits; 429 to the caller.", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.02 + }, + { + "color": "red", + "value": 0.1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 4, + "title": "First token, p95", + "type": "stat", + "description": "What people wait for before anything appears.", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))", + "legendFormat": "", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "textMode": "auto", + "colorMode": "value" + } + }, + { + "id": 5, + "title": "Requests by tenant", + "type": "timeseries", + "description": "Stacked, so the top edge is the whole of it.", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 30 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "title": "Tokens by tenant", + "type": "timeseries", + "description": "Prompt and completion together: what was consumed.", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 30 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "title": "Refused by a limit, by tenant", + "type": "timeseries", + "description": "The share of each tenant's requests a limit turned away.", + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "title": "Failing, by tenant", + "type": "timeseries", + "description": "One tenant alone failing is usually what it sends; all of them together is a model.", + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "title": "First token p95, by tenant", + "type": "timeseries", + "description": "", + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))", + "legendFormat": "{{tenant}}", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "title": "Budgets spent this period", + "type": "bargauge", + "description": "Every organization, team, project and person with a budget. Requests are refused at 100%.", + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(llmgw_budget_spent_ratio)", + "legendFormat": "{{scope}} {{name}} ({{period}})", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "showUnfilled": true + } + }, + { + "id": 11, + "title": "Who used most, last 24 hours", + "type": "table", + "description": "Spend in money is on the people page's usage tab, which reads it from the store.", + "gridPos": { + "x": 0, + "y": 29, + "w": 24, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))", + "legendFormat": "requests", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + }, + { + "refId": "B", + "expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))", + "legendFormat": "tokens", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + }, + { + "refId": "C", + "expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))", + "legendFormat": "limited", + "datasource": { + "type": "prometheus", + "uid": "llmgw-prom" + }, + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "Value #A": "Requests", + "Value #B": "Tokens", + "Value #C": "Refused by a limit" + } + } + } + ] + } + ] +} \ No newline at end of file