From ce006a6e6f63bc1b45da87371d6f2abeeeaa5584 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sat, 3 Oct 2026 19:55:24 +0200 Subject: [PATCH] Publish per-tenant request and budget series, and alert on them Requests by outcome, tokens and first-token time per tenant, at the level router.tenant_label names (team by default), and every budget's share spent this period. Rules alert on budgets at 80% and 100%, tenants hitting limits, failing, spiking past their weekly average, or waiting on first tokens. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 17 +++ deploy/monitoring/llmgw-rules.yml | 80 +++++++++++++ docs/reference.md | 11 ++ go.mod | 1 + internal/config/config.go | 13 +++ internal/router/handlers.go | 4 + internal/router/metrics.go | 7 +- internal/router/pricing_test.go | 38 +++++++ internal/router/router.go | 25 ++++ internal/router/tenantmetrics.go | 182 ++++++++++++++++++++++++++++++ 10 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 internal/router/tenantmetrics.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f39601e..9897e48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -97,6 +97,23 @@ key, person, project and model (`usage_5m`, `usage_1h`, `usage_1d`; migration sums — but `ctl request` finds nothing older. Unset keeps every record, as before. +### Alerts per tenant and per budget + +- **`llmgw_tenant_requests_total{tenant,outcome}`**, `llmgw_tenant_tokens_total` + and `llmgw_tenant_ttft_seconds`: requests by outcome (ok, error, limited), + tokens and time to first token per tenant. The tenant is the level + `router.tenant_label` names — `team` by default, or `org`, `project`, or + `none` — named by its path, "acme/platform". One level rather than every + project, so a cluster of thousands of projects does not become a Prometheus + of millions of series. +- **`llmgw_budget_spent_ratio{scope,id,name,period}`**: this period's spend + over the budget for every organization, team, project and person given one, + refreshed every minute. +- **Alerts** in `deploy/monitoring/llmgw-rules.yml`: `BudgetMostlySpent` (80%), + `BudgetSpent`, `TenantHittingLimits` (a tenth of requests refused), + `TenantErrors`, `TenantTrafficSpike` (three times the weekly average) and + `TenantSlowFirstToken`. + ### How each team's requests fared, not only what they cost - **Percentiles from the sums.** First token and latency, p50, p95 and p99, diff --git a/deploy/monitoring/llmgw-rules.yml b/deploy/monitoring/llmgw-rules.yml index e877019..2ecf061 100644 --- a/deploy/monitoring/llmgw-rules.yml +++ b/deploy/monitoring/llmgw-rules.yml @@ -350,3 +350,83 @@ groups: Working as designed when the cluster is full: the lowest band is held first. Worth knowing about because the callers being refused do not know why. + + # Per tenant: the level router.tenant_label names (a team by default), and + # every organization, team, project and person given a budget. + - name: llmgw-tenants + rules: + - alert: BudgetMostlySpent + expr: llmgw_budget_spent_ratio >= 0.8 and llmgw_budget_spent_ratio < 1 + for: 5m + labels: { severity: warning } + annotations: + summary: "{{ $labels.scope }} {{ $labels.name }} has spent {{ $value | humanizePercentage }} of its {{ $labels.period }}ly budget" + description: >- + Its requests are refused once it reaches all of it, until the + period turns. Raise it on the people page, or tell them now + rather than when they are refused. + + - alert: BudgetSpent + expr: llmgw_budget_spent_ratio >= 1 + for: 1m + labels: { severity: critical } + annotations: + summary: "{{ $labels.scope }} {{ $labels.name }} has spent its {{ $labels.period }}ly budget" + description: >- + Every request under it is refused with 429 until the period + turns, unless a cheaper model is named as its fallback. + + - alert: TenantHittingLimits + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="limited"}[10m])) + / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.1 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests are refused by a limit" + description: >- + A rate, token or budget limit is turning their traffic away. Either + the limit is too low for what they now do, or something of theirs + is looping. + + - alert: TenantErrors + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="error"}[10m])) + / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05 + and sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05 + for: 10m + labels: { severity: warning } + annotations: + summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests fail" + description: >- + Not counting refusals by a limit. If every tenant fails alike it + is a model (ModelFailing); one alone is usually what they send — + prompts past the context, a model they may not use. + + - alert: TenantTrafficSpike + expr: >- + sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) + > 3 * sum by (tenant) (rate(llmgw_tenant_requests_total[7d])) + and sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) > 0.2 + for: 15m + labels: { severity: info } + annotations: + summary: "{{ $labels.tenant }} is sending three times its weekly average" + description: >- + A new workload, a batch job, or a client retrying in a loop. Only + for a tenant sending more than one request every five seconds, so + a quiet team starting work in the morning does not page anybody. + + - alert: TenantSlowFirstToken + expr: >- + histogram_quantile(0.95, + sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket[15m]))) > 10 + and sum by (tenant) (rate(llmgw_tenant_ttft_seconds_count[15m])) > 0.02 + for: 15m + labels: { severity: warning } + annotations: + summary: "{{ $labels.tenant }} waits {{ $value | humanizeDuration }} for a first token at p95" + description: >- + What their people feel. Compare with SlowFirstToken per model: if + only this tenant is slow, it is the models it uses or its queue + band, not the cluster. diff --git a/docs/reference.md b/docs/reference.md index aa1e353..46674a3 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -162,6 +162,7 @@ rather than repeated. | `prefix` | `Prefix` | [Router](#router) | | `session_header` | `string` | [Router](#router) | | `inject_usage` | `bool` | [Router](#router) | +| `tenant_label` | `string` | [Router](#router) | | `max_body_bytes` | `int64` | [Router](#router) | | `timeouts` | `Timeouts` | [Router](#router) | | `scoring` | `Scoring` | [Router](#router) | @@ -1254,6 +1255,16 @@ SessionHeader carries the session id for affinity (plan §7.5 tier 0). InjectUsage asks engines for usage even when the client did not, so token accounting works regardless of client behaviour. +### `tenant_label` + +`string` + +TenantLabel is the level of the tree the per-tenant request series are +labelled by — llmgw_tenant_requests_total and its siblings: "org", +"team" or "project", or "none" for no such series. A label per project +on a cluster of thousands is more series than Prometheus wants, and a +team is what an alert is usually about; "team" is the default. + ### `max_body_bytes` `int64` diff --git a/go.mod b/go.mod index 5c59dba..4a3c016 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 5cd0a9e..a01980b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -371,6 +371,13 @@ type Router struct { // accounting works regardless of client behaviour. InjectUsage bool `koanf:"inject_usage"` + // TenantLabel is the level of the tree the per-tenant request series are + // labelled by — llmgw_tenant_requests_total and its siblings: "org", + // "team" or "project", or "none" for no such series. A label per project + // on a cluster of thousands is more series than Prometheus wants, and a + // team is what an alert is usually about; "team" is the default. + TenantLabel string `koanf:"tenant_label"` + // MaxBodyBytes bounds the request body the router will read. MaxBodyBytes int64 `koanf:"max_body_bytes"` @@ -491,6 +498,7 @@ func Default() Server { Cooldown: 15 * time.Second, FailureThreshold: 3, DefaultProfile: "chat", + TenantLabel: "team", SessionHeader: "x-session-id", InjectUsage: true, MaxBodyBytes: 32 << 20, @@ -581,6 +589,11 @@ func (s Server) Validate() error { if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") } + switch s.Router.TenantLabel { + case "", "none", "org", "team", "project": + default: + return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel) + } if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour { return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention) } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index e48a60c..5345a51 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -261,6 +261,7 @@ type parsedRequest struct { KeyID string UserID string ProjectID string + TeamID, OrgID string Tenant string ProfileOverride string PromptPrefix string @@ -519,6 +520,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too // the truth than charging nothing. u.PromptTokens = int64(req.PromptTokens) } + r.m.tenant.observe(req, status, u.PromptTokens, u.CompletionTokens, res.outcome.TTFT) r.acct.Record(req.reservation, u) } @@ -537,6 +539,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.m.tenant.observe(req, http.StatusTooManyRequests, 0, 0, 0) r.acct.RecordRefused(store.Usage{ At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID, Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests, @@ -584,6 +587,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { p.KeyID = principal.KeyID p.UserID = principal.UserID p.ProjectID = principal.ProjectID + p.TeamID, p.OrgID = principal.TeamID, principal.OrgID // Tenant fairness within a band is per key rather than per session, so // one key cannot take a whole band by opening many sessions. p.Tenant = principal.KeyID diff --git a/internal/router/metrics.go b/internal/router/metrics.go index 79ef1e1..ddc5d38 100644 --- a/internal/router/metrics.go +++ b/internal/router/metrics.go @@ -29,6 +29,8 @@ type metrics struct { admissionRejected *prometheus.CounterVec prefixDecisions *prometheus.CounterVec + + tenant *tenantMetrics } func newMetrics() *metrics { @@ -86,16 +88,17 @@ func newMetrics() *metrics { Name: "llmgw_admission_rejected_total", Help: "Requests refused by flow control, by reason.", }, []string{"band", "reason"}), + tenant: newTenantMetrics(), } } // collectors returns everything to register, so registration lives in one place. func (m *metrics) collectors() []prometheus.Collector { - return []prometheus.Collector{ + return append([]prometheus.Collector{ m.requests, m.duration, m.ttft, m.tokens, m.scorerValue, m.filtered, m.considered, m.admissionWait, m.admissionRejected, m.prefixDecisions, - } + }, m.tenant.collectors()...) } // observePrefix records which branch a prefix decision took. diff --git a/internal/router/pricing_test.go b/internal/router/pricing_test.go index 408f681..e6f836f 100644 --- a/internal/router/pricing_test.go +++ b/internal/router/pricing_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -224,3 +226,39 @@ func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) { } t.Fatal("the refused request was never counted against its project") } + +func TestTenantsAreCountedAndTheirBudgetsGauged(t *testing.T) { + e := newFakeEngine(t, "qwen2.5-14b-instruct") + rt, secret := pricedRouter(t, e.srv.URL) + ctx := t.Context() + orgs, _ := rt.store.AllOrgs(ctx) + teams, _ := rt.store.AllTeams(ctx) + budget := int64(1_000_000) + if err := rt.store.PutLimits(ctx, store.Limits{Scope: store.ScopeTeam, ScopeID: teams[0].ID, + Budget: &budget, BudgetPeriod: store.PeriodMonth}); err != nil { + t.Fatal(err) + } + if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil { + t.Fatal(err) + } + rec := post(t, rt.Handler(), map[string]any{"model": "q14", "messages": []any{ + map[string]any{"role": "user", "content": "hello"}}}, + map[string]string{"Authorization": "Bearer " + secret}) + if rec.Code != 200 { + t.Fatalf("status %d", rec.Code) + } + tenant := orgs[0].Name + "/" + teams[0].Name + if got := testutil.ToFloat64(rt.m.tenant.requests.WithLabelValues(tenant, "ok")); got != 1 { + t.Errorf("%s has %v ok requests, want 1", tenant, got) + } + + u := waitForUsage(t, rt) + if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil { + t.Fatal(err) + } + want := float64(u.CostMicros) / float64(budget) + got := testutil.ToFloat64(rt.m.tenant.budget.WithLabelValues("team", teams[0].ID, tenant, "month")) + if want == 0 || got != want { + t.Errorf("the team's budget gauge reads %v, want %v", got, want) + } +} diff --git a/internal/router/router.go b/internal/router/router.go index 3b62a57..e5f2954 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -191,6 +191,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { // the decision metric is the whole difference between tuning this and // guessing at it. metrics := newMetrics() + metrics.tenant.level = cfg.Router.TenantLabel var prefixScorer *scheduler.PrefixScorer if cfg.Router.Prefix.Enabled { prefixScorer = scheduler.NewPrefixScorer( @@ -418,6 +419,30 @@ func (r *Router) Start(ctx context.Context) { if r.store != nil && r.usageRetention > 0 { go r.pruneUsage(ctx, r.usageRetention) } + if r.store != nil { + go r.refreshTenants(ctx) + } +} + +// refreshTenants keeps the tenant names the request series are labelled with, +// and the budget gauges, a minute fresh. +func (r *Router) refreshTenants(ctx context.Context) { + refresh := func() { + if err := r.m.tenant.refresh(ctx, r.store, time.Now()); err != nil && ctx.Err() == nil { + r.log.Warn("could not refresh tenant names and budget gauges", "err", err) + } + } + refresh() + t := time.NewTicker(time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + refresh() + } + } } // pruneUsage deletes request records older than the retention, hourly and diff --git a/internal/router/tenantmetrics.go b/internal/router/tenantmetrics.go new file mode 100644 index 0000000..7202b7b --- /dev/null +++ b/internal/router/tenantmetrics.go @@ -0,0 +1,182 @@ +package router + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" + storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// tenantMetrics are the router's series per tenant: requests by how they +// ended, tokens, time to first token, and how much of each budget is spent. +// +// The request series carry one level of the tree, chosen by +// router.tenant_label, rather than every key or project: a series per project +// on a cluster of thousands is a Prometheus that falls over, and alerting on a +// team is what an operator does. Budgets are a series per limit that has one, +// which is as many as somebody wrote. +type tenantMetrics struct { + level string + + requests *prometheus.CounterVec + tokens *prometheus.CounterVec + ttft *prometheus.HistogramVec + budget *prometheus.GaugeVec + + mu sync.RWMutex + names map[string]string // "team:" -> "acme/platform" +} + +func newTenantMetrics() *tenantMetrics { + return &tenantMetrics{ + level: "team", + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_tenant_requests_total", + Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.", + }, []string{"tenant", "outcome"}), + tokens: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "llmgw_tenant_tokens_total", + Help: "Tokens by tenant and direction.", + }, []string{"tenant", "direction"}), + ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "llmgw_tenant_ttft_seconds", + Help: "Time to first token by tenant, for streamed requests.", + Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, + }, []string{"tenant"}), + budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "llmgw_budget_spent_ratio", + Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.", + }, []string{"scope", "id", "name", "period"}), + names: map[string]string{}, + } +} + +func (t *tenantMetrics) collectors() []prometheus.Collector { + return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget} +} + +// tenantOf is the label a request is counted under: the name of its +// organization, team or project at the configured level, "personal" for use +// outside any project, or "" when tenants are not labelled at all. +func (t *tenantMetrics) tenantOf(req *parsedRequest) string { + var scope, id string + switch t.level { + case "org": + scope, id = "org", req.OrgID + case "team": + scope, id = "team", req.TeamID + case "project": + scope, id = "project", req.ProjectID + default: + return "" + } + if id == "" { + return "personal" + } + t.mu.RLock() + name, ok := t.names[scope+":"+id] + t.mu.RUnlock() + if !ok { + // Not yet in the cache — made since the last refresh. The id is + // stable, which is what a label has to be; the name follows. + return id + } + return name +} + +// observe counts one finished or refused request. +func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) { + tenant := t.tenantOf(req) + if tenant == "" { + return + } + outcome := "ok" + switch { + case status == 429: + outcome = "limited" + case status >= 400: + outcome = "error" + } + t.requests.WithLabelValues(tenant, outcome).Inc() + if prompt > 0 { + t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt)) + } + if completion > 0 { + t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion)) + } + if ttft > 0 { + t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds()) + } +} + +// tenantStore is what the refresh reads. +type tenantStore interface { + Tree(ctx context.Context) ([]storepkg.TreeOrg, error) + AllLimits(ctx context.Context) (map[string]storepkg.Limits, error) + SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error) + Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error) +} + +// refresh reads the tree's names and every budget's spend. A minute old at +// most, which is plenty for an alert that waits five before firing. +func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error { + tree, err := s.Tree(ctx) + if err != nil { + return err + } + names := map[string]string{} + for _, o := range tree { + names["org:"+o.ID] = o.Name + for _, tm := range o.Teams { + names["team:"+tm.ID] = o.Name + "/" + tm.Name + for _, p := range tm.Projects { + names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name + } + } + } + t.mu.Lock() + t.names = names + t.mu.Unlock() + + limits, err := s.AllLimits(ctx) + if err != nil { + return err + } + emails := map[string]string{} + if users, err := s.Users(ctx, ""); err == nil { + for _, u := range users { + emails[u.ID] = u.Email + } + } + // Rebuilt whole, so a budget removed since the last refresh stops being + // published rather than staying at its last value for ever. + t.budget.Reset() + for _, l := range limits { + if l.Budget == nil || *l.Budget <= 0 { + continue + } + name := "" + switch l.Scope { + case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject: + name = names[string(l.Scope)+":"+l.ScopeID] + case storepkg.ScopeUser: + name = emails[l.ScopeID] + default: + // Per key and per member are too many series and too little + // to act on; their pools are bounded by the levels above. + continue + } + spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now)) + if err != nil { + return err + } + t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)). + Set(float64(spent) / float64(*l.Budget)) + } + return nil +}