Publish per-tenant request and budget series, and alert on them
Requests by outcome, tokens and first-token time per tenant, at the level router.tenant_label names (team by default), and every budget's share spent this period. Rules alert on budgets at 80% and 100%, tenants hitting limits, failing, spiking past their weekly average, or waiting on first tokens. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9f479b9a70
commit
ce006a6e6f
10 files changed
+376
-2
No files matched your search
@@ -371,6 +371,13 @@ type Router struct {
|
||||
// accounting works regardless of client behaviour.
|
||||
InjectUsage bool `koanf:"inject_usage"`
|
||||
|
||||
// TenantLabel is the level of the tree the per-tenant request series are
|
||||
// labelled by — llmgw_tenant_requests_total and its siblings: "org",
|
||||
// "team" or "project", or "none" for no such series. A label per project
|
||||
// on a cluster of thousands is more series than Prometheus wants, and a
|
||||
// team is what an alert is usually about; "team" is the default.
|
||||
TenantLabel string `koanf:"tenant_label"`
|
||||
|
||||
// MaxBodyBytes bounds the request body the router will read.
|
||||
MaxBodyBytes int64 `koanf:"max_body_bytes"`
|
||||
|
||||
@@ -491,6 +498,7 @@ func Default() Server {
|
||||
Cooldown: 15 * time.Second,
|
||||
FailureThreshold: 3,
|
||||
DefaultProfile: "chat",
|
||||
TenantLabel: "team",
|
||||
SessionHeader: "x-session-id",
|
||||
InjectUsage: true,
|
||||
MaxBodyBytes: 32 << 20,
|
||||
@@ -581,6 +589,11 @@ func (s Server) Validate() error {
|
||||
if s.Auth.Enabled && s.Auth.Store == "" {
|
||||
return fmt.Errorf("auth.store is required when auth is enabled")
|
||||
}
|
||||
switch s.Router.TenantLabel {
|
||||
case "", "none", "org", "team", "project":
|
||||
default:
|
||||
return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel)
|
||||
}
|
||||
if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour {
|
||||
return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention)
|
||||
}
|
||||
|
||||
@@ -261,6 +261,7 @@ type parsedRequest struct {
|
||||
KeyID string
|
||||
UserID string
|
||||
ProjectID string
|
||||
TeamID, OrgID string
|
||||
Tenant string
|
||||
ProfileOverride string
|
||||
PromptPrefix string
|
||||
@@ -519,6 +520,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
// the truth than charging nothing.
|
||||
u.PromptTokens = int64(req.PromptTokens)
|
||||
}
|
||||
r.m.tenant.observe(req, status, u.PromptTokens, u.CompletionTokens, res.outcome.TTFT)
|
||||
r.acct.Record(req.reservation, u)
|
||||
}
|
||||
|
||||
@@ -537,6 +539,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
|
||||
// which is absent when waiting does not help.
|
||||
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0)
|
||||
r.m.tenant.observe(req, http.StatusTooManyRequests, 0, 0, 0)
|
||||
r.acct.RecordRefused(store.Usage{
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests,
|
||||
@@ -584,6 +587,7 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
p.KeyID = principal.KeyID
|
||||
p.UserID = principal.UserID
|
||||
p.ProjectID = principal.ProjectID
|
||||
p.TeamID, p.OrgID = principal.TeamID, principal.OrgID
|
||||
// Tenant fairness within a band is per key rather than per session, so
|
||||
// one key cannot take a whole band by opening many sessions.
|
||||
p.Tenant = principal.KeyID
|
||||
|
||||
@@ -29,6 +29,8 @@ type metrics struct {
|
||||
admissionRejected *prometheus.CounterVec
|
||||
|
||||
prefixDecisions *prometheus.CounterVec
|
||||
|
||||
tenant *tenantMetrics
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
@@ -86,16 +88,17 @@ func newMetrics() *metrics {
|
||||
Name: "llmgw_admission_rejected_total",
|
||||
Help: "Requests refused by flow control, by reason.",
|
||||
}, []string{"band", "reason"}),
|
||||
tenant: newTenantMetrics(),
|
||||
}
|
||||
}
|
||||
|
||||
// collectors returns everything to register, so registration lives in one place.
|
||||
func (m *metrics) collectors() []prometheus.Collector {
|
||||
return []prometheus.Collector{
|
||||
return append([]prometheus.Collector{
|
||||
m.requests, m.duration, m.ttft, m.tokens,
|
||||
m.scorerValue, m.filtered, m.considered,
|
||||
m.admissionWait, m.admissionRejected, m.prefixDecisions,
|
||||
}
|
||||
}, m.tenant.collectors()...)
|
||||
}
|
||||
|
||||
// observePrefix records which branch a prefix decision took.
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -224,3 +226,39 @@ func TestARequestALimitRefusesIsCountedAgainstItsOwner(t *testing.T) {
|
||||
}
|
||||
t.Fatal("the refused request was never counted against its project")
|
||||
}
|
||||
|
||||
func TestTenantsAreCountedAndTheirBudgetsGauged(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
ctx := t.Context()
|
||||
orgs, _ := rt.store.AllOrgs(ctx)
|
||||
teams, _ := rt.store.AllTeams(ctx)
|
||||
budget := int64(1_000_000)
|
||||
if err := rt.store.PutLimits(ctx, store.Limits{Scope: store.ScopeTeam, ScopeID: teams[0].ID,
|
||||
Budget: &budget, BudgetPeriod: store.PeriodMonth}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := post(t, rt.Handler(), map[string]any{"model": "q14", "messages": []any{
|
||||
map[string]any{"role": "user", "content": "hello"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("status %d", rec.Code)
|
||||
}
|
||||
tenant := orgs[0].Name + "/" + teams[0].Name
|
||||
if got := testutil.ToFloat64(rt.m.tenant.requests.WithLabelValues(tenant, "ok")); got != 1 {
|
||||
t.Errorf("%s has %v ok requests, want 1", tenant, got)
|
||||
}
|
||||
|
||||
u := waitForUsage(t, rt)
|
||||
if err := rt.m.tenant.refresh(ctx, rt.store, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := float64(u.CostMicros) / float64(budget)
|
||||
got := testutil.ToFloat64(rt.m.tenant.budget.WithLabelValues("team", teams[0].ID, tenant, "month"))
|
||||
if want == 0 || got != want {
|
||||
t.Errorf("the team's budget gauge reads %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -191,6 +191,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
// the decision metric is the whole difference between tuning this and
|
||||
// guessing at it.
|
||||
metrics := newMetrics()
|
||||
metrics.tenant.level = cfg.Router.TenantLabel
|
||||
var prefixScorer *scheduler.PrefixScorer
|
||||
if cfg.Router.Prefix.Enabled {
|
||||
prefixScorer = scheduler.NewPrefixScorer(
|
||||
@@ -418,6 +419,30 @@ func (r *Router) Start(ctx context.Context) {
|
||||
if r.store != nil && r.usageRetention > 0 {
|
||||
go r.pruneUsage(ctx, r.usageRetention)
|
||||
}
|
||||
if r.store != nil {
|
||||
go r.refreshTenants(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// refreshTenants keeps the tenant names the request series are labelled with,
|
||||
// and the budget gauges, a minute fresh.
|
||||
func (r *Router) refreshTenants(ctx context.Context) {
|
||||
refresh := func() {
|
||||
if err := r.m.tenant.refresh(ctx, r.store, time.Now()); err != nil && ctx.Err() == nil {
|
||||
r.log.Warn("could not refresh tenant names and budget gauges", "err", err)
|
||||
}
|
||||
}
|
||||
refresh()
|
||||
t := time.NewTicker(time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
refresh()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pruneUsage deletes request records older than the retention, hourly and
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// tenantMetrics are the router's series per tenant: requests by how they
|
||||
// ended, tokens, time to first token, and how much of each budget is spent.
|
||||
//
|
||||
// The request series carry one level of the tree, chosen by
|
||||
// router.tenant_label, rather than every key or project: a series per project
|
||||
// on a cluster of thousands is a Prometheus that falls over, and alerting on a
|
||||
// team is what an operator does. Budgets are a series per limit that has one,
|
||||
// which is as many as somebody wrote.
|
||||
type tenantMetrics struct {
|
||||
level string
|
||||
|
||||
requests *prometheus.CounterVec
|
||||
tokens *prometheus.CounterVec
|
||||
ttft *prometheus.HistogramVec
|
||||
budget *prometheus.GaugeVec
|
||||
|
||||
mu sync.RWMutex
|
||||
names map[string]string // "team:<id>" -> "acme/platform"
|
||||
}
|
||||
|
||||
func newTenantMetrics() *tenantMetrics {
|
||||
return &tenantMetrics{
|
||||
level: "team",
|
||||
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "llmgw_tenant_requests_total",
|
||||
Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.",
|
||||
}, []string{"tenant", "outcome"}),
|
||||
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "llmgw_tenant_tokens_total",
|
||||
Help: "Tokens by tenant and direction.",
|
||||
}, []string{"tenant", "direction"}),
|
||||
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "llmgw_tenant_ttft_seconds",
|
||||
Help: "Time to first token by tenant, for streamed requests.",
|
||||
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
||||
}, []string{"tenant"}),
|
||||
budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "llmgw_budget_spent_ratio",
|
||||
Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.",
|
||||
}, []string{"scope", "id", "name", "period"}),
|
||||
names: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tenantMetrics) collectors() []prometheus.Collector {
|
||||
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget}
|
||||
}
|
||||
|
||||
// tenantOf is the label a request is counted under: the name of its
|
||||
// organization, team or project at the configured level, "personal" for use
|
||||
// outside any project, or "" when tenants are not labelled at all.
|
||||
func (t *tenantMetrics) tenantOf(req *parsedRequest) string {
|
||||
var scope, id string
|
||||
switch t.level {
|
||||
case "org":
|
||||
scope, id = "org", req.OrgID
|
||||
case "team":
|
||||
scope, id = "team", req.TeamID
|
||||
case "project":
|
||||
scope, id = "project", req.ProjectID
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
if id == "" {
|
||||
return "personal"
|
||||
}
|
||||
t.mu.RLock()
|
||||
name, ok := t.names[scope+":"+id]
|
||||
t.mu.RUnlock()
|
||||
if !ok {
|
||||
// Not yet in the cache — made since the last refresh. The id is
|
||||
// stable, which is what a label has to be; the name follows.
|
||||
return id
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// observe counts one finished or refused request.
|
||||
func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) {
|
||||
tenant := t.tenantOf(req)
|
||||
if tenant == "" {
|
||||
return
|
||||
}
|
||||
outcome := "ok"
|
||||
switch {
|
||||
case status == 429:
|
||||
outcome = "limited"
|
||||
case status >= 400:
|
||||
outcome = "error"
|
||||
}
|
||||
t.requests.WithLabelValues(tenant, outcome).Inc()
|
||||
if prompt > 0 {
|
||||
t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt))
|
||||
}
|
||||
if completion > 0 {
|
||||
t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion))
|
||||
}
|
||||
if ttft > 0 {
|
||||
t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds())
|
||||
}
|
||||
}
|
||||
|
||||
// tenantStore is what the refresh reads.
|
||||
type tenantStore interface {
|
||||
Tree(ctx context.Context) ([]storepkg.TreeOrg, error)
|
||||
AllLimits(ctx context.Context) (map[string]storepkg.Limits, error)
|
||||
SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error)
|
||||
Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error)
|
||||
}
|
||||
|
||||
// refresh reads the tree's names and every budget's spend. A minute old at
|
||||
// most, which is plenty for an alert that waits five before firing.
|
||||
func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error {
|
||||
tree, err := s.Tree(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := map[string]string{}
|
||||
for _, o := range tree {
|
||||
names["org:"+o.ID] = o.Name
|
||||
for _, tm := range o.Teams {
|
||||
names["team:"+tm.ID] = o.Name + "/" + tm.Name
|
||||
for _, p := range tm.Projects {
|
||||
names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
t.mu.Lock()
|
||||
t.names = names
|
||||
t.mu.Unlock()
|
||||
|
||||
limits, err := s.AllLimits(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
emails := map[string]string{}
|
||||
if users, err := s.Users(ctx, ""); err == nil {
|
||||
for _, u := range users {
|
||||
emails[u.ID] = u.Email
|
||||
}
|
||||
}
|
||||
// Rebuilt whole, so a budget removed since the last refresh stops being
|
||||
// published rather than staying at its last value for ever.
|
||||
t.budget.Reset()
|
||||
for _, l := range limits {
|
||||
if l.Budget == nil || *l.Budget <= 0 {
|
||||
continue
|
||||
}
|
||||
name := ""
|
||||
switch l.Scope {
|
||||
case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject:
|
||||
name = names[string(l.Scope)+":"+l.ScopeID]
|
||||
case storepkg.ScopeUser:
|
||||
name = emails[l.ScopeID]
|
||||
default:
|
||||
// Per key and per member are too many series and too little
|
||||
// to act on; their pools are bounded by the levels above.
|
||||
continue
|
||||
}
|
||||
spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)).
|
||||
Set(float64(spent) / float64(*l.Budget))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user