package router import ( "context" "strings" "sync" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // tenantMetrics are the router's series per tenant: requests by how they // ended, tokens, time to first token, and how much of each budget is spent. // // The request series carry one level of the tree, chosen by // router.tenant_label, rather than every key or project: a series per project // on a cluster of thousands is a Prometheus that falls over, and alerting on a // team is what an operator does. Budgets are a series per limit that has one, // which is as many as somebody wrote. type tenantMetrics struct { level string requests *prometheus.CounterVec tokens *prometheus.CounterVec ttft *prometheus.HistogramVec budget *prometheus.GaugeVec // Request tags, for the keys router.tag_labels names. tagKeys map[string]bool tagRequests *prometheus.CounterVec tagTokens *prometheus.CounterVec tagValues map[string]map[string]bool // tag -> values counted apart mu sync.RWMutex names map[string]string // "team:" -> "acme/platform" } func newTenantMetrics() *tenantMetrics { return &tenantMetrics{ level: "team", requests: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_tenant_requests_total", Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.", }, []string{"tenant", "outcome"}), tokens: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_tenant_tokens_total", Help: "Tokens by tenant and direction.", }, []string{"tenant", "direction"}), ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "llmgw_tenant_ttft_seconds", Help: "Time to first token by tenant, for streamed requests.", Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60}, }, []string{"tenant"}), budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "llmgw_budget_spent_ratio", Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.", }, []string{"scope", "id", "name", "period"}), tagRequests: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_tagged_requests_total", Help: "Requests by a request tag router.tag_labels names, its value, and outcome: ok, error, or limited.", }, []string{"tag", "value", "outcome"}), tagTokens: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "llmgw_tagged_tokens_total", Help: "Tokens by a request tag router.tag_labels names, its value, and direction.", }, []string{"tag", "value", "direction"}), tagValues: map[string]map[string]bool{}, names: map[string]string{}, } } // maxTagValues is how many values of one tag are counted apart. const maxTagValues = 100 // setTagLabels names the tags counted. func (t *tenantMetrics) setTagLabels(keys []string) { t.tagKeys = map[string]bool{} for _, k := range keys { t.tagKeys[k] = true } } // tagValue is the label a tag's value is counted under: itself while there // is room, "other" after. func (t *tenantMetrics) tagValue(tag, value string) string { t.mu.Lock() defer t.mu.Unlock() seen := t.tagValues[tag] if seen == nil { seen = map[string]bool{} t.tagValues[tag] = seen } if seen[value] { return value } if len(seen) >= maxTagValues { return "other" } seen[value] = true return value } func (t *tenantMetrics) observeTags(req *parsedRequest, outcome string, prompt, completion int64) { for k, v := range req.Tags { if !t.tagKeys[k] { continue } v = t.tagValue(k, v) t.tagRequests.WithLabelValues(k, v, outcome).Inc() if prompt > 0 { t.tagTokens.WithLabelValues(k, v, "prompt").Add(float64(prompt)) } if completion > 0 { t.tagTokens.WithLabelValues(k, v, "completion").Add(float64(completion)) } } } func (t *tenantMetrics) collectors() []prometheus.Collector { return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget, t.tagRequests, t.tagTokens} } // tenantOf is the label a request is counted under: the name of its // organization, team or project at the configured level, "personal" for use // outside any project, or "" when tenants are not labelled at all. func (t *tenantMetrics) tenantOf(req *parsedRequest) string { var scope, id string switch t.level { case "org": scope, id = "org", req.OrgID case "team": scope, id = "team", req.TeamID case "project": scope, id = "project", req.ProjectID default: return "" } if id == "" { return "personal" } t.mu.RLock() name, ok := t.names[scope+":"+id] t.mu.RUnlock() if !ok { // Not yet in the cache — made since the last refresh. The id is // stable, which is what a label has to be; the name follows. return id } return name } // observe counts one finished or refused request. func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) { outcome := "ok" switch { case status == 429: outcome = "limited" case status >= 400: outcome = "error" } if len(t.tagKeys) > 0 { t.observeTags(req, outcome, prompt, completion) } tenant := t.tenantOf(req) if tenant == "" { return } t.requests.WithLabelValues(tenant, outcome).Inc() if prompt > 0 { t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt)) } if completion > 0 { t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion)) } if ttft > 0 { t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds()) } } // tenantStore is what the refresh reads. type tenantStore interface { Tree(ctx context.Context) ([]storepkg.TreeOrg, error) AllLimits(ctx context.Context) (map[string]storepkg.Limits, error) SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error) Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error) } // refresh reads the tree's names and every budget's spend. A minute old at // most, which is plenty for an alert that waits five before firing. func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error { tree, err := s.Tree(ctx) if err != nil { return err } names := map[string]string{} for _, o := range tree { names["org:"+o.ID] = o.Name for _, tm := range o.Teams { names["team:"+tm.ID] = o.Name + "/" + tm.Name for _, p := range tm.Projects { names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name } } } t.mu.Lock() t.names = names t.mu.Unlock() limits, err := s.AllLimits(ctx) if err != nil { return err } emails := map[string]string{} if users, err := s.Users(ctx, ""); err == nil { for _, u := range users { emails[u.ID] = u.Email } } // Rebuilt whole, so a budget removed since the last refresh stops being // published rather than staying at its last value for ever. t.budget.Reset() for _, l := range limits { if l.Budget == nil || *l.Budget <= 0 { continue } name := "" switch l.Scope { case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject: name = names[string(l.Scope)+":"+l.ScopeID] case storepkg.ScopeUser: name = emails[l.ScopeID] default: // Per key and per member are too many series and too little // to act on; their pools are bounded by the levels above. continue } spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now)) if err != nil { return err } t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)). Set(float64(spent) / float64(*l.Budget)) } return nil }