Files
llmgw/internal/router/tenantmetrics.go
T

248 lines
7.5 KiB
Go

package router
import (
"context"
"strings"
"sync"
"time"
"github.com/prometheus/client_golang/prometheus"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// tenantMetrics are the router's series per tenant: requests by how they
// ended, tokens, time to first token, and how much of each budget is spent.
//
// The request series carry one level of the tree, chosen by
// router.tenant_label, rather than every key or project: a series per project
// on a cluster of thousands is a Prometheus that falls over, and alerting on a
// team is what an operator does. Budgets are a series per limit that has one,
// which is as many as somebody wrote.
type tenantMetrics struct {
level string
requests *prometheus.CounterVec
tokens *prometheus.CounterVec
ttft *prometheus.HistogramVec
budget *prometheus.GaugeVec
// Request tags, for the keys router.tag_labels names.
tagKeys map[string]bool
tagRequests *prometheus.CounterVec
tagTokens *prometheus.CounterVec
tagValues map[string]map[string]bool // tag -> values counted apart
mu sync.RWMutex
names map[string]string // "team:<id>" -> "acme/platform"
}
func newTenantMetrics() *tenantMetrics {
return &tenantMetrics{
level: "team",
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tenant_requests_total",
Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.",
}, []string{"tenant", "outcome"}),
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tenant_tokens_total",
Help: "Tokens by tenant and direction.",
}, []string{"tenant", "direction"}),
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "llmgw_tenant_ttft_seconds",
Help: "Time to first token by tenant, for streamed requests.",
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
}, []string{"tenant"}),
budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "llmgw_budget_spent_ratio",
Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.",
}, []string{"scope", "id", "name", "period"}),
tagRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tagged_requests_total",
Help: "Requests by a request tag router.tag_labels names, its value, and outcome: ok, error, or limited.",
}, []string{"tag", "value", "outcome"}),
tagTokens: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tagged_tokens_total",
Help: "Tokens by a request tag router.tag_labels names, its value, and direction.",
}, []string{"tag", "value", "direction"}),
tagValues: map[string]map[string]bool{},
names: map[string]string{},
}
}
// maxTagValues is how many values of one tag are counted apart.
const maxTagValues = 100
// setTagLabels names the tags counted.
func (t *tenantMetrics) setTagLabels(keys []string) {
t.tagKeys = map[string]bool{}
for _, k := range keys {
t.tagKeys[k] = true
}
}
// tagValue is the label a tag's value is counted under: itself while there
// is room, "other" after.
func (t *tenantMetrics) tagValue(tag, value string) string {
t.mu.Lock()
defer t.mu.Unlock()
seen := t.tagValues[tag]
if seen == nil {
seen = map[string]bool{}
t.tagValues[tag] = seen
}
if seen[value] {
return value
}
if len(seen) >= maxTagValues {
return "other"
}
seen[value] = true
return value
}
func (t *tenantMetrics) observeTags(req *parsedRequest, outcome string, prompt, completion int64) {
for k, v := range req.Tags {
if !t.tagKeys[k] {
continue
}
v = t.tagValue(k, v)
t.tagRequests.WithLabelValues(k, v, outcome).Inc()
if prompt > 0 {
t.tagTokens.WithLabelValues(k, v, "prompt").Add(float64(prompt))
}
if completion > 0 {
t.tagTokens.WithLabelValues(k, v, "completion").Add(float64(completion))
}
}
}
func (t *tenantMetrics) collectors() []prometheus.Collector {
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget, t.tagRequests, t.tagTokens}
}
// tenantOf is the label a request is counted under: the name of its
// organization, team or project at the configured level, "personal" for use
// outside any project, or "" when tenants are not labelled at all.
func (t *tenantMetrics) tenantOf(req *parsedRequest) string {
var scope, id string
switch t.level {
case "org":
scope, id = "org", req.OrgID
case "team":
scope, id = "team", req.TeamID
case "project":
scope, id = "project", req.ProjectID
default:
return ""
}
if id == "" {
return "personal"
}
t.mu.RLock()
name, ok := t.names[scope+":"+id]
t.mu.RUnlock()
if !ok {
// Not yet in the cache — made since the last refresh. The id is
// stable, which is what a label has to be; the name follows.
return id
}
return name
}
// observe counts one finished or refused request.
func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) {
outcome := "ok"
switch {
case status == 429:
outcome = "limited"
case status >= 400:
outcome = "error"
}
if len(t.tagKeys) > 0 {
t.observeTags(req, outcome, prompt, completion)
}
tenant := t.tenantOf(req)
if tenant == "" {
return
}
t.requests.WithLabelValues(tenant, outcome).Inc()
if prompt > 0 {
t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt))
}
if completion > 0 {
t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion))
}
if ttft > 0 {
t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds())
}
}
// tenantStore is what the refresh reads.
type tenantStore interface {
Tree(ctx context.Context) ([]storepkg.TreeOrg, error)
AllLimits(ctx context.Context) (map[string]storepkg.Limits, error)
SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error)
Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error)
}
// refresh reads the tree's names and every budget's spend. A minute old at
// most, which is plenty for an alert that waits five before firing.
func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error {
tree, err := s.Tree(ctx)
if err != nil {
return err
}
names := map[string]string{}
for _, o := range tree {
names["org:"+o.ID] = o.Name
for _, tm := range o.Teams {
names["team:"+tm.ID] = o.Name + "/" + tm.Name
for _, p := range tm.Projects {
names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name
}
}
}
t.mu.Lock()
t.names = names
t.mu.Unlock()
limits, err := s.AllLimits(ctx)
if err != nil {
return err
}
emails := map[string]string{}
if users, err := s.Users(ctx, ""); err == nil {
for _, u := range users {
emails[u.ID] = u.Email
}
}
// Rebuilt whole, so a budget removed since the last refresh stops being
// published rather than staying at its last value for ever.
t.budget.Reset()
for _, l := range limits {
if l.Budget == nil || *l.Budget <= 0 {
continue
}
name := ""
switch l.Scope {
case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject:
name = names[string(l.Scope)+":"+l.ScopeID]
case storepkg.ScopeUser:
name = emails[l.ScopeID]
default:
// Per key and per member are too many series and too little
// to act on; their pools are bounded by the levels above.
continue
}
spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now))
if err != nil {
return err
}
t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)).
Set(float64(spent) / float64(*l.Budget))
}
return nil
}