Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
248 lines
7.5 KiB
Go
248 lines
7.5 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// tenantMetrics are the router's series per tenant: requests by how they
|
|
// ended, tokens, time to first token, and how much of each budget is spent.
|
|
//
|
|
// The request series carry one level of the tree, chosen by
|
|
// router.tenant_label, rather than every key or project: a series per project
|
|
// on a cluster of thousands is a Prometheus that falls over, and alerting on a
|
|
// team is what an operator does. Budgets are a series per limit that has one,
|
|
// which is as many as somebody wrote.
|
|
type tenantMetrics struct {
|
|
level string
|
|
|
|
requests *prometheus.CounterVec
|
|
tokens *prometheus.CounterVec
|
|
ttft *prometheus.HistogramVec
|
|
budget *prometheus.GaugeVec
|
|
|
|
// Request tags, for the keys router.tag_labels names.
|
|
tagKeys map[string]bool
|
|
tagRequests *prometheus.CounterVec
|
|
tagTokens *prometheus.CounterVec
|
|
tagValues map[string]map[string]bool // tag -> values counted apart
|
|
|
|
mu sync.RWMutex
|
|
names map[string]string // "team:<id>" -> "acme/platform"
|
|
}
|
|
|
|
func newTenantMetrics() *tenantMetrics {
|
|
return &tenantMetrics{
|
|
level: "team",
|
|
requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tenant_requests_total",
|
|
Help: "Requests by tenant (the level router.tenant_label names) and outcome: ok, error, or limited — refused by a rate, token or budget limit.",
|
|
}, []string{"tenant", "outcome"}),
|
|
tokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tenant_tokens_total",
|
|
Help: "Tokens by tenant and direction.",
|
|
}, []string{"tenant", "direction"}),
|
|
ttft: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "llmgw_tenant_ttft_seconds",
|
|
Help: "Time to first token by tenant, for streamed requests.",
|
|
Buckets: []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
|
}, []string{"tenant"}),
|
|
budget: prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
|
Name: "llmgw_budget_spent_ratio",
|
|
Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.",
|
|
}, []string{"scope", "id", "name", "period"}),
|
|
tagRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tagged_requests_total",
|
|
Help: "Requests by a request tag router.tag_labels names, its value, and outcome: ok, error, or limited.",
|
|
}, []string{"tag", "value", "outcome"}),
|
|
tagTokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "llmgw_tagged_tokens_total",
|
|
Help: "Tokens by a request tag router.tag_labels names, its value, and direction.",
|
|
}, []string{"tag", "value", "direction"}),
|
|
tagValues: map[string]map[string]bool{},
|
|
names: map[string]string{},
|
|
}
|
|
}
|
|
|
|
// maxTagValues is how many values of one tag are counted apart.
|
|
const maxTagValues = 100
|
|
|
|
// setTagLabels names the tags counted.
|
|
func (t *tenantMetrics) setTagLabels(keys []string) {
|
|
t.tagKeys = map[string]bool{}
|
|
for _, k := range keys {
|
|
t.tagKeys[k] = true
|
|
}
|
|
}
|
|
|
|
// tagValue is the label a tag's value is counted under: itself while there
|
|
// is room, "other" after.
|
|
func (t *tenantMetrics) tagValue(tag, value string) string {
|
|
t.mu.Lock()
|
|
defer t.mu.Unlock()
|
|
seen := t.tagValues[tag]
|
|
if seen == nil {
|
|
seen = map[string]bool{}
|
|
t.tagValues[tag] = seen
|
|
}
|
|
if seen[value] {
|
|
return value
|
|
}
|
|
if len(seen) >= maxTagValues {
|
|
return "other"
|
|
}
|
|
seen[value] = true
|
|
return value
|
|
}
|
|
|
|
func (t *tenantMetrics) observeTags(req *parsedRequest, outcome string, prompt, completion int64) {
|
|
for k, v := range req.Tags {
|
|
if !t.tagKeys[k] {
|
|
continue
|
|
}
|
|
v = t.tagValue(k, v)
|
|
t.tagRequests.WithLabelValues(k, v, outcome).Inc()
|
|
if prompt > 0 {
|
|
t.tagTokens.WithLabelValues(k, v, "prompt").Add(float64(prompt))
|
|
}
|
|
if completion > 0 {
|
|
t.tagTokens.WithLabelValues(k, v, "completion").Add(float64(completion))
|
|
}
|
|
}
|
|
}
|
|
|
|
func (t *tenantMetrics) collectors() []prometheus.Collector {
|
|
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget, t.tagRequests, t.tagTokens}
|
|
}
|
|
|
|
// tenantOf is the label a request is counted under: the name of its
|
|
// organization, team or project at the configured level, "personal" for use
|
|
// outside any project, or "" when tenants are not labelled at all.
|
|
func (t *tenantMetrics) tenantOf(req *parsedRequest) string {
|
|
var scope, id string
|
|
switch t.level {
|
|
case "org":
|
|
scope, id = "org", req.OrgID
|
|
case "team":
|
|
scope, id = "team", req.TeamID
|
|
case "project":
|
|
scope, id = "project", req.ProjectID
|
|
default:
|
|
return ""
|
|
}
|
|
if id == "" {
|
|
return "personal"
|
|
}
|
|
t.mu.RLock()
|
|
name, ok := t.names[scope+":"+id]
|
|
t.mu.RUnlock()
|
|
if !ok {
|
|
// Not yet in the cache — made since the last refresh. The id is
|
|
// stable, which is what a label has to be; the name follows.
|
|
return id
|
|
}
|
|
return name
|
|
}
|
|
|
|
// observe counts one finished or refused request.
|
|
func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) {
|
|
outcome := "ok"
|
|
switch {
|
|
case status == 429:
|
|
outcome = "limited"
|
|
case status >= 400:
|
|
outcome = "error"
|
|
}
|
|
if len(t.tagKeys) > 0 {
|
|
t.observeTags(req, outcome, prompt, completion)
|
|
}
|
|
tenant := t.tenantOf(req)
|
|
if tenant == "" {
|
|
return
|
|
}
|
|
t.requests.WithLabelValues(tenant, outcome).Inc()
|
|
if prompt > 0 {
|
|
t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt))
|
|
}
|
|
if completion > 0 {
|
|
t.tokens.WithLabelValues(tenant, "completion").Add(float64(completion))
|
|
}
|
|
if ttft > 0 {
|
|
t.ttft.WithLabelValues(tenant).Observe(ttft.Seconds())
|
|
}
|
|
}
|
|
|
|
// tenantStore is what the refresh reads.
|
|
type tenantStore interface {
|
|
Tree(ctx context.Context) ([]storepkg.TreeOrg, error)
|
|
AllLimits(ctx context.Context) (map[string]storepkg.Limits, error)
|
|
SpendOfPool(ctx context.Context, l storepkg.Limits, since time.Time) (int64, error)
|
|
Users(ctx context.Context, state storepkg.UserState) ([]storepkg.User, error)
|
|
}
|
|
|
|
// refresh reads the tree's names and every budget's spend. A minute old at
|
|
// most, which is plenty for an alert that waits five before firing.
|
|
func (t *tenantMetrics) refresh(ctx context.Context, s tenantStore, now time.Time) error {
|
|
tree, err := s.Tree(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
names := map[string]string{}
|
|
for _, o := range tree {
|
|
names["org:"+o.ID] = o.Name
|
|
for _, tm := range o.Teams {
|
|
names["team:"+tm.ID] = o.Name + "/" + tm.Name
|
|
for _, p := range tm.Projects {
|
|
names["project:"+p.ID] = o.Name + "/" + tm.Name + "/" + p.Name
|
|
}
|
|
}
|
|
}
|
|
t.mu.Lock()
|
|
t.names = names
|
|
t.mu.Unlock()
|
|
|
|
limits, err := s.AllLimits(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
emails := map[string]string{}
|
|
if users, err := s.Users(ctx, ""); err == nil {
|
|
for _, u := range users {
|
|
emails[u.ID] = u.Email
|
|
}
|
|
}
|
|
// Rebuilt whole, so a budget removed since the last refresh stops being
|
|
// published rather than staying at its last value for ever.
|
|
t.budget.Reset()
|
|
for _, l := range limits {
|
|
if l.Budget == nil || *l.Budget <= 0 {
|
|
continue
|
|
}
|
|
name := ""
|
|
switch l.Scope {
|
|
case storepkg.ScopeOrg, storepkg.ScopeTeam, storepkg.ScopeProject:
|
|
name = names[string(l.Scope)+":"+l.ScopeID]
|
|
case storepkg.ScopeUser:
|
|
name = emails[l.ScopeID]
|
|
default:
|
|
// Per key and per member are too many series and too little
|
|
// to act on; their pools are bounded by the levels above.
|
|
continue
|
|
}
|
|
spent, err := s.SpendOfPool(ctx, l, accounting.PeriodStart(l.BudgetPeriod, now))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
t.budget.WithLabelValues(string(l.Scope), l.ScopeID, strings.TrimSpace(name), string(l.BudgetPeriod)).
|
|
Set(float64(spent) / float64(*l.Budget))
|
|
}
|
|
return nil
|
|
}
|