Requests can be tagged: metadata, x-llmgw-tags and the user field on the usage record, and counted by the tags router.tag_labels names

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 19:45:37 +02:00
1 parent e59ff52932
commit 29a7347df2
19 files changed
+436 -17

No files matched your search

+1
View File
@@ -135,6 +135,7 @@ func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guar
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: req.Model, Status: status,
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, QueueMs: req.QueueMs,
EndUser: req.EndUser, Tags: req.Tags,
}
guardFields(&u, v, guard.Verdict{})
// Settled at nothing: the estimate the limit check charged is
+24 -4
View File
@@ -280,6 +280,10 @@ func (r *Router) Handler() http.Handler {
// business interpreting.
type parsedRequest struct {
Path string
// Tags are the client's metadata and x-llmgw-tags; EndUser is OpenAI's
// user field. Both are recorded with the usage, never sent on.
Tags map[string]string
EndUser string
// Model is what will be dispatched, which is not always what was asked
// for: a caller out of budget may be moved to a model they can afford.
Model string
@@ -338,9 +342,10 @@ type parsedRequest struct {
// inferenceBody covers the fields the router reads across all inference
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
type inferenceBody struct {
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Model string `json:"model"`
Stream bool `json:"stream"`
User string `json:"user"`
Metadata json.RawMessage `json:"metadata"`
Messages []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
@@ -587,6 +592,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
LatencyMs: took.Milliseconds(),
RequestID: req.RequestID, Replica: res.endpointID,
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
EndUser: req.EndUser, Tags: req.Tags,
}
guardFields(&u, req.guardIn, res.guard)
if u.ServedModel == "" {
@@ -625,6 +631,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests,
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID,
EndUser: req.EndUser, Tags: req.Tags,
})
r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start))
}
@@ -644,7 +651,14 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
return nil, errors.New("model is required")
}
tags, err := requestTags(b.Metadata, req.Header.Get(tagsHeader))
if err != nil {
return nil, err
}
p := &parsedRequest{
Tags: tags,
EndUser: endUser(b.User),
Path: path,
Model: b.Model,
RequestedModel: b.Model,
@@ -680,10 +694,16 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
// see promptBytes.
p.PromptTokens = promptBytes(&b) / 4
// Tags are the gateway's: an engine has no use for them, and its logs
// are no place for what a client wrote there.
if len(b.Metadata) > 0 {
p.UpstreamBody = withoutField(p.UpstreamBody, "metadata")
}
// Ask the engine for usage the client did not request, so accounting works
// regardless of client behaviour; the extra chunk is dropped on the way back.
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
if injected, ok := injectIncludeUsage(body); ok {
if injected, ok := injectIncludeUsage(p.UpstreamBody); ok {
p.UpstreamBody = injected
p.SuppressUsageChunk = true
}
+1
View File
@@ -209,6 +209,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
// guessing at it.
metrics := newMetrics()
metrics.tenant.level = cfg.Router.TenantLabel
metrics.tenant.setTagLabels(cfg.Router.TagLabels)
var prefixScorer *scheduler.PrefixScorer
if cfg.Router.Prefix.Enabled {
prefixScorer = scheduler.NewPrefixScorer(
+91
View File
@@ -0,0 +1,91 @@
package router
import (
"encoding/json"
"fmt"
"regexp"
"strings"
)
// What a client may say about a request, for attributing spend inside a
// project: OpenAI's metadata (at most 16 keys, as OpenAI allows) and the
// x-llmgw-tags header, which wins for a key both give.
const (
tagsHeader = "x-llmgw-tags"
maxTags = 16
maxTagValue = 512
maxEndUser = 256
)
var tagKey = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)
// requestTags reads metadata from the body and x-llmgw-tags ("team=search,
// feature=autocomplete") from the header. An invalid one is the client's
// mistake and is said, not dropped: a tag silently lost is spend that
// cannot be found again.
func requestTags(metadata json.RawMessage, header string) (map[string]string, error) {
tags := map[string]string{}
if len(metadata) > 0 && string(metadata) != "null" {
var m map[string]any
if err := json.Unmarshal(metadata, &m); err != nil {
return nil, fmt.Errorf("metadata is an object of strings")
}
for k, v := range m {
s, ok := v.(string)
if !ok {
return nil, fmt.Errorf("metadata.%s is not a string", k)
}
tags[k] = s
}
}
if header = strings.TrimSpace(header); header != "" {
for _, part := range strings.Split(header, ",") {
k, v, ok := strings.Cut(strings.TrimSpace(part), "=")
if !ok {
return nil, fmt.Errorf("%s: %q is not key=value", tagsHeader, part)
}
tags[strings.TrimSpace(k)] = strings.TrimSpace(v)
}
}
if len(tags) > maxTags {
return nil, fmt.Errorf("%d tags; at most %d", len(tags), maxTags)
}
for k, v := range tags {
if !tagKey.MatchString(k) {
return nil, fmt.Errorf("tag %q: a key is 1 to 64 letters, digits, '_', '.' or '-'", k)
}
if len(v) > maxTagValue {
return nil, fmt.Errorf("tag %s: the value is longer than %d bytes", k, maxTagValue)
}
}
if len(tags) == 0 {
return nil, nil
}
return tags, nil
}
// endUser is OpenAI's user field as kept: bounded, since it is the client's.
func endUser(u string) string {
if len(u) > maxEndUser {
return u[:maxEndUser]
}
return u
}
// withoutField removes a top-level field from a JSON body, or returns it
// unchanged when the field is not there or the body is not an object.
func withoutField(body []byte, field string) []byte {
var m map[string]json.RawMessage
if err := json.Unmarshal(body, &m); err != nil {
return body
}
if _, ok := m[field]; !ok {
return body
}
delete(m, field)
out, err := json.Marshal(m)
if err != nil {
return body
}
return out
}
+101
View File
@@ -0,0 +1,101 @@
package router
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/testutil"
)
func TestTagsComeFromMetadataAndTheHeader(t *testing.T) {
got, err := requestTags(json.RawMessage(`{"feature":"search","team":"a"}`), "team=b, env=prod")
if err != nil {
t.Fatal(err)
}
if len(got) != 3 || got["team"] != "b" || got["feature"] != "search" || got["env"] != "prod" {
t.Errorf("%v", got)
}
if got, err := requestTags(nil, ""); got != nil || err != nil {
t.Errorf("none: %v %v", got, err)
}
many := map[string]string{}
for i := range 17 {
many[strings.Repeat("k", i+1)] = "v"
}
b, _ := json.Marshal(many)
for name, c := range map[string]struct {
meta string
header string
}{
"not strings": {`{"n": 1}`, ""},
"not an object": {`["a"]`, ""},
"no equals": {"", "feature"},
"bad key": {`{"a b": "x"}`, ""},
"long value": {`{"a": "` + strings.Repeat("x", 513) + `"}`, ""},
"too many": {string(b), ""},
} {
if _, err := requestTags(json.RawMessage(c.meta), c.header); err == nil {
t.Errorf("%s: accepted", name)
}
}
}
func TestTagsAreRecordedAndNotSentToTheEngine(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
rt.m.tenant.setTagLabels([]string{"feature"})
rec := post(t, rt.Handler(),
map[string]any{"model": "q14", "user": "customer-42", "metadata": map[string]string{"feature": "search"},
"messages": []any{map[string]any{"role": "user", "content": "hello"}}},
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "tagged-1", "X-Llmgw-Tags": "env=prod"})
if rec.Code != 200 {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
sent, _ := e.last.Load().([]byte)
if strings.Contains(string(sent), "metadata") || strings.Contains(string(sent), "search") {
t.Errorf("the engine was sent the tags: %s", sent)
}
if !strings.Contains(string(sent), "customer-42") {
t.Errorf("the user field, which engines know, was not passed on: %s", sent)
}
deadline := time.Now().Add(5 * time.Second)
for {
u, ok, err := rt.store.UsageByRequest(t.Context(), "tagged-1")
if err != nil {
t.Fatal(err)
}
if ok {
if u.EndUser != "customer-42" || u.Tags["feature"] != "search" || u.Tags["env"] != "prod" {
t.Errorf("recorded end user %q tags %v", u.EndUser, u.Tags)
}
break
}
if time.Now().After(deadline) {
t.Fatal("no usage recorded")
}
time.Sleep(20 * time.Millisecond)
}
bad := post(t, rt.Handler(),
map[string]any{"model": "q14", "metadata": map[string]any{"n": 1},
"messages": []any{map[string]any{"role": "user", "content": "hello"}}},
map[string]string{"Authorization": "Bearer " + secret})
if bad.Code != 400 || !strings.Contains(bad.Body.String(), "metadata.n") {
t.Errorf("a bad tag: %d %s", bad.Code, bad.Body.String())
}
}
func TestOnlyNamedTagsAreCountedAndTheirValuesBounded(t *testing.T) {
m := newTenantMetrics()
m.setTagLabels([]string{"feature"})
for i := range maxTagValues + 20 {
m.observe(&parsedRequest{Tags: map[string]string{"feature": strings.Repeat("f", i+1), "secret": "x"}}, 200, 1, 1, 0)
}
if n := testutil.CollectAndCount(m.tagRequests); n != maxTagValues+1 {
t.Errorf("%d series; want %d values and other", testutil.CollectAndCount(m.tagRequests), maxTagValues+1)
}
}
+71 -6
View File
@@ -28,6 +28,12 @@ type tenantMetrics struct {
ttft *prometheus.HistogramVec
budget *prometheus.GaugeVec
// Request tags, for the keys router.tag_labels names.
tagKeys map[string]bool
tagRequests *prometheus.CounterVec
tagTokens *prometheus.CounterVec
tagValues map[string]map[string]bool // tag -> values counted apart
mu sync.RWMutex
names map[string]string // "team:<id>" -> "acme/platform"
}
@@ -52,12 +58,68 @@ func newTenantMetrics() *tenantMetrics {
Name: "llmgw_budget_spent_ratio",
Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.",
}, []string{"scope", "id", "name", "period"}),
names: map[string]string{},
tagRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tagged_requests_total",
Help: "Requests by a request tag router.tag_labels names, its value, and outcome: ok, error, or limited.",
}, []string{"tag", "value", "outcome"}),
tagTokens: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "llmgw_tagged_tokens_total",
Help: "Tokens by a request tag router.tag_labels names, its value, and direction.",
}, []string{"tag", "value", "direction"}),
tagValues: map[string]map[string]bool{},
names: map[string]string{},
}
}
// maxTagValues is how many values of one tag are counted apart.
const maxTagValues = 100
// setTagLabels names the tags counted.
func (t *tenantMetrics) setTagLabels(keys []string) {
t.tagKeys = map[string]bool{}
for _, k := range keys {
t.tagKeys[k] = true
}
}
// tagValue is the label a tag's value is counted under: itself while there
// is room, "other" after.
func (t *tenantMetrics) tagValue(tag, value string) string {
t.mu.Lock()
defer t.mu.Unlock()
seen := t.tagValues[tag]
if seen == nil {
seen = map[string]bool{}
t.tagValues[tag] = seen
}
if seen[value] {
return value
}
if len(seen) >= maxTagValues {
return "other"
}
seen[value] = true
return value
}
func (t *tenantMetrics) observeTags(req *parsedRequest, outcome string, prompt, completion int64) {
for k, v := range req.Tags {
if !t.tagKeys[k] {
continue
}
v = t.tagValue(k, v)
t.tagRequests.WithLabelValues(k, v, outcome).Inc()
if prompt > 0 {
t.tagTokens.WithLabelValues(k, v, "prompt").Add(float64(prompt))
}
if completion > 0 {
t.tagTokens.WithLabelValues(k, v, "completion").Add(float64(completion))
}
}
}
func (t *tenantMetrics) collectors() []prometheus.Collector {
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget}
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget, t.tagRequests, t.tagTokens}
}
// tenantOf is the label a request is counted under: the name of its
@@ -91,10 +153,6 @@ func (t *tenantMetrics) tenantOf(req *parsedRequest) string {
// observe counts one finished or refused request.
func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) {
tenant := t.tenantOf(req)
if tenant == "" {
return
}
outcome := "ok"
switch {
case status == 429:
@@ -102,6 +160,13 @@ func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completi
case status >= 400:
outcome = "error"
}
if len(t.tagKeys) > 0 {
t.observeTags(req, outcome, prompt, completion)
}
tenant := t.tenantOf(req)
if tenant == "" {
return
}
t.requests.WithLabelValues(tenant, outcome).Inc()
if prompt > 0 {
t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt))