Requests can be tagged: metadata, x-llmgw-tags and the user field on the usage record, and counted by the tags router.tag_labels names
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
29a7347df2
19 files changed
+436
-17
No files matched your search
@@ -135,6 +135,7 @@ func (r *Router) refuseGuarded(w http.ResponseWriter, req *parsedRequest, v guar
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: req.Model, Status: status,
|
||||
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID, QueueMs: req.QueueMs,
|
||||
EndUser: req.EndUser, Tags: req.Tags,
|
||||
}
|
||||
guardFields(&u, v, guard.Verdict{})
|
||||
// Settled at nothing: the estimate the limit check charged is
|
||||
|
||||
@@ -280,6 +280,10 @@ func (r *Router) Handler() http.Handler {
|
||||
// business interpreting.
|
||||
type parsedRequest struct {
|
||||
Path string
|
||||
// Tags are the client's metadata and x-llmgw-tags; EndUser is OpenAI's
|
||||
// user field. Both are recorded with the usage, never sent on.
|
||||
Tags map[string]string
|
||||
EndUser string
|
||||
// Model is what will be dispatched, which is not always what was asked
|
||||
// for: a caller out of budget may be moved to a model they can afford.
|
||||
Model string
|
||||
@@ -338,9 +342,10 @@ type parsedRequest struct {
|
||||
// inferenceBody covers the fields the router reads across all inference
|
||||
// endpoints. Unknown fields are ignored here and preserved in UpstreamBody.
|
||||
type inferenceBody struct {
|
||||
Model string `json:"model"`
|
||||
Stream bool `json:"stream"`
|
||||
User string `json:"user"`
|
||||
Model string `json:"model"`
|
||||
Stream bool `json:"stream"`
|
||||
User string `json:"user"`
|
||||
Metadata json.RawMessage `json:"metadata"`
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
@@ -587,6 +592,7 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
LatencyMs: took.Milliseconds(),
|
||||
RequestID: req.RequestID, Replica: res.endpointID,
|
||||
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
|
||||
EndUser: req.EndUser, Tags: req.Tags,
|
||||
}
|
||||
guardFields(&u, req.guardIn, res.guard)
|
||||
if u.ServedModel == "" {
|
||||
@@ -625,6 +631,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: req.Model, Status: http.StatusTooManyRequests,
|
||||
LatencyMs: time.Since(start).Milliseconds(), RequestID: req.RequestID,
|
||||
EndUser: req.EndUser, Tags: req.Tags,
|
||||
})
|
||||
r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start))
|
||||
}
|
||||
@@ -644,7 +651,14 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
|
||||
tags, err := requestTags(b.Metadata, req.Header.Get(tagsHeader))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
p := &parsedRequest{
|
||||
Tags: tags,
|
||||
EndUser: endUser(b.User),
|
||||
Path: path,
|
||||
Model: b.Model,
|
||||
RequestedModel: b.Model,
|
||||
@@ -680,10 +694,16 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
// see promptBytes.
|
||||
p.PromptTokens = promptBytes(&b) / 4
|
||||
|
||||
// Tags are the gateway's: an engine has no use for them, and its logs
|
||||
// are no place for what a client wrote there.
|
||||
if len(b.Metadata) > 0 {
|
||||
p.UpstreamBody = withoutField(p.UpstreamBody, "metadata")
|
||||
}
|
||||
|
||||
// Ask the engine for usage the client did not request, so accounting works
|
||||
// regardless of client behaviour; the extra chunk is dropped on the way back.
|
||||
if r.cfg.InjectUsage && b.Stream && (b.StreamOptions == nil || !b.StreamOptions.IncludeUsage) {
|
||||
if injected, ok := injectIncludeUsage(body); ok {
|
||||
if injected, ok := injectIncludeUsage(p.UpstreamBody); ok {
|
||||
p.UpstreamBody = injected
|
||||
p.SuppressUsageChunk = true
|
||||
}
|
||||
|
||||
@@ -209,6 +209,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
// guessing at it.
|
||||
metrics := newMetrics()
|
||||
metrics.tenant.level = cfg.Router.TenantLabel
|
||||
metrics.tenant.setTagLabels(cfg.Router.TagLabels)
|
||||
var prefixScorer *scheduler.PrefixScorer
|
||||
if cfg.Router.Prefix.Enabled {
|
||||
prefixScorer = scheduler.NewPrefixScorer(
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// What a client may say about a request, for attributing spend inside a
|
||||
// project: OpenAI's metadata (at most 16 keys, as OpenAI allows) and the
|
||||
// x-llmgw-tags header, which wins for a key both give.
|
||||
const (
|
||||
tagsHeader = "x-llmgw-tags"
|
||||
maxTags = 16
|
||||
maxTagValue = 512
|
||||
maxEndUser = 256
|
||||
)
|
||||
|
||||
var tagKey = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`)
|
||||
|
||||
// requestTags reads metadata from the body and x-llmgw-tags ("team=search,
|
||||
// feature=autocomplete") from the header. An invalid one is the client's
|
||||
// mistake and is said, not dropped: a tag silently lost is spend that
|
||||
// cannot be found again.
|
||||
func requestTags(metadata json.RawMessage, header string) (map[string]string, error) {
|
||||
tags := map[string]string{}
|
||||
if len(metadata) > 0 && string(metadata) != "null" {
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(metadata, &m); err != nil {
|
||||
return nil, fmt.Errorf("metadata is an object of strings")
|
||||
}
|
||||
for k, v := range m {
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("metadata.%s is not a string", k)
|
||||
}
|
||||
tags[k] = s
|
||||
}
|
||||
}
|
||||
if header = strings.TrimSpace(header); header != "" {
|
||||
for _, part := range strings.Split(header, ",") {
|
||||
k, v, ok := strings.Cut(strings.TrimSpace(part), "=")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s: %q is not key=value", tagsHeader, part)
|
||||
}
|
||||
tags[strings.TrimSpace(k)] = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
if len(tags) > maxTags {
|
||||
return nil, fmt.Errorf("%d tags; at most %d", len(tags), maxTags)
|
||||
}
|
||||
for k, v := range tags {
|
||||
if !tagKey.MatchString(k) {
|
||||
return nil, fmt.Errorf("tag %q: a key is 1 to 64 letters, digits, '_', '.' or '-'", k)
|
||||
}
|
||||
if len(v) > maxTagValue {
|
||||
return nil, fmt.Errorf("tag %s: the value is longer than %d bytes", k, maxTagValue)
|
||||
}
|
||||
}
|
||||
if len(tags) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// endUser is OpenAI's user field as kept: bounded, since it is the client's.
|
||||
func endUser(u string) string {
|
||||
if len(u) > maxEndUser {
|
||||
return u[:maxEndUser]
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// withoutField removes a top-level field from a JSON body, or returns it
|
||||
// unchanged when the field is not there or the body is not an object.
|
||||
func withoutField(body []byte, field string) []byte {
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return body
|
||||
}
|
||||
if _, ok := m[field]; !ok {
|
||||
return body
|
||||
}
|
||||
delete(m, field)
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return body
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
func TestTagsComeFromMetadataAndTheHeader(t *testing.T) {
|
||||
got, err := requestTags(json.RawMessage(`{"feature":"search","team":"a"}`), "team=b, env=prod")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 || got["team"] != "b" || got["feature"] != "search" || got["env"] != "prod" {
|
||||
t.Errorf("%v", got)
|
||||
}
|
||||
if got, err := requestTags(nil, ""); got != nil || err != nil {
|
||||
t.Errorf("none: %v %v", got, err)
|
||||
}
|
||||
many := map[string]string{}
|
||||
for i := range 17 {
|
||||
many[strings.Repeat("k", i+1)] = "v"
|
||||
}
|
||||
b, _ := json.Marshal(many)
|
||||
for name, c := range map[string]struct {
|
||||
meta string
|
||||
header string
|
||||
}{
|
||||
"not strings": {`{"n": 1}`, ""},
|
||||
"not an object": {`["a"]`, ""},
|
||||
"no equals": {"", "feature"},
|
||||
"bad key": {`{"a b": "x"}`, ""},
|
||||
"long value": {`{"a": "` + strings.Repeat("x", 513) + `"}`, ""},
|
||||
"too many": {string(b), ""},
|
||||
} {
|
||||
if _, err := requestTags(json.RawMessage(c.meta), c.header); err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTagsAreRecordedAndNotSentToTheEngine(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
rt.m.tenant.setTagLabels([]string{"feature"})
|
||||
|
||||
rec := post(t, rt.Handler(),
|
||||
map[string]any{"model": "q14", "user": "customer-42", "metadata": map[string]string{"feature": "search"},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "tagged-1", "X-Llmgw-Tags": "env=prod"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
sent, _ := e.last.Load().([]byte)
|
||||
if strings.Contains(string(sent), "metadata") || strings.Contains(string(sent), "search") {
|
||||
t.Errorf("the engine was sent the tags: %s", sent)
|
||||
}
|
||||
if !strings.Contains(string(sent), "customer-42") {
|
||||
t.Errorf("the user field, which engines know, was not passed on: %s", sent)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
u, ok, err := rt.store.UsageByRequest(t.Context(), "tagged-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
if u.EndUser != "customer-42" || u.Tags["feature"] != "search" || u.Tags["env"] != "prod" {
|
||||
t.Errorf("recorded end user %q tags %v", u.EndUser, u.Tags)
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("no usage recorded")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
|
||||
bad := post(t, rt.Handler(),
|
||||
map[string]any{"model": "q14", "metadata": map[string]any{"n": 1},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret})
|
||||
if bad.Code != 400 || !strings.Contains(bad.Body.String(), "metadata.n") {
|
||||
t.Errorf("a bad tag: %d %s", bad.Code, bad.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyNamedTagsAreCountedAndTheirValuesBounded(t *testing.T) {
|
||||
m := newTenantMetrics()
|
||||
m.setTagLabels([]string{"feature"})
|
||||
for i := range maxTagValues + 20 {
|
||||
m.observe(&parsedRequest{Tags: map[string]string{"feature": strings.Repeat("f", i+1), "secret": "x"}}, 200, 1, 1, 0)
|
||||
}
|
||||
if n := testutil.CollectAndCount(m.tagRequests); n != maxTagValues+1 {
|
||||
t.Errorf("%d series; want %d values and other", testutil.CollectAndCount(m.tagRequests), maxTagValues+1)
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,12 @@ type tenantMetrics struct {
|
||||
ttft *prometheus.HistogramVec
|
||||
budget *prometheus.GaugeVec
|
||||
|
||||
// Request tags, for the keys router.tag_labels names.
|
||||
tagKeys map[string]bool
|
||||
tagRequests *prometheus.CounterVec
|
||||
tagTokens *prometheus.CounterVec
|
||||
tagValues map[string]map[string]bool // tag -> values counted apart
|
||||
|
||||
mu sync.RWMutex
|
||||
names map[string]string // "team:<id>" -> "acme/platform"
|
||||
}
|
||||
@@ -52,12 +58,68 @@ func newTenantMetrics() *tenantMetrics {
|
||||
Name: "llmgw_budget_spent_ratio",
|
||||
Help: "Spend this period over the budget, for every organization, team, project and person given one. 1 is spent.",
|
||||
}, []string{"scope", "id", "name", "period"}),
|
||||
names: map[string]string{},
|
||||
tagRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "llmgw_tagged_requests_total",
|
||||
Help: "Requests by a request tag router.tag_labels names, its value, and outcome: ok, error, or limited.",
|
||||
}, []string{"tag", "value", "outcome"}),
|
||||
tagTokens: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "llmgw_tagged_tokens_total",
|
||||
Help: "Tokens by a request tag router.tag_labels names, its value, and direction.",
|
||||
}, []string{"tag", "value", "direction"}),
|
||||
tagValues: map[string]map[string]bool{},
|
||||
names: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
// maxTagValues is how many values of one tag are counted apart.
|
||||
const maxTagValues = 100
|
||||
|
||||
// setTagLabels names the tags counted.
|
||||
func (t *tenantMetrics) setTagLabels(keys []string) {
|
||||
t.tagKeys = map[string]bool{}
|
||||
for _, k := range keys {
|
||||
t.tagKeys[k] = true
|
||||
}
|
||||
}
|
||||
|
||||
// tagValue is the label a tag's value is counted under: itself while there
|
||||
// is room, "other" after.
|
||||
func (t *tenantMetrics) tagValue(tag, value string) string {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
seen := t.tagValues[tag]
|
||||
if seen == nil {
|
||||
seen = map[string]bool{}
|
||||
t.tagValues[tag] = seen
|
||||
}
|
||||
if seen[value] {
|
||||
return value
|
||||
}
|
||||
if len(seen) >= maxTagValues {
|
||||
return "other"
|
||||
}
|
||||
seen[value] = true
|
||||
return value
|
||||
}
|
||||
|
||||
func (t *tenantMetrics) observeTags(req *parsedRequest, outcome string, prompt, completion int64) {
|
||||
for k, v := range req.Tags {
|
||||
if !t.tagKeys[k] {
|
||||
continue
|
||||
}
|
||||
v = t.tagValue(k, v)
|
||||
t.tagRequests.WithLabelValues(k, v, outcome).Inc()
|
||||
if prompt > 0 {
|
||||
t.tagTokens.WithLabelValues(k, v, "prompt").Add(float64(prompt))
|
||||
}
|
||||
if completion > 0 {
|
||||
t.tagTokens.WithLabelValues(k, v, "completion").Add(float64(completion))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *tenantMetrics) collectors() []prometheus.Collector {
|
||||
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget}
|
||||
return []prometheus.Collector{t.requests, t.tokens, t.ttft, t.budget, t.tagRequests, t.tagTokens}
|
||||
}
|
||||
|
||||
// tenantOf is the label a request is counted under: the name of its
|
||||
@@ -91,10 +153,6 @@ func (t *tenantMetrics) tenantOf(req *parsedRequest) string {
|
||||
|
||||
// observe counts one finished or refused request.
|
||||
func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completion int64, ttft time.Duration) {
|
||||
tenant := t.tenantOf(req)
|
||||
if tenant == "" {
|
||||
return
|
||||
}
|
||||
outcome := "ok"
|
||||
switch {
|
||||
case status == 429:
|
||||
@@ -102,6 +160,13 @@ func (t *tenantMetrics) observe(req *parsedRequest, status int, prompt, completi
|
||||
case status >= 400:
|
||||
outcome = "error"
|
||||
}
|
||||
if len(t.tagKeys) > 0 {
|
||||
t.observeTags(req, outcome, prompt, completion)
|
||||
}
|
||||
tenant := t.tenantOf(req)
|
||||
if tenant == "" {
|
||||
return
|
||||
}
|
||||
t.requests.WithLabelValues(tenant, outcome).Inc()
|
||||
if prompt > 0 {
|
||||
t.tokens.WithLabelValues(tenant, "prompt").Add(float64(prompt))
|
||||
|
||||
Reference in new issue
Block a user