From 92733317a9b58884e34db8d96d1ff2dab6e55234 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 9 Oct 2026 11:43:21 +0200 Subject: [PATCH] A batch of embeddings is checked text by text against the model's context llmgw-kb sends many passages in one /v1/embeddings request. Their sizes were added together and compared with the context, so a batch of passages that each fit was refused as one over-long prompt. The longest input is what has to fit; the total still counts for usage and limits. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/router/budget_fallback.go | 2 +- internal/router/handlers.go | 37 ++++++++++++++++++++++++++++-- internal/router/router_test.go | 25 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/internal/router/budget_fallback.go b/internal/router/budget_fallback.go index 66c8aef..0db375d 100644 --- a/internal/router/budget_fallback.go +++ b/internal/router/budget_fallback.go @@ -42,7 +42,7 @@ func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accountin // answer than saying the prompt is too large, and it would be done // silently, to a request that was already being quietly moved. canonical, cand := r.resolve(name) - if why := checkContextFits(cand, req.PromptTokens, req.MaxTokens); why != "" { + if why := checkContextFits(cand, req.contextTokens(), req.MaxTokens); why != "" { continue } res, err := r.acct.Check(p, canonical, req.estimatedTokens()) diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 0f0654c..3b8c97a 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -391,7 +391,11 @@ type parsedRequest struct { ProfileOverride string PromptPrefix string PromptTokens int - MaxTokens int + // ContextTokens is what has to fit in one context: the prompt, except + // for embeddings, where each input is embedded on its own and the + // longest one is what a context must hold. + ContextTokens int + MaxTokens int // ThreadID is the conversation this request continues, empty for the // stateless path every request took before threads existed. @@ -477,7 +481,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // cannot fit should not consume a quota reservation or a queue slot on // its way to being refused. if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 { - if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" { + if why := checkContextFits(entry, parsed.contextTokens(), parsed.MaxTokens); why != "" { writeError(w, http.StatusBadRequest, "invalid_request_error", why) r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusBadRequest, time.Since(start), nil, 0) return @@ -825,6 +829,12 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) { // plan §14). Measured over the whole prompt, not the routing prefix — // see promptBytes. p.PromptTokens = promptBytes(&b) / 4 + p.ContextTokens = p.PromptTokens + if path == "/v1/embeddings" { + if longest, ok := longestInput(b.Input); ok { + p.ContextTokens = longest / 4 + } + } // Tags are the gateway's: an engine has no use for them, and its logs // are no place for what a client wrote there. @@ -897,6 +907,29 @@ func promptPrefix(b *inferenceBody) string { // bounded head of the conversation; a token estimate wants all of it, and using // one for the other meant TPM limits undercounted exactly the requests that // consume the most, and nothing could tell whether a prompt fit in a context. +// longestInput is the length of an embeddings request's longest input, when +// input is a list of strings: a batch of texts, each its own context. +func longestInput(raw json.RawMessage) (int, bool) { + var list []string + if len(raw) == 0 || raw[0] != '[' || json.Unmarshal(raw, &list) != nil || len(list) == 0 { + return 0, false + } + n := 0 + for _, s := range list { + n = max(n, len(s)) + } + return n, true +} + +// contextTokens is what has to fit in one context; requests built inside +// the router set only PromptTokens. +func (p *parsedRequest) contextTokens() int { + if p.ContextTokens > 0 { + return p.ContextTokens + } + return p.PromptTokens +} + func promptBytes(b *inferenceBody) int { n := 0 add := func(raw json.RawMessage) { diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 8bb42a1..6f9089c 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -783,3 +783,28 @@ func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) { t.Errorf("started again, still refused: %s", rec.Body.String()) } } + +func TestABatchOfEmbeddingsIsCheckedTextByText(t *testing.T) { + // llmgw-kb sends many passages in one request; each is embedded on its + // own, so their sum over the context is no reason to refuse them. One + // passage over the context still is. + e := newFakeEngine(t, "emb") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"emb"}}) + m := catModel("emb", nil) + m.ContextLength = 2048 + cfg.Models = []catalog.Model{m} + rt := newTestRouter(t, cfg) + send := func(input []string) *httptest.ResponseRecorder { + b, _ := json.Marshal(map[string]any{"model": "emb", "input": input}) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/embeddings", bytes.NewReader(b))) + return rec + } + passage := strings.Repeat("x", 6000) // about 1500 tokens + if rec := send([]string{passage, passage, passage, passage}); rec.Code == http.StatusBadRequest { + t.Errorf("four passages that each fit were refused: %s", rec.Body.String()) + } + if rec := send([]string{"short", strings.Repeat("x", 40_000)}); rec.Code != http.StatusBadRequest { + t.Errorf("a passage over the context was let through: %d", rec.Code) + } +}