A batch of embeddings is checked text by text against the model's context

llmgw-kb sends many passages in one /v1/embeddings request. Their sizes
were added together and compared with the context, so a batch of passages
that each fit was refused as one over-long prompt. The longest input is
what has to fit; the total still counts for usage and limits.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-09 11:43:21 +02:00
1 parent 37ad1657ee
commit 92733317a9
3 files changed
+61 -3

No files matched your search

+1 -1
View File
@@ -42,7 +42,7 @@ func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accountin
// answer than saying the prompt is too large, and it would be done
// silently, to a request that was already being quietly moved.
canonical, cand := r.resolve(name)
if why := checkContextFits(cand, req.PromptTokens, req.MaxTokens); why != "" {
if why := checkContextFits(cand, req.contextTokens(), req.MaxTokens); why != "" {
continue
}
res, err := r.acct.Check(p, canonical, req.estimatedTokens())
+35 -2
View File
@@ -391,7 +391,11 @@ type parsedRequest struct {
ProfileOverride string
PromptPrefix string
PromptTokens int
MaxTokens int
// ContextTokens is what has to fit in one context: the prompt, except
// for embeddings, where each input is embedded on its own and the
// longest one is what a context must hold.
ContextTokens int
MaxTokens int
// ThreadID is the conversation this request continues, empty for the
// stateless path every request took before threads existed.
@@ -477,7 +481,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
// cannot fit should not consume a quota reservation or a queue slot on
// its way to being refused.
if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 {
if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" {
if why := checkContextFits(entry, parsed.contextTokens(), parsed.MaxTokens); why != "" {
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusBadRequest, time.Since(start), nil, 0)
return
@@ -825,6 +829,12 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
// plan §14). Measured over the whole prompt, not the routing prefix —
// see promptBytes.
p.PromptTokens = promptBytes(&b) / 4
p.ContextTokens = p.PromptTokens
if path == "/v1/embeddings" {
if longest, ok := longestInput(b.Input); ok {
p.ContextTokens = longest / 4
}
}
// Tags are the gateway's: an engine has no use for them, and its logs
// are no place for what a client wrote there.
@@ -897,6 +907,29 @@ func promptPrefix(b *inferenceBody) string {
// bounded head of the conversation; a token estimate wants all of it, and using
// one for the other meant TPM limits undercounted exactly the requests that
// consume the most, and nothing could tell whether a prompt fit in a context.
// longestInput is the length of an embeddings request's longest input, when
// input is a list of strings: a batch of texts, each its own context.
func longestInput(raw json.RawMessage) (int, bool) {
var list []string
if len(raw) == 0 || raw[0] != '[' || json.Unmarshal(raw, &list) != nil || len(list) == 0 {
return 0, false
}
n := 0
for _, s := range list {
n = max(n, len(s))
}
return n, true
}
// contextTokens is what has to fit in one context; requests built inside
// the router set only PromptTokens.
func (p *parsedRequest) contextTokens() int {
if p.ContextTokens > 0 {
return p.ContextTokens
}
return p.PromptTokens
}
func promptBytes(b *inferenceBody) int {
n := 0
add := func(raw json.RawMessage) {
+25
View File
@@ -783,3 +783,28 @@ func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) {
t.Errorf("started again, still refused: %s", rec.Body.String())
}
}
func TestABatchOfEmbeddingsIsCheckedTextByText(t *testing.T) {
// llmgw-kb sends many passages in one request; each is embedded on its
// own, so their sum over the context is no reason to refuse them. One
// passage over the context still is.
e := newFakeEngine(t, "emb")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"emb"}})
m := catModel("emb", nil)
m.ContextLength = 2048
cfg.Models = []catalog.Model{m}
rt := newTestRouter(t, cfg)
send := func(input []string) *httptest.ResponseRecorder {
b, _ := json.Marshal(map[string]any{"model": "emb", "input": input})
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/embeddings", bytes.NewReader(b)))
return rec
}
passage := strings.Repeat("x", 6000) // about 1500 tokens
if rec := send([]string{passage, passage, passage, passage}); rec.Code == http.StatusBadRequest {
t.Errorf("four passages that each fit were refused: %s", rec.Body.String())
}
if rec := send([]string{"short", strings.Repeat("x", 40_000)}); rec.Code != http.StatusBadRequest {
t.Errorf("a passage over the context was let through: %d", rec.Code)
}
}