A batch of embeddings is checked text by text against the model's context
llmgw-kb sends many passages in one /v1/embeddings request. Their sizes were added together and compared with the context, so a batch of passages that each fit was refused as one over-long prompt. The longest input is what has to fit; the total still counts for usage and limits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
37ad1657ee
commit
92733317a9
3 files changed
+61
-3
No files matched your search
@@ -42,7 +42,7 @@ func (r *Router) budgetFallback(p auth.Principal, req *parsedRequest) (accountin
|
||||
// answer than saying the prompt is too large, and it would be done
|
||||
// silently, to a request that was already being quietly moved.
|
||||
canonical, cand := r.resolve(name)
|
||||
if why := checkContextFits(cand, req.PromptTokens, req.MaxTokens); why != "" {
|
||||
if why := checkContextFits(cand, req.contextTokens(), req.MaxTokens); why != "" {
|
||||
continue
|
||||
}
|
||||
res, err := r.acct.Check(p, canonical, req.estimatedTokens())
|
||||
|
||||
@@ -391,7 +391,11 @@ type parsedRequest struct {
|
||||
ProfileOverride string
|
||||
PromptPrefix string
|
||||
PromptTokens int
|
||||
MaxTokens int
|
||||
// ContextTokens is what has to fit in one context: the prompt, except
|
||||
// for embeddings, where each input is embedded on its own and the
|
||||
// longest one is what a context must hold.
|
||||
ContextTokens int
|
||||
MaxTokens int
|
||||
|
||||
// ThreadID is the conversation this request continues, empty for the
|
||||
// stateless path every request took before threads existed.
|
||||
@@ -477,7 +481,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
// cannot fit should not consume a quota reservation or a queue slot on
|
||||
// its way to being refused.
|
||||
if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 {
|
||||
if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" {
|
||||
if why := checkContextFits(entry, parsed.contextTokens(), parsed.MaxTokens); why != "" {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
|
||||
r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusBadRequest, time.Since(start), nil, 0)
|
||||
return
|
||||
@@ -825,6 +829,12 @@ func (r *Router) parse(req *http.Request, path string) (*parsedRequest, error) {
|
||||
// plan §14). Measured over the whole prompt, not the routing prefix —
|
||||
// see promptBytes.
|
||||
p.PromptTokens = promptBytes(&b) / 4
|
||||
p.ContextTokens = p.PromptTokens
|
||||
if path == "/v1/embeddings" {
|
||||
if longest, ok := longestInput(b.Input); ok {
|
||||
p.ContextTokens = longest / 4
|
||||
}
|
||||
}
|
||||
|
||||
// Tags are the gateway's: an engine has no use for them, and its logs
|
||||
// are no place for what a client wrote there.
|
||||
@@ -897,6 +907,29 @@ func promptPrefix(b *inferenceBody) string {
|
||||
// bounded head of the conversation; a token estimate wants all of it, and using
|
||||
// one for the other meant TPM limits undercounted exactly the requests that
|
||||
// consume the most, and nothing could tell whether a prompt fit in a context.
|
||||
// longestInput is the length of an embeddings request's longest input, when
|
||||
// input is a list of strings: a batch of texts, each its own context.
|
||||
func longestInput(raw json.RawMessage) (int, bool) {
|
||||
var list []string
|
||||
if len(raw) == 0 || raw[0] != '[' || json.Unmarshal(raw, &list) != nil || len(list) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
n := 0
|
||||
for _, s := range list {
|
||||
n = max(n, len(s))
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
|
||||
// contextTokens is what has to fit in one context; requests built inside
|
||||
// the router set only PromptTokens.
|
||||
func (p *parsedRequest) contextTokens() int {
|
||||
if p.ContextTokens > 0 {
|
||||
return p.ContextTokens
|
||||
}
|
||||
return p.PromptTokens
|
||||
}
|
||||
|
||||
func promptBytes(b *inferenceBody) int {
|
||||
n := 0
|
||||
add := func(raw json.RawMessage) {
|
||||
|
||||
@@ -783,3 +783,28 @@ func TestAStoppedServiceAnswers503AndStillListsModels(t *testing.T) {
|
||||
t.Errorf("started again, still refused: %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestABatchOfEmbeddingsIsCheckedTextByText(t *testing.T) {
|
||||
// llmgw-kb sends many passages in one request; each is embedded on its
|
||||
// own, so their sum over the context is no reason to refuse them. One
|
||||
// passage over the context still is.
|
||||
e := newFakeEngine(t, "emb")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"emb"}})
|
||||
m := catModel("emb", nil)
|
||||
m.ContextLength = 2048
|
||||
cfg.Models = []catalog.Model{m}
|
||||
rt := newTestRouter(t, cfg)
|
||||
send := func(input []string) *httptest.ResponseRecorder {
|
||||
b, _ := json.Marshal(map[string]any{"model": "emb", "input": input})
|
||||
rec := httptest.NewRecorder()
|
||||
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/embeddings", bytes.NewReader(b)))
|
||||
return rec
|
||||
}
|
||||
passage := strings.Repeat("x", 6000) // about 1500 tokens
|
||||
if rec := send([]string{passage, passage, passage, passage}); rec.Code == http.StatusBadRequest {
|
||||
t.Errorf("four passages that each fit were refused: %s", rec.Body.String())
|
||||
}
|
||||
if rec := send([]string{"short", strings.Repeat("x", 40_000)}); rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("a passage over the context was let through: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user