diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c806c6..f49cd57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,10 @@ APIs change between minors without a deprecation period. - An on-demand replica adopted after a restart was never idle enough to evict, so a model waiting for its card waited out its whole idle timeout; it counts as used at adoption. +- A picture in a prompt was counted as its base64 by the context check, so + one 1280x720 PNG was "138 thousand tokens" and a vision model refused it; + a picture or a sound now counts as about a thousand. + ## v0.14.3 — 2026-10-10 diff --git a/internal/router/context_limit_test.go b/internal/router/context_limit_test.go index 4295b0a..7da430a 100644 --- a/internal/router/context_limit_test.go +++ b/internal/router/context_limit_test.go @@ -103,3 +103,21 @@ func TestStructuredContentIsCounted(t *testing.T) { t.Errorf("structured content counted as %d bytes", got) } } + +func TestAPictureIsCountedAsWhatAModelMakesOfItNotItsBase64(t *testing.T) { + // A 1280x720 PNG was counted as 138 thousand tokens of base64 and an 8k + // vision model refused it; its projector makes about a thousand of it. + pic := `{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 550_000) + `"}}` + b := &inferenceBody{Messages: []struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` + }{{Role: "user", Content: json.RawMessage(`[{"type":"text","text":"check it"},` + pic + `,` + pic + `]`)}}} + got := promptBytes(b) / 4 + if got < 2000 || got > 2200 { + t.Errorf("two pictures and a line estimated as %d tokens", got) + } + m := catalog.Model{Name: "vlm", ContextLength: 8192} + if why := checkContextFits(m, got, 300); why != "" { + t.Errorf("refused: %s", why) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 4c07a6b..b9ca6a3 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -997,6 +997,23 @@ func (p *parsedRequest) contextTokens() int { return p.PromptTokens } +// mediaPartBytes is what one picture or sound in a prompt is counted as, +// in the bytes-over-four estimate: a thousand tokens, about what a vision +// model's projector makes of a 1280x720 picture. Engines differ, and a large +// picture can be more; the estimate errs low, as it does for text, and the +// engine has the last word. +const mediaPartBytes = 1024 * 4 + +// promptBytesOf is promptBytes of a request body; the body's length when it +// cannot be read as one. +func promptBytesOf(body []byte) int { + var b inferenceBody + if err := json.Unmarshal(body, &b); err != nil { + return len(body) + } + return promptBytes(&b) +} + func promptBytes(b *inferenceBody) int { n := 0 add := func(raw json.RawMessage) { @@ -1008,9 +1025,31 @@ func promptBytes(b *inferenceBody) int { n += len(str) return } - // Not a plain string: structured content, an array of parts, an image. - // The raw JSON overstates the text slightly, which is the safe - // direction for a limit. + // An array of parts: text by its text, a picture or a sound by + // what an engine makes of one. Counted as raw JSON, a 1280x720 PNG + // was 138 thousand "tokens" of base64 and refused by an 8k model + // that takes it as about one thousand. + var parts []map[string]json.RawMessage + if err := json.Unmarshal(raw, &parts); err == nil { + for _, part := range parts { + var typ string + _ = json.Unmarshal(part["type"], &typ) + switch typ { + case "text", "input_text": + var t string + _ = json.Unmarshal(part["text"], &t) + n += len(t) + case "image_url", "input_image", "image", "input_audio": + n += mediaPartBytes + default: + b, _ := json.Marshal(part) + n += len(b) + } + } + return + } + // Anything else structured: the raw JSON overstates the text + // slightly, which is the safe direction for a limit. n += len(raw) } for _, m := range b.Messages { diff --git a/internal/router/rag.go b/internal/router/rag.go index bb05620..41b7090 100644 --- a/internal/router/rag.go +++ b/internal/router/rag.go @@ -224,7 +224,7 @@ func (r *Router) retrieve(ctx context.Context, w http.ResponseWriter, req *parse return fail(&kbError{status: http.StatusBadRequest, kind: "invalid_request_error", msg: err.Error()}) } req.UpstreamBody = body - req.PromptTokens = len(body) / 4 + req.PromptTokens = promptBytesOf(body) / 4 req.knowledge = true // The collections searched, which is who else could have had these // passages added: the prefix cache is shared among exactly them. diff --git a/internal/router/threads.go b/internal/router/threads.go index c7d5c37..7c2a46f 100644 --- a/internal/router/threads.go +++ b/internal/router/threads.go @@ -293,7 +293,7 @@ func (r *Router) loadThread(ctx context.Context, req *parsedRequest) (int, error return 0, err } req.UpstreamBody = body - req.PromptTokens = len(body) / 4 + req.PromptTokens = promptBytesOf(body) / 4 return dropped, nil }