A picture in a prompt counts as what a model makes of it, not its base64
CI / test (push) Successful in 10m43s

A 1280x720 PNG sent to qwen3-vl-8b was estimated at 138 thousand prompt
tokens and refused against its 8k context before reaching the engine, which
makes about a thousand of it. Parts are counted by kind: text by its text,
a picture or a sound as a thousand tokens.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-11 20:45:06 +02:00
1 parent ac4360b3fd
commit 3858114ada
5 files changed
+66 -5

No files matched your search

+4
View File
@@ -83,6 +83,10 @@ APIs change between minors without a deprecation period.
- An on-demand replica adopted after a restart was never idle enough to
evict, so a model waiting for its card waited out its whole idle timeout;
it counts as used at adoption.
- A picture in a prompt was counted as its base64 by the context check, so
one 1280x720 PNG was "138 thousand tokens" and a vision model refused it;
a picture or a sound now counts as about a thousand.
## v0.14.3 — 2026-10-10
+18
View File
@@ -103,3 +103,21 @@ func TestStructuredContentIsCounted(t *testing.T) {
t.Errorf("structured content counted as %d bytes", got)
}
}
func TestAPictureIsCountedAsWhatAModelMakesOfItNotItsBase64(t *testing.T) {
// A 1280x720 PNG was counted as 138 thousand tokens of base64 and an 8k
// vision model refused it; its projector makes about a thousand of it.
pic := `{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 550_000) + `"}}`
b := &inferenceBody{Messages: []struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
}{{Role: "user", Content: json.RawMessage(`[{"type":"text","text":"check it"},` + pic + `,` + pic + `]`)}}}
got := promptBytes(b) / 4
if got < 2000 || got > 2200 {
t.Errorf("two pictures and a line estimated as %d tokens", got)
}
m := catalog.Model{Name: "vlm", ContextLength: 8192}
if why := checkContextFits(m, got, 300); why != "" {
t.Errorf("refused: %s", why)
}
}
+42 -3
View File
@@ -997,6 +997,23 @@ func (p *parsedRequest) contextTokens() int {
return p.PromptTokens
}
// mediaPartBytes is what one picture or sound in a prompt is counted as,
// in the bytes-over-four estimate: a thousand tokens, about what a vision
// model's projector makes of a 1280x720 picture. Engines differ, and a large
// picture can be more; the estimate errs low, as it does for text, and the
// engine has the last word.
const mediaPartBytes = 1024 * 4
// promptBytesOf is promptBytes of a request body; the body's length when it
// cannot be read as one.
func promptBytesOf(body []byte) int {
var b inferenceBody
if err := json.Unmarshal(body, &b); err != nil {
return len(body)
}
return promptBytes(&b)
}
func promptBytes(b *inferenceBody) int {
n := 0
add := func(raw json.RawMessage) {
@@ -1008,9 +1025,31 @@ func promptBytes(b *inferenceBody) int {
n += len(str)
return
}
// Not a plain string: structured content, an array of parts, an image.
// The raw JSON overstates the text slightly, which is the safe
// direction for a limit.
// An array of parts: text by its text, a picture or a sound by
// what an engine makes of one. Counted as raw JSON, a 1280x720 PNG
// was 138 thousand "tokens" of base64 and refused by an 8k model
// that takes it as about one thousand.
var parts []map[string]json.RawMessage
if err := json.Unmarshal(raw, &parts); err == nil {
for _, part := range parts {
var typ string
_ = json.Unmarshal(part["type"], &typ)
switch typ {
case "text", "input_text":
var t string
_ = json.Unmarshal(part["text"], &t)
n += len(t)
case "image_url", "input_image", "image", "input_audio":
n += mediaPartBytes
default:
b, _ := json.Marshal(part)
n += len(b)
}
}
return
}
// Anything else structured: the raw JSON overstates the text
// slightly, which is the safe direction for a limit.
n += len(raw)
}
for _, m := range b.Messages {
+1 -1
View File
@@ -224,7 +224,7 @@ func (r *Router) retrieve(ctx context.Context, w http.ResponseWriter, req *parse
return fail(&kbError{status: http.StatusBadRequest, kind: "invalid_request_error", msg: err.Error()})
}
req.UpstreamBody = body
req.PromptTokens = len(body) / 4
req.PromptTokens = promptBytesOf(body) / 4
req.knowledge = true
// The collections searched, which is who else could have had these
// passages added: the prefix cache is shared among exactly them.
+1 -1
View File
@@ -293,7 +293,7 @@ func (r *Router) loadThread(ctx context.Context, req *parsedRequest) (int, error
return 0, err
}
req.UpstreamBody = body
req.PromptTokens = len(body) / 4
req.PromptTokens = promptBytesOf(body) / 4
return dropped, nil
}