A picture in a prompt counts as what a model makes of it, not its base64
CI / test (push) Successful in 10m43s
CI / test (push) Successful in 10m43s
A 1280x720 PNG sent to qwen3-vl-8b was estimated at 138 thousand prompt tokens and refused against its 8k context before reaching the engine, which makes about a thousand of it. Parts are counted by kind: text by its text, a picture or a sound as a thousand tokens. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ac4360b3fd
commit
3858114ada
5 files changed
+66
-5
No files matched your search
@@ -83,6 +83,10 @@ APIs change between minors without a deprecation period.
|
||||
- An on-demand replica adopted after a restart was never idle enough to
|
||||
evict, so a model waiting for its card waited out its whole idle timeout;
|
||||
it counts as used at adoption.
|
||||
- A picture in a prompt was counted as its base64 by the context check, so
|
||||
one 1280x720 PNG was "138 thousand tokens" and a vision model refused it;
|
||||
a picture or a sound now counts as about a thousand.
|
||||
|
||||
|
||||
## v0.14.3 — 2026-10-10
|
||||
|
||||
|
||||
@@ -103,3 +103,21 @@ func TestStructuredContentIsCounted(t *testing.T) {
|
||||
t.Errorf("structured content counted as %d bytes", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPictureIsCountedAsWhatAModelMakesOfItNotItsBase64(t *testing.T) {
|
||||
// A 1280x720 PNG was counted as 138 thousand tokens of base64 and an 8k
|
||||
// vision model refused it; its projector makes about a thousand of it.
|
||||
pic := `{"type":"image_url","image_url":{"url":"data:image/png;base64,` + strings.Repeat("A", 550_000) + `"}}`
|
||||
b := &inferenceBody{Messages: []struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
}{{Role: "user", Content: json.RawMessage(`[{"type":"text","text":"check it"},` + pic + `,` + pic + `]`)}}}
|
||||
got := promptBytes(b) / 4
|
||||
if got < 2000 || got > 2200 {
|
||||
t.Errorf("two pictures and a line estimated as %d tokens", got)
|
||||
}
|
||||
m := catalog.Model{Name: "vlm", ContextLength: 8192}
|
||||
if why := checkContextFits(m, got, 300); why != "" {
|
||||
t.Errorf("refused: %s", why)
|
||||
}
|
||||
}
|
||||
@@ -997,6 +997,23 @@ func (p *parsedRequest) contextTokens() int {
|
||||
return p.PromptTokens
|
||||
}
|
||||
|
||||
// mediaPartBytes is what one picture or sound in a prompt is counted as,
|
||||
// in the bytes-over-four estimate: a thousand tokens, about what a vision
|
||||
// model's projector makes of a 1280x720 picture. Engines differ, and a large
|
||||
// picture can be more; the estimate errs low, as it does for text, and the
|
||||
// engine has the last word.
|
||||
const mediaPartBytes = 1024 * 4
|
||||
|
||||
// promptBytesOf is promptBytes of a request body; the body's length when it
|
||||
// cannot be read as one.
|
||||
func promptBytesOf(body []byte) int {
|
||||
var b inferenceBody
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
return len(body)
|
||||
}
|
||||
return promptBytes(&b)
|
||||
}
|
||||
|
||||
func promptBytes(b *inferenceBody) int {
|
||||
n := 0
|
||||
add := func(raw json.RawMessage) {
|
||||
@@ -1008,9 +1025,31 @@ func promptBytes(b *inferenceBody) int {
|
||||
n += len(str)
|
||||
return
|
||||
}
|
||||
// Not a plain string: structured content, an array of parts, an image.
|
||||
// The raw JSON overstates the text slightly, which is the safe
|
||||
// direction for a limit.
|
||||
// An array of parts: text by its text, a picture or a sound by
|
||||
// what an engine makes of one. Counted as raw JSON, a 1280x720 PNG
|
||||
// was 138 thousand "tokens" of base64 and refused by an 8k model
|
||||
// that takes it as about one thousand.
|
||||
var parts []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &parts); err == nil {
|
||||
for _, part := range parts {
|
||||
var typ string
|
||||
_ = json.Unmarshal(part["type"], &typ)
|
||||
switch typ {
|
||||
case "text", "input_text":
|
||||
var t string
|
||||
_ = json.Unmarshal(part["text"], &t)
|
||||
n += len(t)
|
||||
case "image_url", "input_image", "image", "input_audio":
|
||||
n += mediaPartBytes
|
||||
default:
|
||||
b, _ := json.Marshal(part)
|
||||
n += len(b)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
// Anything else structured: the raw JSON overstates the text
|
||||
// slightly, which is the safe direction for a limit.
|
||||
n += len(raw)
|
||||
}
|
||||
for _, m := range b.Messages {
|
||||
|
||||
@@ -224,7 +224,7 @@ func (r *Router) retrieve(ctx context.Context, w http.ResponseWriter, req *parse
|
||||
return fail(&kbError{status: http.StatusBadRequest, kind: "invalid_request_error", msg: err.Error()})
|
||||
}
|
||||
req.UpstreamBody = body
|
||||
req.PromptTokens = len(body) / 4
|
||||
req.PromptTokens = promptBytesOf(body) / 4
|
||||
req.knowledge = true
|
||||
// The collections searched, which is who else could have had these
|
||||
// passages added: the prefix cache is shared among exactly them.
|
||||
|
||||
@@ -293,7 +293,7 @@ func (r *Router) loadThread(ctx context.Context, req *parsedRequest) (int, error
|
||||
return 0, err
|
||||
}
|
||||
req.UpstreamBody = body
|
||||
req.PromptTokens = len(body) / 4
|
||||
req.PromptTokens = promptBytesOf(body) / 4
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user