Stop the chat page asking for more tokens than the model allows
CI / test (push) Successful in 1m49s

The page imposes its own ceiling on a turn, so a playground request cannot
occupy a replica for ten minutes. It sent that ceiling regardless of the
model, and a model declaring a smaller max_output then answered every turn
with "max_tokens is 2048, but this model allows at most 1024" — the gateway
refusing its own page, over a number the person never chose and has no way
to change.

The ceiling is the page's to impose on itself and the model's limit is not
negotiable, so it sends the smaller of the two.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-28 00:48:07 +02:00
1 parent c848b2c73a
commit f80eef9fa6
2 files changed
+76 -1

No files matched your search

+58
View File
@@ -3,6 +3,7 @@ package router
import (
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
@@ -78,3 +79,60 @@ func waitForEndpoint(t *testing.T, rt *Router) {
}
t.Fatal("no endpoint became available")
}
func TestTheChatPageDoesNotAskForMoreThanTheModelAllows(t *testing.T) {
// The page imposes its own ceiling so a playground turn cannot occupy a
// replica for ten minutes. Sending that ceiling regardless of the model
// made the gateway refuse its own page: every turn on a model declaring a
// smaller max_output came back as a 400 about a number nobody typed.
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
cfg := baseConfig(config.Endpoint{
ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"},
})
cfg.Models = []catalog.Model{{
Name: "qwen2.5-coder-7b-instruct",
// Smaller than the page's own cap, which is the whole case.
ContextLength: 32768, MaxOutput: 1024,
}}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
rt := newTestRouter(t, cfg)
org, err := rt.store.CreateOrg(t.Context(), "acme")
if err != nil {
t.Fatal(err)
}
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
if err != nil {
t.Fatal(err)
}
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
if err != nil {
t.Fatal(err)
}
user := store.User{ID: "usr-1", ProjectID: project}
waitForEndpoint(t, rt)
body := `{"model":"qwen2.5-coder-7b-instruct","messages":[{"role":"user","content":"hi"}]}`
req := httptest.NewRequest("POST", "/chat/send", strings.NewReader(body))
rec := httptest.NewRecorder()
rt.chatSend(rec, req, user)
if rec.Code != 200 {
t.Fatalf("the page's own turn was refused: %d %s", rec.Code, rec.Body.String())
}
}
func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
// The lower of the two, not the model's: a model with no ceiling, or a
// generous one, must not let a playground turn run unbounded.
if got := chatTokenCap(catalog.Model{MaxOutput: 8192}); got != chatMaxTokens {
t.Errorf("a generous model raised the page's cap to %d", got)
}
if got := chatTokenCap(catalog.Model{}); got != chatMaxTokens {
t.Errorf("a model that declares nothing gave %d", got)
}
if got := chatTokenCap(catalog.Model{MaxOutput: 1024}); got != 1024 {
t.Errorf("a model allowing 1024 got %d", got)
}
}