Stop the chat page asking for more tokens than the model allows

The page imposes its own ceiling on a turn, so a playground request cannot
occupy a replica for ten minutes. It sent that ceiling regardless of the
model, and a model declaring a smaller max_output then answered every turn
with "max_tokens is 2048, but this model allows at most 1024" — the gateway
refusing its own page, over a number the person never chose and has no way
to change.

The ceiling is the page's to impose on itself and the model's limit is not
negotiable, so it sends the smaller of the two.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

internal/router/chat.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/web"
@@ -65,11 +66,12 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
 	// Rebuilt rather than forwarded, so the page cannot set fields it has no
 	// business setting — a stream:false that would hang the reader, a
 	// max_tokens large enough to occupy a replica for an hour.
+	_, model := r.resolve(in.Model)
 	upstream, err := json.Marshal(map[string]any{
 		"model":      in.Model,
 		"messages":   in.Messages,
 		"stream":     true,
-		"max_tokens": chatMaxTokens,
+		"max_tokens": chatTokenCap(model),
 	})
 	if err != nil {
 		http.Error(w, "could not build the request", http.StatusInternalServerError)
@@ -98,6 +100,21 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
 // ending silently was the actual defect, and both are fixed.
 const chatMaxTokens = 2048
 
+// chatTokenCap is that bound, lowered to whatever the model allows.
+//
+// Sending the constant regardless made the gateway refuse its own page: a model
+// declaring max_output 1024 answered every turn from /chat with "max_tokens is
+// 2048, but this model allows at most 1024", which is a true sentence about a
+// number the person never chose and cannot change. The ceiling is the page's to
+// impose on itself, and the model's limit is not negotiable, so the smaller of
+// the two is the only answer that is right in both directions.
+func chatTokenCap(m catalog.Model) int {
+	if m.MaxOutput > 0 && m.MaxOutput < chatMaxTokens {
+		return m.MaxOutput
+	}
+	return chatMaxTokens
+}
+
 // principalFor builds the identity a signed-in person acts under.
 //
 // The key id is synthetic and marked as a session, because there is no key: it

internal/router/chat_models_test.go

@@ -3,6 +3,7 @@ package router
 import (
 	"net/http/httptest"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
@@ -78,3 +79,60 @@ func waitForEndpoint(t *testing.T, rt *Router) {
 	}
 	t.Fatal("no endpoint became available")
 }
+
+func TestTheChatPageDoesNotAskForMoreThanTheModelAllows(t *testing.T) {
+	// The page imposes its own ceiling so a playground turn cannot occupy a
+	// replica for ten minutes. Sending that ceiling regardless of the model
+	// made the gateway refuse its own page: every turn on a model declaring a
+	// smaller max_output came back as a 400 about a number nobody typed.
+	e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
+	cfg := baseConfig(config.Endpoint{
+		ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"},
+	})
+	cfg.Models = []catalog.Model{{
+		Name: "qwen2.5-coder-7b-instruct",
+		// Smaller than the page's own cap, which is the whole case.
+		ContextLength: 32768, MaxOutput: 1024,
+	}}
+	cfg.Auth.Enabled = true
+	cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
+	rt := newTestRouter(t, cfg)
+
+	org, err := rt.store.CreateOrg(t.Context(), "acme")
+	if err != nil {
+		t.Fatal(err)
+	}
+	team, err := rt.store.CreateTeam(t.Context(), org, "platform")
+	if err != nil {
+		t.Fatal(err)
+	}
+	project, err := rt.store.CreateProject(t.Context(), team, "gateway")
+	if err != nil {
+		t.Fatal(err)
+	}
+	user := store.User{ID: "usr-1", ProjectID: project}
+	waitForEndpoint(t, rt)
+
+	body := `{"model":"qwen2.5-coder-7b-instruct","messages":[{"role":"user","content":"hi"}]}`
+	req := httptest.NewRequest("POST", "/chat/send", strings.NewReader(body))
+	rec := httptest.NewRecorder()
+	rt.chatSend(rec, req, user)
+
+	if rec.Code != 200 {
+		t.Fatalf("the page's own turn was refused: %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
+	// The lower of the two, not the model's: a model with no ceiling, or a
+	// generous one, must not let a playground turn run unbounded.
+	if got := chatTokenCap(catalog.Model{MaxOutput: 8192}); got != chatMaxTokens {
+		t.Errorf("a generous model raised the page's cap to %d", got)
+	}
+	if got := chatTokenCap(catalog.Model{}); got != chatMaxTokens {
+		t.Errorf("a model that declares nothing gave %d", got)
+	}
+	if got := chatTokenCap(catalog.Model{MaxOutput: 1024}); got != 1024 {
+		t.Errorf("a model allowing 1024 got %d", got)
+	}
+}