diff --git a/internal/router/chat.go b/internal/router/chat.go index d972761..ea75d4a 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -10,6 +10,7 @@ import ( "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/version" "gitea.mynodes.xyz/ilker/llmgw/internal/web" @@ -65,11 +66,12 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U // Rebuilt rather than forwarded, so the page cannot set fields it has no // business setting — a stream:false that would hang the reader, a // max_tokens large enough to occupy a replica for an hour. + _, model := r.resolve(in.Model) upstream, err := json.Marshal(map[string]any{ "model": in.Model, "messages": in.Messages, "stream": true, - "max_tokens": chatMaxTokens, + "max_tokens": chatTokenCap(model), }) if err != nil { http.Error(w, "could not build the request", http.StatusInternalServerError) @@ -98,6 +100,21 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U // ending silently was the actual defect, and both are fixed. const chatMaxTokens = 2048 +// chatTokenCap is that bound, lowered to whatever the model allows. +// +// Sending the constant regardless made the gateway refuse its own page: a model +// declaring max_output 1024 answered every turn from /chat with "max_tokens is +// 2048, but this model allows at most 1024", which is a true sentence about a +// number the person never chose and cannot change. The ceiling is the page's to +// impose on itself, and the model's limit is not negotiable, so the smaller of +// the two is the only answer that is right in both directions. +func chatTokenCap(m catalog.Model) int { + if m.MaxOutput > 0 && m.MaxOutput < chatMaxTokens { + return m.MaxOutput + } + return chatMaxTokens +} + // principalFor builds the identity a signed-in person acts under. // // The key id is synthetic and marked as a session, because there is no key: it diff --git a/internal/router/chat_models_test.go b/internal/router/chat_models_test.go index a4870ca..66da593 100644 --- a/internal/router/chat_models_test.go +++ b/internal/router/chat_models_test.go @@ -3,6 +3,7 @@ package router import ( "net/http/httptest" "path/filepath" + "strings" "testing" "time" @@ -78,3 +79,60 @@ func waitForEndpoint(t *testing.T, rt *Router) { } t.Fatal("no endpoint became available") } + +func TestTheChatPageDoesNotAskForMoreThanTheModelAllows(t *testing.T) { + // The page imposes its own ceiling so a playground turn cannot occupy a + // replica for ten minutes. Sending that ceiling regardless of the model + // made the gateway refuse its own page: every turn on a model declaring a + // smaller max_output came back as a 400 about a number nobody typed. + e := newFakeEngine(t, "qwen2.5-coder-7b-instruct") + cfg := baseConfig(config.Endpoint{ + ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"}, + }) + cfg.Models = []catalog.Model{{ + Name: "qwen2.5-coder-7b-instruct", + // Smaller than the page's own cap, which is the whole case. + ContextLength: 32768, MaxOutput: 1024, + }} + cfg.Auth.Enabled = true + cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db") + rt := newTestRouter(t, cfg) + + org, err := rt.store.CreateOrg(t.Context(), "acme") + if err != nil { + t.Fatal(err) + } + team, err := rt.store.CreateTeam(t.Context(), org, "platform") + if err != nil { + t.Fatal(err) + } + project, err := rt.store.CreateProject(t.Context(), team, "gateway") + if err != nil { + t.Fatal(err) + } + user := store.User{ID: "usr-1", ProjectID: project} + waitForEndpoint(t, rt) + + body := `{"model":"qwen2.5-coder-7b-instruct","messages":[{"role":"user","content":"hi"}]}` + req := httptest.NewRequest("POST", "/chat/send", strings.NewReader(body)) + rec := httptest.NewRecorder() + rt.chatSend(rec, req, user) + + if rec.Code != 200 { + t.Fatalf("the page's own turn was refused: %d %s", rec.Code, rec.Body.String()) + } +} + +func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) { + // The lower of the two, not the model's: a model with no ceiling, or a + // generous one, must not let a playground turn run unbounded. + if got := chatTokenCap(catalog.Model{MaxOutput: 8192}); got != chatMaxTokens { + t.Errorf("a generous model raised the page's cap to %d", got) + } + if got := chatTokenCap(catalog.Model{}); got != chatMaxTokens { + t.Errorf("a model that declares nothing gave %d", got) + } + if got := chatTokenCap(catalog.Model{MaxOutput: 1024}); got != 1024 { + t.Errorf("a model allowing 1024 got %d", got) + } +}