Files
llmgw/internal/router/chat_models_test.go
T
Ilker ManapandClaude Opus 5 f80eef9fa6
CI / test (push) Successful in 1m49s
Stop the chat page asking for more tokens than the model allows
The page imposes its own ceiling on a turn, so a playground request cannot
occupy a replica for ten minutes. It sent that ceiling regardless of the
model, and a model declaring a smaller max_output then answered every turn
with "max_tokens is 2048, but this model allows at most 1024" — the gateway
refusing its own page, over a number the person never chose and has no way
to change.

The ceiling is the page's to impose on itself and the model's limit is not
negotiable, so it sends the smaller of the two.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-28 00:48:07 +02:00

139 lines
4.5 KiB
Go

package router
import (
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// chatRouter mirrors what the control plane publishes: an endpoint advertises
// every name a client may address it by, which is the model's own name and all
// of its aliases.
func chatRouter(t *testing.T, engineURL string) (*Router, store.User) {
t.Helper()
cfg := baseConfig(config.Endpoint{
ID: "a", URL: engineURL,
Models: []string{"qwen2.5-coder-7b-instruct", "coder-7b", "coder"},
})
cfg.Models = []catalog.Model{{
Name: "qwen2.5-coder-7b-instruct", Aliases: []string{"coder-7b", "coder"},
}}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
rt := newTestRouter(t, cfg)
org, err := rt.store.CreateOrg(t.Context(), "acme")
if err != nil {
t.Fatal(err)
}
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
if err != nil {
t.Fatal(err)
}
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
if err != nil {
t.Fatal(err)
}
return rt, store.User{ID: "usr-1", ProjectID: project}
}
func TestTheChatPickerOffersAModelOnceRatherThanOncePerAlias(t *testing.T) {
// An endpoint advertises the model's name and every alias, because that is
// what the router matches a request against. Listing those strings as they
// came gave the picker "coder", "coder-7b" and the full name as three
// separate choices for one model.
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
rt, user := chatRouter(t, e.srv.URL)
waitForEndpoint(t, rt)
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
if len(got) != 1 {
names := make([]string, len(got))
for i, m := range got {
names[i] = m.Name
}
t.Fatalf("the picker offers %d entries for one model: %v", len(got), names)
}
// The catalog name, not whichever alias happened to be listed first: it is
// the one that appears in the model list, the usage table and the logs.
if got[0].Name != "qwen2.5-coder-7b-instruct" {
t.Errorf("offered as %q, expected the catalog name", got[0].Name)
}
}
func waitForEndpoint(t *testing.T, rt *Router) {
t.Helper()
deadline := time.Now().Add(3 * time.Second)
for time.Now().Before(deadline) {
if len(rt.reg.Snapshots("")) > 0 {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("no endpoint became available")
}
func TestTheChatPageDoesNotAskForMoreThanTheModelAllows(t *testing.T) {
// The page imposes its own ceiling so a playground turn cannot occupy a
// replica for ten minutes. Sending that ceiling regardless of the model
// made the gateway refuse its own page: every turn on a model declaring a
// smaller max_output came back as a 400 about a number nobody typed.
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
cfg := baseConfig(config.Endpoint{
ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"},
})
cfg.Models = []catalog.Model{{
Name: "qwen2.5-coder-7b-instruct",
// Smaller than the page's own cap, which is the whole case.
ContextLength: 32768, MaxOutput: 1024,
}}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
rt := newTestRouter(t, cfg)
org, err := rt.store.CreateOrg(t.Context(), "acme")
if err != nil {
t.Fatal(err)
}
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
if err != nil {
t.Fatal(err)
}
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
if err != nil {
t.Fatal(err)
}
user := store.User{ID: "usr-1", ProjectID: project}
waitForEndpoint(t, rt)
body := `{"model":"qwen2.5-coder-7b-instruct","messages":[{"role":"user","content":"hi"}]}`
req := httptest.NewRequest("POST", "/chat/send", strings.NewReader(body))
rec := httptest.NewRecorder()
rt.chatSend(rec, req, user)
if rec.Code != 200 {
t.Fatalf("the page's own turn was refused: %d %s", rec.Code, rec.Body.String())
}
}
func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
// The lower of the two, not the model's: a model with no ceiling, or a
// generous one, must not let a playground turn run unbounded.
if got := chatTokenCap(catalog.Model{MaxOutput: 8192}); got != chatMaxTokens {
t.Errorf("a generous model raised the page's cap to %d", got)
}
if got := chatTokenCap(catalog.Model{}); got != chatMaxTokens {
t.Errorf("a model that declares nothing gave %d", got)
}
if got := chatTokenCap(catalog.Model{MaxOutput: 1024}); got != 1024 {
t.Errorf("a model allowing 1024 got %d", got)
}
}