CI / test (push) Successful in 1m49s
The page imposes its own ceiling on a turn, so a playground request cannot occupy a replica for ten minutes. It sent that ceiling regardless of the model, and a model declaring a smaller max_output then answered every turn with "max_tokens is 2048, but this model allows at most 1024" — the gateway refusing its own page, over a number the person never chose and has no way to change. The ceiling is the page's to impose on itself and the model's limit is not negotiable, so it sends the smaller of the two. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
139 lines
4.5 KiB
Go
139 lines
4.5 KiB
Go
package router
|
|
|
|
import (
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// chatRouter mirrors what the control plane publishes: an endpoint advertises
|
|
// every name a client may address it by, which is the model's own name and all
|
|
// of its aliases.
|
|
func chatRouter(t *testing.T, engineURL string) (*Router, store.User) {
|
|
t.Helper()
|
|
|
|
cfg := baseConfig(config.Endpoint{
|
|
ID: "a", URL: engineURL,
|
|
Models: []string{"qwen2.5-coder-7b-instruct", "coder-7b", "coder"},
|
|
})
|
|
cfg.Models = []catalog.Model{{
|
|
Name: "qwen2.5-coder-7b-instruct", Aliases: []string{"coder-7b", "coder"},
|
|
}}
|
|
cfg.Auth.Enabled = true
|
|
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
|
rt := newTestRouter(t, cfg)
|
|
|
|
org, err := rt.store.CreateOrg(t.Context(), "acme")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rt, store.User{ID: "usr-1", ProjectID: project}
|
|
}
|
|
|
|
func TestTheChatPickerOffersAModelOnceRatherThanOncePerAlias(t *testing.T) {
|
|
// An endpoint advertises the model's name and every alias, because that is
|
|
// what the router matches a request against. Listing those strings as they
|
|
// came gave the picker "coder", "coder-7b" and the full name as three
|
|
// separate choices for one model.
|
|
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
|
rt, user := chatRouter(t, e.srv.URL)
|
|
waitForEndpoint(t, rt)
|
|
|
|
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
|
|
if len(got) != 1 {
|
|
names := make([]string, len(got))
|
|
for i, m := range got {
|
|
names[i] = m.Name
|
|
}
|
|
t.Fatalf("the picker offers %d entries for one model: %v", len(got), names)
|
|
}
|
|
// The catalog name, not whichever alias happened to be listed first: it is
|
|
// the one that appears in the model list, the usage table and the logs.
|
|
if got[0].Name != "qwen2.5-coder-7b-instruct" {
|
|
t.Errorf("offered as %q, expected the catalog name", got[0].Name)
|
|
}
|
|
}
|
|
|
|
func waitForEndpoint(t *testing.T, rt *Router) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(3 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if len(rt.reg.Snapshots("")) > 0 {
|
|
return
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Fatal("no endpoint became available")
|
|
}
|
|
|
|
func TestTheChatPageDoesNotAskForMoreThanTheModelAllows(t *testing.T) {
|
|
// The page imposes its own ceiling so a playground turn cannot occupy a
|
|
// replica for ten minutes. Sending that ceiling regardless of the model
|
|
// made the gateway refuse its own page: every turn on a model declaring a
|
|
// smaller max_output came back as a 400 about a number nobody typed.
|
|
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
|
cfg := baseConfig(config.Endpoint{
|
|
ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-coder-7b-instruct"},
|
|
})
|
|
cfg.Models = []catalog.Model{{
|
|
Name: "qwen2.5-coder-7b-instruct",
|
|
// Smaller than the page's own cap, which is the whole case.
|
|
ContextLength: 32768, MaxOutput: 1024,
|
|
}}
|
|
cfg.Auth.Enabled = true
|
|
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
|
rt := newTestRouter(t, cfg)
|
|
|
|
org, err := rt.store.CreateOrg(t.Context(), "acme")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
team, err := rt.store.CreateTeam(t.Context(), org, "platform")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
user := store.User{ID: "usr-1", ProjectID: project}
|
|
waitForEndpoint(t, rt)
|
|
|
|
body := `{"model":"qwen2.5-coder-7b-instruct","messages":[{"role":"user","content":"hi"}]}`
|
|
req := httptest.NewRequest("POST", "/chat/send", strings.NewReader(body))
|
|
rec := httptest.NewRecorder()
|
|
rt.chatSend(rec, req, user)
|
|
|
|
if rec.Code != 200 {
|
|
t.Fatalf("the page's own turn was refused: %d %s", rec.Code, rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
|
|
// The lower of the two, not the model's: a model with no ceiling, or a
|
|
// generous one, must not let a playground turn run unbounded.
|
|
if got := chatTokenCap(catalog.Model{MaxOutput: 8192}); got != chatMaxTokens {
|
|
t.Errorf("a generous model raised the page's cap to %d", got)
|
|
}
|
|
if got := chatTokenCap(catalog.Model{}); got != chatMaxTokens {
|
|
t.Errorf("a model that declares nothing gave %d", got)
|
|
}
|
|
if got := chatTokenCap(catalog.Model{MaxOutput: 1024}); got != 1024 {
|
|
t.Errorf("a model allowing 1024 got %d", got)
|
|
}
|
|
}
|