diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 9f64255..c53e161 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -313,3 +313,24 @@ func (c *Catalog) String() string { } return b.String() } + +// Simulated reports whether this model produces synthetic text rather than +// inference output. +// +// Derived from the profiles rather than configured, because an operator who +// added a sim profile and forgot the flag would be shipping placeholder text to +// users under a real model's name. A model counts as simulated only when every +// profile is simulated: one real profile means a request can land on a real +// engine, and a model that sometimes answers for real must not be labelled as +// though it never does. +func (m Model) Simulated() bool { + if len(m.Profiles) == 0 { + return false + } + for _, p := range m.Profiles { + if p.Engine != "sim" { + return false + } + } + return true +} diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index bb630ba..c47e52d 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -233,3 +233,29 @@ func TestNamesAndModelsOrder(t *testing.T) { t.Errorf("Names order=%v", got) } } + +func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) { + // The question the flag answers is "can this request come back as real + // inference?". One real profile means yes, however many sim profiles sit + // beside it, so a mixed model must not be labelled simulated. + for _, tc := range []struct { + name string + engines []string + wantSim bool + }{ + {"all simulated", []string{"sim", "sim"}, true}, + {"one real profile", []string{"sim", "llamacpp"}, false}, + {"no simulation", []string{"llamacpp"}, false}, + {"no profiles at all", nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var m Model + for _, e := range tc.engines { + m.Profiles = append(m.Profiles, Profile{Engine: e}) + } + if got := m.Simulated(); got != tc.wantSim { + t.Errorf("Simulated() = %v, want %v", got, tc.wantSim) + } + }) + } +} diff --git a/internal/router/chat.go b/internal/router/chat.go index a99d1bc..5a6880e 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "slices" + "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -115,13 +116,18 @@ func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principa // Served rather than catalogued: offering a model with no replica produces a // 503 the reader cannot interpret, and the allowlist is applied so the picker // never shows a model the request would refuse. -func (r *Router) modelsFor(req *http.Request, user store.User) []string { +// +// Real models are ordered ahead of simulated ones so the browser's default +// selection — the first option — is always a model that answers. Sorted purely +// by name, "big-model" came first and every visitor who pressed Send without +// touching the picker got placeholder text. +func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel { principal, err := r.principalFor(req, user) if err != nil { return nil } seen := map[string]struct{}{} - var out []string + var out []web.ChatModel for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { if _, dup := seen[m]; dup { @@ -129,10 +135,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []string { } seen[m] = struct{}{} if r.permits(principal, m) { - out = append(out, m) + out = append(out, web.ChatModel{Name: m, Simulated: r.models[m].Simulated()}) } } } - slices.Sort(out) + slices.SortFunc(out, func(a, b web.ChatModel) int { + if a.Simulated != b.Simulated { + if b.Simulated { + return -1 + } + return 1 + } + return strings.Compare(a.Name, b.Name) + }) return out } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 31db5c9..366f1ff 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -499,8 +499,16 @@ func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { data := make([]map[string]any, 0, len(names)) for _, n := range names { + // A simulated model is owned by "llmgw-sim", which is what the sim + // engine already calls itself on its own /v1/models. Flattening it to + // "llmgw" here was hiding the one distinction a client needs: this + // model returns placeholder text, not inference. + owner := "llmgw" + if r.models[n].Simulated() { + owner = "llmgw-sim" + } data = append(data, map[string]any{ - "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw", + "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner, }) } writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) diff --git a/internal/router/router_test.go b/internal/router/router_test.go index d3947f8..c839f68 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -257,6 +257,46 @@ func TestModelsEndpointListsAliases(t *testing.T) { } } +func TestModelsEndpointMarksSimulatedModels(t *testing.T) { + // A client picking a model from this list has nothing else to go on. Both + // entries are OpenAI-shaped and only owned_by distinguishes the one that + // returns placeholder text, so it has to be right for aliases too. + e := newFakeEngine(t, "real") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"real", "fake"}}) + sim := catModel("fake", []string{"fake-alias"}) + sim.Profiles = []catalog.Profile{{Name: "p", Engine: "sim", GPUClass: "h100-80g", TP: 1}} + cfg.Models = []catalog.Model{catModel("real", []string{"real-alias"}), sim} + rt := newTestRouter(t, cfg) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, req) + + var out struct { + Data []struct { + ID string `json:"id"` + OwnedBy string `json:"owned_by"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + owner := map[string]string{} + for _, d := range out.Data { + owner[d.ID] = d.OwnedBy + } + for _, name := range []string{"fake", "fake-alias"} { + if owner[name] != "llmgw-sim" { + t.Errorf("%s: owned_by = %q, want llmgw-sim", name, owner[name]) + } + } + for _, name := range []string{"real", "real-alias"} { + if owner[name] != "llmgw" { + t.Errorf("%s: owned_by = %q, want llmgw", name, owner[name]) + } + } +} + func TestInjectIncludeUsagePreservesFields(t *testing.T) { // The one place the router rewrites a request body; nothing may be lost. in := []byte(`{"model":"m","stream":true,"temperature":0.7,"tools":[{"x":1}]}`) diff --git a/internal/web/chat.go b/internal/web/chat.go index eda240f..f3f3051 100644 --- a/internal/web/chat.go +++ b/internal/web/chat.go @@ -7,16 +7,25 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) +// ChatModel is one entry in the chat page's model picker. +type ChatModel struct { + Name string + // Simulated marks a model that returns synthetic text rather than + // inference output. The picker says so, because the two are + // indistinguishable from the outside until the reply arrives. + Simulated bool +} + type chatData struct { User store.User - Models []string + Models []ChatModel CSRF string Version string Now string } // Chat renders the in-browser chat for one signed-in person. -func Chat(user store.User, models []string, version string) http.HandlerFunc { +func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { noStore(w) render(w, "chat", chatData{ diff --git a/internal/web/chat_test.go b/internal/web/chat_test.go index 47b02c5..7616369 100644 --- a/internal/web/chat_test.go +++ b/internal/web/chat_test.go @@ -9,7 +9,16 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) -func chatPage(t *testing.T, models []string) string { +func chatPage(t *testing.T, names ...string) string { + t.Helper() + models := make([]ChatModel, 0, len(names)) + for _, n := range names { + models = append(models, ChatModel{Name: n}) + } + return chatPageWith(t, models) +} + +func chatPageWith(t *testing.T, models []ChatModel) string { t.Helper() h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1") r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil) @@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) { // rather than remembering: the reply is put in place with textContent and // only then walked for maths. KaTeX replaces the spans it recognises, so // nothing the model wrote is ever parsed as HTML. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") if !strings.Contains(page, "/_static/katex/katex.min.css") || !strings.Contains(page, "/_static/katex/auto-render.min.js") { @@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) { func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) { // Typesettingevery chunk would try to render half an expression on every token // and flicker through the whole reply. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") body := page[strings.Index(page, "form.onsubmit"):] renderAt := strings.Index(body, "renderMath(out)") @@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) { func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) { // $$…$$ has to be tried before $…$, or the opening $$ is read as an empty // inline expression and everything after it is wrong. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") dd := strings.Index(page, `left: "$$"`) d := strings.Index(page, `left: "$"`) if dd < 0 || d < 0 { @@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) { } func TestTheChatPageOffersWhatItWasGiven(t *testing.T) { - page := chatPage(t, []string{"alpha", "beta"}) + page := chatPage(t, "alpha", "beta") for _, m := range []string{"alpha", "beta"} { if !strings.Contains(page, `value="`+m+`"`) { t.Errorf("the picker does not offer %s", m) } } - empty := chatPage(t, nil) + empty := chatPageWith(t, nil) if strings.Contains(empty, "