diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 9f64255f..c53e161d 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -313,3 +313,24 @@ func (c *Catalog) String() string { } return b.String() } + +// Simulated reports whether this model produces synthetic text rather than +// inference output. +// +// Derived from the profiles rather than configured, because an operator who +// added a sim profile and forgot the flag would be shipping placeholder text to +// users under a real model's name. A model counts as simulated only when every +// profile is simulated: one real profile means a request can land on a real +// engine, and a model that sometimes answers for real must not be labelled as +// though it never does. +func (m Model) Simulated() bool { + if len(m.Profiles) == 0 { + return false + } + for _, p := range m.Profiles { + if p.Engine != "sim" { + return false + } + } + return true +} diff --git a/internal/control/catalog/catalog_test.go b/internal/control/catalog/catalog_test.go index bb630bab..c47e52d1 100644 --- a/internal/control/catalog/catalog_test.go +++ b/internal/control/catalog/catalog_test.go @@ -233,3 +233,29 @@ func TestNamesAndModelsOrder(t *testing.T) { t.Errorf("Names order=%v", got) } } + +func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) { + // The question the flag answers is "can this request come back as real + // inference?". One real profile means yes, however many sim profiles sit + // beside it, so a mixed model must not be labelled simulated. + for _, tc := range []struct { + name string + engines []string + wantSim bool + }{ + {"all simulated", []string{"sim", "sim"}, true}, + {"one real profile", []string{"sim", "llamacpp"}, false}, + {"no simulation", []string{"llamacpp"}, false}, + {"no profiles at all", nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var m Model + for _, e := range tc.engines { + m.Profiles = append(m.Profiles, Profile{Engine: e}) + } + if got := m.Simulated(); got != tc.wantSim { + t.Errorf("Simulated() = %v, want %v", got, tc.wantSim) + } + }) + } +} diff --git a/internal/router/chat.go b/internal/router/chat.go index a99d1bc5..5a6880e1 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "slices" + "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -115,13 +116,18 @@ func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principa // Served rather than catalogued: offering a model with no replica produces a // 503 the reader cannot interpret, and the allowlist is applied so the picker // never shows a model the request would refuse. -func (r *Router) modelsFor(req *http.Request, user store.User) []string { +// +// Real models are ordered ahead of simulated ones so the browser's default +// selection — the first option — is always a model that answers. Sorted purely +// by name, "big-model" came first and every visitor who pressed Send without +// touching the picker got placeholder text. +func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel { principal, err := r.principalFor(req, user) if err != nil { return nil } seen := map[string]struct{}{} - var out []string + var out []web.ChatModel for _, s := range r.reg.Snapshots("") { for _, m := range s.Models { if _, dup := seen[m]; dup { @@ -129,10 +135,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []string { } seen[m] = struct{}{} if r.permits(principal, m) { - out = append(out, m) + out = append(out, web.ChatModel{Name: m, Simulated: r.models[m].Simulated()}) } } } - slices.Sort(out) + slices.SortFunc(out, func(a, b web.ChatModel) int { + if a.Simulated != b.Simulated { + if b.Simulated { + return -1 + } + return 1 + } + return strings.Compare(a.Name, b.Name) + }) return out } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 31db5c9a..366f1ffa 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -499,8 +499,16 @@ func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { data := make([]map[string]any, 0, len(names)) for _, n := range names { + // A simulated model is owned by "llmgw-sim", which is what the sim + // engine already calls itself on its own /v1/models. Flattening it to + // "llmgw" here was hiding the one distinction a client needs: this + // model returns placeholder text, not inference. + owner := "llmgw" + if r.models[n].Simulated() { + owner = "llmgw-sim" + } data = append(data, map[string]any{ - "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw", + "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner, }) } writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data}) diff --git a/internal/router/router_test.go b/internal/router/router_test.go index d3947f8b..c839f687 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -257,6 +257,46 @@ func TestModelsEndpointListsAliases(t *testing.T) { } } +func TestModelsEndpointMarksSimulatedModels(t *testing.T) { + // A client picking a model from this list has nothing else to go on. Both + // entries are OpenAI-shaped and only owned_by distinguishes the one that + // returns placeholder text, so it has to be right for aliases too. + e := newFakeEngine(t, "real") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"real", "fake"}}) + sim := catModel("fake", []string{"fake-alias"}) + sim.Profiles = []catalog.Profile{{Name: "p", Engine: "sim", GPUClass: "h100-80g", TP: 1}} + cfg.Models = []catalog.Model{catModel("real", []string{"real-alias"}), sim} + rt := newTestRouter(t, cfg) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, req) + + var out struct { + Data []struct { + ID string `json:"id"` + OwnedBy string `json:"owned_by"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatal(err) + } + owner := map[string]string{} + for _, d := range out.Data { + owner[d.ID] = d.OwnedBy + } + for _, name := range []string{"fake", "fake-alias"} { + if owner[name] != "llmgw-sim" { + t.Errorf("%s: owned_by = %q, want llmgw-sim", name, owner[name]) + } + } + for _, name := range []string{"real", "real-alias"} { + if owner[name] != "llmgw" { + t.Errorf("%s: owned_by = %q, want llmgw", name, owner[name]) + } + } +} + func TestInjectIncludeUsagePreservesFields(t *testing.T) { // The one place the router rewrites a request body; nothing may be lost. in := []byte(`{"model":"m","stream":true,"temperature":0.7,"tools":[{"x":1}]}`) diff --git a/internal/web/chat.go b/internal/web/chat.go index eda240fe..f3f3051b 100644 --- a/internal/web/chat.go +++ b/internal/web/chat.go @@ -7,16 +7,25 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) +// ChatModel is one entry in the chat page's model picker. +type ChatModel struct { + Name string + // Simulated marks a model that returns synthetic text rather than + // inference output. The picker says so, because the two are + // indistinguishable from the outside until the reply arrives. + Simulated bool +} + type chatData struct { User store.User - Models []string + Models []ChatModel CSRF string Version string Now string } // Chat renders the in-browser chat for one signed-in person. -func Chat(user store.User, models []string, version string) http.HandlerFunc { +func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { noStore(w) render(w, "chat", chatData{ diff --git a/internal/web/chat_test.go b/internal/web/chat_test.go index 47b02c5d..76163699 100644 --- a/internal/web/chat_test.go +++ b/internal/web/chat_test.go @@ -9,7 +9,16 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) -func chatPage(t *testing.T, models []string) string { +func chatPage(t *testing.T, names ...string) string { + t.Helper() + models := make([]ChatModel, 0, len(names)) + for _, n := range names { + models = append(models, ChatModel{Name: n}) + } + return chatPageWith(t, models) +} + +func chatPageWith(t *testing.T, models []ChatModel) string { t.Helper() h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1") r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil) @@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) { // rather than remembering: the reply is put in place with textContent and // only then walked for maths. KaTeX replaces the spans it recognises, so // nothing the model wrote is ever parsed as HTML. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") if !strings.Contains(page, "/_static/katex/katex.min.css") || !strings.Contains(page, "/_static/katex/auto-render.min.js") { @@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) { func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) { // Typesettingevery chunk would try to render half an expression on every token // and flicker through the whole reply. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") body := page[strings.Index(page, "form.onsubmit"):] renderAt := strings.Index(body, "renderMath(out)") @@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) { func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) { // $$…$$ has to be tried before $…$, or the opening $$ is read as an empty // inline expression and everything after it is wrong. - page := chatPage(t, []string{"m"}) + page := chatPage(t, "m") dd := strings.Index(page, `left: "$$"`) d := strings.Index(page, `left: "$"`) if dd < 0 || d < 0 { @@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) { } func TestTheChatPageOffersWhatItWasGiven(t *testing.T) { - page := chatPage(t, []string{"alpha", "beta"}) + page := chatPage(t, "alpha", "beta") for _, m := range []string{"alpha", "beta"} { if !strings.Contains(page, `value="`+m+`"`) { t.Errorf("the picker does not offer %s", m) } } - empty := chatPage(t, nil) + empty := chatPageWith(t, nil) if strings.Contains(empty, "
big-model (simulated)<`) { + t.Error("the simulated model is offered without saying it is simulated") + } + if strings.Contains(page, `>qwen (simulated)<`) { + t.Error("a real model was labelled simulated") + } + if !strings.Contains(page, `value="big-model" data-simulated="1"`) || + !strings.Contains(page, `value="qwen" data-simulated="0"`) { + t.Error("the script cannot tell the two apart") + } + if !strings.Contains(page, "placeholder text, not inference") { + t.Error("the page never explains what a simulated model returns") + } +} + +func TestTheFirstOfferedModelAnswers(t *testing.T) { + // A browser selects the first option, so whatever leads the list is what + // someone gets when they type and press Send without looking at the picker. + page := chatPageWith(t, []ChatModel{ + {Name: "qwen"}, + {Name: "big-model", Simulated: true}, + }) + real, sim := strings.Index(page, `value="qwen"`), strings.Index(page, `value="big-model"`) + if real < 0 || sim < 0 { + t.Fatal("both models should be offered") + } + if real > sim { + t.Error("a simulated model is preselected; the default reply would be placeholder text") + } +} diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index 211097f1..317c8313 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -20,11 +20,16 @@ {{if .Models}}
+ +
@@ -62,6 +67,16 @@ access to it.

document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; }; + // The note follows the picker rather than the reply: someone should know what + // they are about to get before they spend a minute waiting for it. + const simNote = document.getElementById("sim-note"); + function showSimNote() { + const opt = model.selectedOptions[0]; + simNote.hidden = !opt || opt.dataset.simulated !== "1"; + } + model.addEventListener("change", showSimNote); + showSimNote(); + function bubble(role) { const el = document.createElement("div"); el.className = "bubble " + role;