Say which model is simulated instead of letting it look broken
CI / test (push) Successful in 1m31s

big-model runs on the sim engine, whose whole vocabulary is the Greek
alphabet, cycled at a fixed rate until max_tokens. That is what it is for:
with no H100s it is the only thing exercising the TP8 two-node path, the
fabric invariant and rebuild-after-node-loss. The output is the point.

What was wrong is that nothing said so. /v1/models returned it with
owned_by "llmgw", identical to the real model, and the chat picker sorted
purely by name — so "big-model" led the list, was preselected by the
browser, and anyone who typed a question and pressed Send got Greek
letters back with no explanation. Reported as the model spitting out crap,
which is a fair reading of what it looks like from outside.

Simulated is derived from the profiles rather than configured: an operator
who adds a sim profile and forgets a flag would be shipping placeholder
text under a real model's name. A model counts as simulated only when
every profile is, because one real profile means the request can still
come back as inference.

The marker on /v1/models is owned_by "llmgw-sim", which is already what
the sim engine calls itself on its own /v1/models — the gateway was
flattening a distinction that existed one layer down.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 17:11:56 +02:00
1 parent cbf276a41b
commit 72b03a1a1c
8 files changed
+195 -14

No files matched your search

+21
View File
@@ -313,3 +313,24 @@ func (c *Catalog) String() string {
}
return b.String()
}
// Simulated reports whether this model produces synthetic text rather than
// inference output.
//
// Derived from the profiles rather than configured, because an operator who
// added a sim profile and forgot the flag would be shipping placeholder text to
// users under a real model's name. A model counts as simulated only when every
// profile is simulated: one real profile means a request can land on a real
// engine, and a model that sometimes answers for real must not be labelled as
// though it never does.
func (m Model) Simulated() bool {
if len(m.Profiles) == 0 {
return false
}
for _, p := range m.Profiles {
if p.Engine != "sim" {
return false
}
}
return true
}
+26
View File
@@ -233,3 +233,29 @@ func TestNamesAndModelsOrder(t *testing.T) {
t.Errorf("Names order=%v", got)
}
}
func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) {
// The question the flag answers is "can this request come back as real
// inference?". One real profile means yes, however many sim profiles sit
// beside it, so a mixed model must not be labelled simulated.
for _, tc := range []struct {
name string
engines []string
wantSim bool
}{
{"all simulated", []string{"sim", "sim"}, true},
{"one real profile", []string{"sim", "llamacpp"}, false},
{"no simulation", []string{"llamacpp"}, false},
{"no profiles at all", nil, false},
} {
t.Run(tc.name, func(t *testing.T) {
var m Model
for _, e := range tc.engines {
m.Profiles = append(m.Profiles, Profile{Engine: e})
}
if got := m.Simulated(); got != tc.wantSim {
t.Errorf("Simulated() = %v, want %v", got, tc.wantSim)
}
})
}
}
+18 -4
View File
@@ -7,6 +7,7 @@ import (
"io"
"net/http"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -115,13 +116,18 @@ func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principa
// Served rather than catalogued: offering a model with no replica produces a
// 503 the reader cannot interpret, and the allowlist is applied so the picker
// never shows a model the request would refuse.
func (r *Router) modelsFor(req *http.Request, user store.User) []string {
//
// Real models are ordered ahead of simulated ones so the browser's default
// selection — the first option — is always a model that answers. Sorted purely
// by name, "big-model" came first and every visitor who pressed Send without
// touching the picker got placeholder text.
func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
principal, err := r.principalFor(req, user)
if err != nil {
return nil
}
seen := map[string]struct{}{}
var out []string
var out []web.ChatModel
for _, s := range r.reg.Snapshots("") {
for _, m := range s.Models {
if _, dup := seen[m]; dup {
@@ -129,10 +135,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []string {
}
seen[m] = struct{}{}
if r.permits(principal, m) {
out = append(out, m)
out = append(out, web.ChatModel{Name: m, Simulated: r.models[m].Simulated()})
}
}
}
slices.Sort(out)
slices.SortFunc(out, func(a, b web.ChatModel) int {
if a.Simulated != b.Simulated {
if b.Simulated {
return -1
}
return 1
}
return strings.Compare(a.Name, b.Name)
})
return out
}
+9 -1
View File
@@ -499,8 +499,16 @@ func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
data := make([]map[string]any, 0, len(names))
for _, n := range names {
// A simulated model is owned by "llmgw-sim", which is what the sim
// engine already calls itself on its own /v1/models. Flattening it to
// "llmgw" here was hiding the one distinction a client needs: this
// model returns placeholder text, not inference.
owner := "llmgw"
if r.models[n].Simulated() {
owner = "llmgw-sim"
}
data = append(data, map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
})
}
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
+40
View File
@@ -257,6 +257,46 @@ func TestModelsEndpointListsAliases(t *testing.T) {
}
}
func TestModelsEndpointMarksSimulatedModels(t *testing.T) {
// A client picking a model from this list has nothing else to go on. Both
// entries are OpenAI-shaped and only owned_by distinguishes the one that
// returns placeholder text, so it has to be right for aliases too.
e := newFakeEngine(t, "real")
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"real", "fake"}})
sim := catModel("fake", []string{"fake-alias"})
sim.Profiles = []catalog.Profile{{Name: "p", Engine: "sim", GPUClass: "h100-80g", TP: 1}}
cfg.Models = []catalog.Model{catModel("real", []string{"real-alias"}), sim}
rt := newTestRouter(t, cfg)
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, req)
var out struct {
Data []struct {
ID string `json:"id"`
OwnedBy string `json:"owned_by"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
owner := map[string]string{}
for _, d := range out.Data {
owner[d.ID] = d.OwnedBy
}
for _, name := range []string{"fake", "fake-alias"} {
if owner[name] != "llmgw-sim" {
t.Errorf("%s: owned_by = %q, want llmgw-sim", name, owner[name])
}
}
for _, name := range []string{"real", "real-alias"} {
if owner[name] != "llmgw" {
t.Errorf("%s: owned_by = %q, want llmgw", name, owner[name])
}
}
}
func TestInjectIncludeUsagePreservesFields(t *testing.T) {
// The one place the router rewrites a request body; nothing may be lost.
in := []byte(`{"model":"m","stream":true,"temperature":0.7,"tools":[{"x":1}]}`)
+11 -2
View File
@@ -7,16 +7,25 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// ChatModel is one entry in the chat page's model picker.
type ChatModel struct {
Name string
// Simulated marks a model that returns synthetic text rather than
// inference output. The picker says so, because the two are
// indistinguishable from the outside until the reply arrives.
Simulated bool
}
type chatData struct {
User store.User
Models []string
Models []ChatModel
CSRF string
Version string
Now string
}
// Chat renders the in-browser chat for one signed-in person.
func Chat(user store.User, models []string, version string) http.HandlerFunc {
func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
noStore(w)
render(w, "chat", chatData{
+54 -6
View File
@@ -9,7 +9,16 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func chatPage(t *testing.T, models []string) string {
func chatPage(t *testing.T, names ...string) string {
t.Helper()
models := make([]ChatModel, 0, len(names))
for _, n := range names {
models = append(models, ChatModel{Name: n})
}
return chatPageWith(t, models)
}
func chatPageWith(t *testing.T, models []ChatModel) string {
t.Helper()
h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1")
r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil)
@@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
// rather than remembering: the reply is put in place with textContent and
// only then walked for maths. KaTeX replaces the spans it recognises, so
// nothing the model wrote is ever parsed as HTML.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
if !strings.Contains(page, "/_static/katex/katex.min.css") ||
!strings.Contains(page, "/_static/katex/auto-render.min.js") {
@@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
// Typesettingevery chunk would try to render half an expression on every token
// and flicker through the whole reply.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
body := page[strings.Index(page, "form.onsubmit"):]
renderAt := strings.Index(body, "renderMath(out)")
@@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
// $$…$$ has to be tried before $…$, or the opening $$ is read as an empty
// inline expression and everything after it is wrong.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
dd := strings.Index(page, `left: "$$"`)
d := strings.Index(page, `left: "$"`)
if dd < 0 || d < 0 {
@@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
}
func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
page := chatPage(t, []string{"alpha", "beta"})
page := chatPage(t, "alpha", "beta")
for _, m := range []string{"alpha", "beta"} {
if !strings.Contains(page, `value="`+m+`"`) {
t.Errorf("the picker does not offer %s", m)
}
}
empty := chatPage(t, nil)
empty := chatPageWith(t, nil)
if strings.Contains(empty, "<form id=\"composer\"") {
t.Error("a composer was rendered with no model to send to")
}
@@ -101,3 +110,42 @@ func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
t.Error("the page does not explain why it is empty")
}
}
func TestTheSimulatedModelSaysSo(t *testing.T) {
// big-model returns Greek letters at a fixed rate. Offered next to a real
// model with nothing to tell them apart, it reads as a broken model rather
// than a load generator, which is how it was first reported.
page := chatPageWith(t, []ChatModel{
{Name: "qwen"},
{Name: "big-model", Simulated: true},
})
if !strings.Contains(page, `>big-model (simulated)<`) {
t.Error("the simulated model is offered without saying it is simulated")
}
if strings.Contains(page, `>qwen (simulated)<`) {
t.Error("a real model was labelled simulated")
}
if !strings.Contains(page, `value="big-model" data-simulated="1"`) ||
!strings.Contains(page, `value="qwen" data-simulated="0"`) {
t.Error("the script cannot tell the two apart")
}
if !strings.Contains(page, "placeholder text, not inference") {
t.Error("the page never explains what a simulated model returns")
}
}
func TestTheFirstOfferedModelAnswers(t *testing.T) {
// A browser selects the first option, so whatever leads the list is what
// someone gets when they type and press Send without looking at the picker.
page := chatPageWith(t, []ChatModel{
{Name: "qwen"},
{Name: "big-model", Simulated: true},
})
real, sim := strings.Index(page, `value="qwen"`), strings.Index(page, `value="big-model"`)
if real < 0 || sim < 0 {
t.Fatal("both models should be offered")
}
if real > sim {
t.Error("a simulated model is preselected; the default reply would be placeholder text")
}
}
+16 -1
View File
@@ -20,11 +20,16 @@
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.}}">{{.}}</option>{{end}}</select>
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
<form id="composer" class="chat-composer">
@@ -62,6 +67,16 @@ access to it.</p>
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
const simNote = document.getElementById("sim-note");
function showSimNote() {
const opt = model.selectedOptions[0];
simNote.hidden = !opt || opt.dataset.simulated !== "1";
}
model.addEventListener("change", showSimNote);
showSimNote();
function bubble(role) {
const el = document.createElement("div");
el.className = "bubble " + role;