Say which model is simulated instead of letting it look broken
CI / test (push) Successful in 1m31s

big-model runs on the sim engine, whose whole vocabulary is the Greek
alphabet, cycled at a fixed rate until max_tokens. That is what it is for:
with no H100s it is the only thing exercising the TP8 two-node path, the
fabric invariant and rebuild-after-node-loss. The output is the point.

What was wrong is that nothing said so. /v1/models returned it with
owned_by "llmgw", identical to the real model, and the chat picker sorted
purely by name — so "big-model" led the list, was preselected by the
browser, and anyone who typed a question and pressed Send got Greek
letters back with no explanation. Reported as the model spitting out crap,
which is a fair reading of what it looks like from outside.

Simulated is derived from the profiles rather than configured: an operator
who adds a sim profile and forgets a flag would be shipping placeholder
text under a real model's name. A model counts as simulated only when
every profile is, because one real profile means the request can still
come back as inference.

The marker on /v1/models is owned_by "llmgw-sim", which is already what
the sim engine calls itself on its own /v1/models — the gateway was
flattening a distinction that existed one layer down.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 17:11:56 +02:00
1 parent cbf276a41b
commit 72b03a1a1c
8 files changed
+195 -14

No files matched your search

+11 -2
View File
@@ -7,16 +7,25 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// ChatModel is one entry in the chat page's model picker.
type ChatModel struct {
Name string
// Simulated marks a model that returns synthetic text rather than
// inference output. The picker says so, because the two are
// indistinguishable from the outside until the reply arrives.
Simulated bool
}
type chatData struct {
User store.User
Models []string
Models []ChatModel
CSRF string
Version string
Now string
}
// Chat renders the in-browser chat for one signed-in person.
func Chat(user store.User, models []string, version string) http.HandlerFunc {
func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
noStore(w)
render(w, "chat", chatData{
+54 -6
View File
@@ -9,7 +9,16 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func chatPage(t *testing.T, models []string) string {
func chatPage(t *testing.T, names ...string) string {
t.Helper()
models := make([]ChatModel, 0, len(names))
for _, n := range names {
models = append(models, ChatModel{Name: n})
}
return chatPageWith(t, models)
}
func chatPageWith(t *testing.T, models []ChatModel) string {
t.Helper()
h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1")
r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil)
@@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
// rather than remembering: the reply is put in place with textContent and
// only then walked for maths. KaTeX replaces the spans it recognises, so
// nothing the model wrote is ever parsed as HTML.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
if !strings.Contains(page, "/_static/katex/katex.min.css") ||
!strings.Contains(page, "/_static/katex/auto-render.min.js") {
@@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
// Typesettingevery chunk would try to render half an expression on every token
// and flicker through the whole reply.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
body := page[strings.Index(page, "form.onsubmit"):]
renderAt := strings.Index(body, "renderMath(out)")
@@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
// $$…$$ has to be tried before $…$, or the opening $$ is read as an empty
// inline expression and everything after it is wrong.
page := chatPage(t, []string{"m"})
page := chatPage(t, "m")
dd := strings.Index(page, `left: "$$"`)
d := strings.Index(page, `left: "$"`)
if dd < 0 || d < 0 {
@@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
}
func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
page := chatPage(t, []string{"alpha", "beta"})
page := chatPage(t, "alpha", "beta")
for _, m := range []string{"alpha", "beta"} {
if !strings.Contains(page, `value="`+m+`"`) {
t.Errorf("the picker does not offer %s", m)
}
}
empty := chatPage(t, nil)
empty := chatPageWith(t, nil)
if strings.Contains(empty, "<form id=\"composer\"") {
t.Error("a composer was rendered with no model to send to")
}
@@ -101,3 +110,42 @@ func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
t.Error("the page does not explain why it is empty")
}
}
func TestTheSimulatedModelSaysSo(t *testing.T) {
// big-model returns Greek letters at a fixed rate. Offered next to a real
// model with nothing to tell them apart, it reads as a broken model rather
// than a load generator, which is how it was first reported.
page := chatPageWith(t, []ChatModel{
{Name: "qwen"},
{Name: "big-model", Simulated: true},
})
if !strings.Contains(page, `>big-model (simulated)<`) {
t.Error("the simulated model is offered without saying it is simulated")
}
if strings.Contains(page, `>qwen (simulated)<`) {
t.Error("a real model was labelled simulated")
}
if !strings.Contains(page, `value="big-model" data-simulated="1"`) ||
!strings.Contains(page, `value="qwen" data-simulated="0"`) {
t.Error("the script cannot tell the two apart")
}
if !strings.Contains(page, "placeholder text, not inference") {
t.Error("the page never explains what a simulated model returns")
}
}
func TestTheFirstOfferedModelAnswers(t *testing.T) {
// A browser selects the first option, so whatever leads the list is what
// someone gets when they type and press Send without looking at the picker.
page := chatPageWith(t, []ChatModel{
{Name: "qwen"},
{Name: "big-model", Simulated: true},
})
real, sim := strings.Index(page, `value="qwen"`), strings.Index(page, `value="big-model"`)
if real < 0 || sim < 0 {
t.Fatal("both models should be offered")
}
if real > sim {
t.Error("a simulated model is preselected; the default reply would be placeholder text")
}
}
+16 -1
View File
@@ -20,11 +20,16 @@
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.}}">{{.}}</option>{{end}}</select>
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
<form id="composer" class="chat-composer">
@@ -62,6 +67,16 @@ access to it.</p>
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
const simNote = document.getElementById("sim-note");
function showSimNote() {
const opt = model.selectedOptions[0];
simNote.hidden = !opt || opt.dataset.simulated !== "1";
}
model.addEventListener("change", showSimNote);
showSimNote();
function bubble(role) {
const el = document.createElement("div");
el.className = "bubble " + role;