big-model runs on the sim engine, whose whole vocabulary is the Greek alphabet, cycled at a fixed rate until max_tokens. That is what it is for: with no H100s it is the only thing exercising the TP8 two-node path, the fabric invariant and rebuild-after-node-loss. The output is the point. What was wrong is that nothing said so. /v1/models returned it with owned_by "llmgw", identical to the real model, and the chat picker sorted purely by name — so "big-model" led the list, was preselected by the browser, and anyone who typed a question and pressed Send got Greek letters back with no explanation. Reported as the model spitting out crap, which is a fair reading of what it looks like from outside. Simulated is derived from the profiles rather than configured: an operator who adds a sim profile and forgets a flag would be shipping placeholder text under a real model's name. A model counts as simulated only when every profile is, because one real profile means the request can still come back as inference. The marker on /v1/models is owned_by "llmgw-sim", which is already what the sim engine calls itself on its own /v1/models — the gateway was flattening a distinction that existed one layer down. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbf276a41b
commit
72b03a1a1c
8 files changed
+195
-14
No files matched your search
+11
-2
@@ -7,16 +7,25 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ChatModel is one entry in the chat page's model picker.
|
||||
type ChatModel struct {
|
||||
Name string
|
||||
// Simulated marks a model that returns synthetic text rather than
|
||||
// inference output. The picker says so, because the two are
|
||||
// indistinguishable from the outside until the reply arrives.
|
||||
Simulated bool
|
||||
}
|
||||
|
||||
type chatData struct {
|
||||
User store.User
|
||||
Models []string
|
||||
Models []ChatModel
|
||||
CSRF string
|
||||
Version string
|
||||
Now string
|
||||
}
|
||||
|
||||
// Chat renders the in-browser chat for one signed-in person.
|
||||
func Chat(user store.User, models []string, version string) http.HandlerFunc {
|
||||
func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
noStore(w)
|
||||
render(w, "chat", chatData{
|
||||
|
||||
@@ -9,7 +9,16 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func chatPage(t *testing.T, models []string) string {
|
||||
func chatPage(t *testing.T, names ...string) string {
|
||||
t.Helper()
|
||||
models := make([]ChatModel, 0, len(names))
|
||||
for _, n := range names {
|
||||
models = append(models, ChatModel{Name: n})
|
||||
}
|
||||
return chatPageWith(t, models)
|
||||
}
|
||||
|
||||
func chatPageWith(t *testing.T, models []ChatModel) string {
|
||||
t.Helper()
|
||||
h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1")
|
||||
r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil)
|
||||
@@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
|
||||
// rather than remembering: the reply is put in place with textContent and
|
||||
// only then walked for maths. KaTeX replaces the spans it recognises, so
|
||||
// nothing the model wrote is ever parsed as HTML.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
|
||||
if !strings.Contains(page, "/_static/katex/katex.min.css") ||
|
||||
!strings.Contains(page, "/_static/katex/auto-render.min.js") {
|
||||
@@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
|
||||
func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
|
||||
// Typesettingevery chunk would try to render half an expression on every token
|
||||
// and flicker through the whole reply.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
body := page[strings.Index(page, "form.onsubmit"):]
|
||||
|
||||
renderAt := strings.Index(body, "renderMath(out)")
|
||||
@@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
|
||||
func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
|
||||
// $$…$$ has to be tried before $…$, or the opening $$ is read as an empty
|
||||
// inline expression and everything after it is wrong.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
dd := strings.Index(page, `left: "$$"`)
|
||||
d := strings.Index(page, `left: "$"`)
|
||||
if dd < 0 || d < 0 {
|
||||
@@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
|
||||
page := chatPage(t, []string{"alpha", "beta"})
|
||||
page := chatPage(t, "alpha", "beta")
|
||||
for _, m := range []string{"alpha", "beta"} {
|
||||
if !strings.Contains(page, `value="`+m+`"`) {
|
||||
t.Errorf("the picker does not offer %s", m)
|
||||
}
|
||||
}
|
||||
empty := chatPage(t, nil)
|
||||
empty := chatPageWith(t, nil)
|
||||
if strings.Contains(empty, "<form id=\"composer\"") {
|
||||
t.Error("a composer was rendered with no model to send to")
|
||||
}
|
||||
@@ -101,3 +110,42 @@ func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
|
||||
t.Error("the page does not explain why it is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSimulatedModelSaysSo(t *testing.T) {
|
||||
// big-model returns Greek letters at a fixed rate. Offered next to a real
|
||||
// model with nothing to tell them apart, it reads as a broken model rather
|
||||
// than a load generator, which is how it was first reported.
|
||||
page := chatPageWith(t, []ChatModel{
|
||||
{Name: "qwen"},
|
||||
{Name: "big-model", Simulated: true},
|
||||
})
|
||||
if !strings.Contains(page, `>big-model (simulated)<`) {
|
||||
t.Error("the simulated model is offered without saying it is simulated")
|
||||
}
|
||||
if strings.Contains(page, `>qwen (simulated)<`) {
|
||||
t.Error("a real model was labelled simulated")
|
||||
}
|
||||
if !strings.Contains(page, `value="big-model" data-simulated="1"`) ||
|
||||
!strings.Contains(page, `value="qwen" data-simulated="0"`) {
|
||||
t.Error("the script cannot tell the two apart")
|
||||
}
|
||||
if !strings.Contains(page, "placeholder text, not inference") {
|
||||
t.Error("the page never explains what a simulated model returns")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFirstOfferedModelAnswers(t *testing.T) {
|
||||
// A browser selects the first option, so whatever leads the list is what
|
||||
// someone gets when they type and press Send without looking at the picker.
|
||||
page := chatPageWith(t, []ChatModel{
|
||||
{Name: "qwen"},
|
||||
{Name: "big-model", Simulated: true},
|
||||
})
|
||||
real, sim := strings.Index(page, `value="qwen"`), strings.Index(page, `value="big-model"`)
|
||||
if real < 0 || sim < 0 {
|
||||
t.Fatal("both models should be offered")
|
||||
}
|
||||
if real > sim {
|
||||
t.Error("a simulated model is preselected; the default reply would be placeholder text")
|
||||
}
|
||||
}
|
||||
@@ -20,11 +20,16 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.}}">{{.}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
|
||||
<p id="sim-note" class="note" hidden>
|
||||
This model is simulated: it exercises scheduling and reports timings, and its
|
||||
replies are placeholder text, not inference.
|
||||
</p>
|
||||
|
||||
<div id="log" class="chat-log" aria-live="polite"></div>
|
||||
|
||||
<form id="composer" class="chat-composer">
|
||||
@@ -62,6 +67,16 @@ access to it.</p>
|
||||
|
||||
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
|
||||
|
||||
// The note follows the picker rather than the reply: someone should know what
|
||||
// they are about to get before they spend a minute waiting for it.
|
||||
const simNote = document.getElementById("sim-note");
|
||||
function showSimNote() {
|
||||
const opt = model.selectedOptions[0];
|
||||
simNote.hidden = !opt || opt.dataset.simulated !== "1";
|
||||
}
|
||||
model.addEventListener("change", showSimNote);
|
||||
showSimNote();
|
||||
|
||||
function bubble(role) {
|
||||
const el = document.createElement("div");
|
||||
el.className = "bubble " + role;
|
||||
|
||||
Reference in new issue
Block a user