Say which model is simulated instead of letting it look broken
big-model runs on the sim engine, whose whole vocabulary is the Greek alphabet, cycled at a fixed rate until max_tokens. That is what it is for: with no H100s it is the only thing exercising the TP8 two-node path, the fabric invariant and rebuild-after-node-loss. The output is the point. What was wrong is that nothing said so. /v1/models returned it with owned_by "llmgw", identical to the real model, and the chat picker sorted purely by name — so "big-model" led the list, was preselected by the browser, and anyone who typed a question and pressed Send got Greek letters back with no explanation. Reported as the model spitting out crap, which is a fair reading of what it looks like from outside. Simulated is derived from the profiles rather than configured: an operator who adds a sim profile and forgets a flag would be shipping placeholder text under a real model's name. A model counts as simulated only when every profile is, because one real profile means the request can still come back as inference. The marker on /v1/models is owned_by "llmgw-sim", which is already what the sim engine calls itself on its own /v1/models — the gateway was flattening a distinction that existed one layer down. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbf276a41b
commit
72b03a1a1c
8 files changed
+195
-14
No files matched your search
@@ -313,3 +313,24 @@ func (c *Catalog) String() string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Simulated reports whether this model produces synthetic text rather than
|
||||
// inference output.
|
||||
//
|
||||
// Derived from the profiles rather than configured, because an operator who
|
||||
// added a sim profile and forgot the flag would be shipping placeholder text to
|
||||
// users under a real model's name. A model counts as simulated only when every
|
||||
// profile is simulated: one real profile means a request can land on a real
|
||||
// engine, and a model that sometimes answers for real must not be labelled as
|
||||
// though it never does.
|
||||
func (m Model) Simulated() bool {
|
||||
if len(m.Profiles) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine != "sim" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -233,3 +233,29 @@ func TestNamesAndModelsOrder(t *testing.T) {
|
||||
t.Errorf("Names order=%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) {
|
||||
// The question the flag answers is "can this request come back as real
|
||||
// inference?". One real profile means yes, however many sim profiles sit
|
||||
// beside it, so a mixed model must not be labelled simulated.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
engines []string
|
||||
wantSim bool
|
||||
}{
|
||||
{"all simulated", []string{"sim", "sim"}, true},
|
||||
{"one real profile", []string{"sim", "llamacpp"}, false},
|
||||
{"no simulation", []string{"llamacpp"}, false},
|
||||
{"no profiles at all", nil, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var m Model
|
||||
for _, e := range tc.engines {
|
||||
m.Profiles = append(m.Profiles, Profile{Engine: e})
|
||||
}
|
||||
if got := m.Simulated(); got != tc.wantSim {
|
||||
t.Errorf("Simulated() = %v, want %v", got, tc.wantSim)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+18
-4
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -115,13 +116,18 @@ func (r *Router) principalFor(req *http.Request, user store.User) (auth.Principa
|
||||
// Served rather than catalogued: offering a model with no replica produces a
|
||||
// 503 the reader cannot interpret, and the allowlist is applied so the picker
|
||||
// never shows a model the request would refuse.
|
||||
func (r *Router) modelsFor(req *http.Request, user store.User) []string {
|
||||
//
|
||||
// Real models are ordered ahead of simulated ones so the browser's default
|
||||
// selection — the first option — is always a model that answers. Sorted purely
|
||||
// by name, "big-model" came first and every visitor who pressed Send without
|
||||
// touching the picker got placeholder text.
|
||||
func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
principal, err := r.principalFor(req, user)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]struct{}{}
|
||||
var out []string
|
||||
var out []web.ChatModel
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
for _, m := range s.Models {
|
||||
if _, dup := seen[m]; dup {
|
||||
@@ -129,10 +135,18 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []string {
|
||||
}
|
||||
seen[m] = struct{}{}
|
||||
if r.permits(principal, m) {
|
||||
out = append(out, m)
|
||||
out = append(out, web.ChatModel{Name: m, Simulated: r.models[m].Simulated()})
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
slices.SortFunc(out, func(a, b web.ChatModel) int {
|
||||
if a.Simulated != b.Simulated {
|
||||
if b.Simulated {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(a.Name, b.Name)
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -499,8 +499,16 @@ func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
data := make([]map[string]any, 0, len(names))
|
||||
for _, n := range names {
|
||||
// A simulated model is owned by "llmgw-sim", which is what the sim
|
||||
// engine already calls itself on its own /v1/models. Flattening it to
|
||||
// "llmgw" here was hiding the one distinction a client needs: this
|
||||
// model returns placeholder text, not inference.
|
||||
owner := "llmgw"
|
||||
if r.models[n].Simulated() {
|
||||
owner = "llmgw-sim"
|
||||
}
|
||||
data = append(data, map[string]any{
|
||||
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": "llmgw",
|
||||
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": data})
|
||||
|
||||
@@ -257,6 +257,46 @@ func TestModelsEndpointListsAliases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsEndpointMarksSimulatedModels(t *testing.T) {
|
||||
// A client picking a model from this list has nothing else to go on. Both
|
||||
// entries are OpenAI-shaped and only owned_by distinguishes the one that
|
||||
// returns placeholder text, so it has to be right for aliases too.
|
||||
e := newFakeEngine(t, "real")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"real", "fake"}})
|
||||
sim := catModel("fake", []string{"fake-alias"})
|
||||
sim.Profiles = []catalog.Profile{{Name: "p", Engine: "sim", GPUClass: "h100-80g", TP: 1}}
|
||||
cfg.Models = []catalog.Model{catModel("real", []string{"real-alias"}), sim}
|
||||
rt := newTestRouter(t, cfg)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
rt.Handler().ServeHTTP(rec, req)
|
||||
|
||||
var out struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
OwnedBy string `json:"owned_by"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
owner := map[string]string{}
|
||||
for _, d := range out.Data {
|
||||
owner[d.ID] = d.OwnedBy
|
||||
}
|
||||
for _, name := range []string{"fake", "fake-alias"} {
|
||||
if owner[name] != "llmgw-sim" {
|
||||
t.Errorf("%s: owned_by = %q, want llmgw-sim", name, owner[name])
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"real", "real-alias"} {
|
||||
if owner[name] != "llmgw" {
|
||||
t.Errorf("%s: owned_by = %q, want llmgw", name, owner[name])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectIncludeUsagePreservesFields(t *testing.T) {
|
||||
// The one place the router rewrites a request body; nothing may be lost.
|
||||
in := []byte(`{"model":"m","stream":true,"temperature":0.7,"tools":[{"x":1}]}`)
|
||||
|
||||
+11
-2
@@ -7,16 +7,25 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ChatModel is one entry in the chat page's model picker.
|
||||
type ChatModel struct {
|
||||
Name string
|
||||
// Simulated marks a model that returns synthetic text rather than
|
||||
// inference output. The picker says so, because the two are
|
||||
// indistinguishable from the outside until the reply arrives.
|
||||
Simulated bool
|
||||
}
|
||||
|
||||
type chatData struct {
|
||||
User store.User
|
||||
Models []string
|
||||
Models []ChatModel
|
||||
CSRF string
|
||||
Version string
|
||||
Now string
|
||||
}
|
||||
|
||||
// Chat renders the in-browser chat for one signed-in person.
|
||||
func Chat(user store.User, models []string, version string) http.HandlerFunc {
|
||||
func Chat(user store.User, models []ChatModel, version string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
noStore(w)
|
||||
render(w, "chat", chatData{
|
||||
|
||||
@@ -9,7 +9,16 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func chatPage(t *testing.T, models []string) string {
|
||||
func chatPage(t *testing.T, names ...string) string {
|
||||
t.Helper()
|
||||
models := make([]ChatModel, 0, len(names))
|
||||
for _, n := range names {
|
||||
models = append(models, ChatModel{Name: n})
|
||||
}
|
||||
return chatPageWith(t, models)
|
||||
}
|
||||
|
||||
func chatPageWith(t *testing.T, models []ChatModel) string {
|
||||
t.Helper()
|
||||
h := Chat(store.User{Email: "a@example.com", ProjectID: "proj"}, models, "v1")
|
||||
r := httptest.NewRequest(http.MethodGet, "http://x/chat", nil)
|
||||
@@ -26,7 +35,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
|
||||
// rather than remembering: the reply is put in place with textContent and
|
||||
// only then walked for maths. KaTeX replaces the spans it recognises, so
|
||||
// nothing the model wrote is ever parsed as HTML.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
|
||||
if !strings.Contains(page, "/_static/katex/katex.min.css") ||
|
||||
!strings.Contains(page, "/_static/katex/auto-render.min.js") {
|
||||
@@ -59,7 +68,7 @@ func TestMathIsRenderedWithoutTrustingTheModel(t *testing.T) {
|
||||
func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
|
||||
// Typesettingevery chunk would try to render half an expression on every token
|
||||
// and flicker through the whole reply.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
body := page[strings.Index(page, "form.onsubmit"):]
|
||||
|
||||
renderAt := strings.Index(body, "renderMath(out)")
|
||||
@@ -75,7 +84,7 @@ func TestMathIsRenderedOnceTheReplyIsComplete(t *testing.T) {
|
||||
func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
|
||||
// $$…$$ has to be tried before $…$, or the opening $$ is read as an empty
|
||||
// inline expression and everything after it is wrong.
|
||||
page := chatPage(t, []string{"m"})
|
||||
page := chatPage(t, "m")
|
||||
dd := strings.Index(page, `left: "$$"`)
|
||||
d := strings.Index(page, `left: "$"`)
|
||||
if dd < 0 || d < 0 {
|
||||
@@ -87,13 +96,13 @@ func TestDisplayDelimitersComeBeforeInlineOnes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
|
||||
page := chatPage(t, []string{"alpha", "beta"})
|
||||
page := chatPage(t, "alpha", "beta")
|
||||
for _, m := range []string{"alpha", "beta"} {
|
||||
if !strings.Contains(page, `value="`+m+`"`) {
|
||||
t.Errorf("the picker does not offer %s", m)
|
||||
}
|
||||
}
|
||||
empty := chatPage(t, nil)
|
||||
empty := chatPageWith(t, nil)
|
||||
if strings.Contains(empty, "<form id=\"composer\"") {
|
||||
t.Error("a composer was rendered with no model to send to")
|
||||
}
|
||||
@@ -101,3 +110,42 @@ func TestTheChatPageOffersWhatItWasGiven(t *testing.T) {
|
||||
t.Error("the page does not explain why it is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSimulatedModelSaysSo(t *testing.T) {
|
||||
// big-model returns Greek letters at a fixed rate. Offered next to a real
|
||||
// model with nothing to tell them apart, it reads as a broken model rather
|
||||
// than a load generator, which is how it was first reported.
|
||||
page := chatPageWith(t, []ChatModel{
|
||||
{Name: "qwen"},
|
||||
{Name: "big-model", Simulated: true},
|
||||
})
|
||||
if !strings.Contains(page, `>big-model (simulated)<`) {
|
||||
t.Error("the simulated model is offered without saying it is simulated")
|
||||
}
|
||||
if strings.Contains(page, `>qwen (simulated)<`) {
|
||||
t.Error("a real model was labelled simulated")
|
||||
}
|
||||
if !strings.Contains(page, `value="big-model" data-simulated="1"`) ||
|
||||
!strings.Contains(page, `value="qwen" data-simulated="0"`) {
|
||||
t.Error("the script cannot tell the two apart")
|
||||
}
|
||||
if !strings.Contains(page, "placeholder text, not inference") {
|
||||
t.Error("the page never explains what a simulated model returns")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFirstOfferedModelAnswers(t *testing.T) {
|
||||
// A browser selects the first option, so whatever leads the list is what
|
||||
// someone gets when they type and press Send without looking at the picker.
|
||||
page := chatPageWith(t, []ChatModel{
|
||||
{Name: "qwen"},
|
||||
{Name: "big-model", Simulated: true},
|
||||
})
|
||||
real, sim := strings.Index(page, `value="qwen"`), strings.Index(page, `value="big-model"`)
|
||||
if real < 0 || sim < 0 {
|
||||
t.Fatal("both models should be offered")
|
||||
}
|
||||
if real > sim {
|
||||
t.Error("a simulated model is preselected; the default reply would be placeholder text")
|
||||
}
|
||||
}
|
||||
@@ -20,11 +20,16 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.}}">{{.}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
|
||||
<p id="sim-note" class="note" hidden>
|
||||
This model is simulated: it exercises scheduling and reports timings, and its
|
||||
replies are placeholder text, not inference.
|
||||
</p>
|
||||
|
||||
<div id="log" class="chat-log" aria-live="polite"></div>
|
||||
|
||||
<form id="composer" class="chat-composer">
|
||||
@@ -62,6 +67,16 @@ access to it.</p>
|
||||
|
||||
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
|
||||
|
||||
// The note follows the picker rather than the reply: someone should know what
|
||||
// they are about to get before they spend a minute waiting for it.
|
||||
const simNote = document.getElementById("sim-note");
|
||||
function showSimNote() {
|
||||
const opt = model.selectedOptions[0];
|
||||
simNote.hidden = !opt || opt.dataset.simulated !== "1";
|
||||
}
|
||||
model.addEventListener("change", showSimNote);
|
||||
showSimNote();
|
||||
|
||||
function bubble(role) {
|
||||
const el = document.createElement("div");
|
||||
el.className = "bubble " + role;
|
||||
|
||||
Reference in new issue
Block a user