big-model runs on the sim engine, whose whole vocabulary is the Greek alphabet, cycled at a fixed rate until max_tokens. That is what it is for: with no H100s it is the only thing exercising the TP8 two-node path, the fabric invariant and rebuild-after-node-loss. The output is the point. What was wrong is that nothing said so. /v1/models returned it with owned_by "llmgw", identical to the real model, and the chat picker sorted purely by name — so "big-model" led the list, was preselected by the browser, and anyone who typed a question and pressed Send got Greek letters back with no explanation. Reported as the model spitting out crap, which is a fair reading of what it looks like from outside. Simulated is derived from the profiles rather than configured: an operator who adds a sim profile and forgets a flag would be shipping placeholder text under a real model's name. A model counts as simulated only when every profile is, because one real profile means the request can still come back as inference. The marker on /v1/models is owned_by "llmgw-sim", which is already what the sim engine calls itself on its own /v1/models — the gateway was flattening a distinction that existed one layer down. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cbf276a41b
commit
72b03a1a1c
8 files changed
+195
-14
No files matched your search
@@ -313,3 +313,24 @@ func (c *Catalog) String() string {
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Simulated reports whether this model produces synthetic text rather than
|
||||
// inference output.
|
||||
//
|
||||
// Derived from the profiles rather than configured, because an operator who
|
||||
// added a sim profile and forgot the flag would be shipping placeholder text to
|
||||
// users under a real model's name. A model counts as simulated only when every
|
||||
// profile is simulated: one real profile means a request can land on a real
|
||||
// engine, and a model that sometimes answers for real must not be labelled as
|
||||
// though it never does.
|
||||
func (m Model) Simulated() bool {
|
||||
if len(m.Profiles) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine != "sim" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -233,3 +233,29 @@ func TestNamesAndModelsOrder(t *testing.T) {
|
||||
t.Errorf("Names order=%v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) {
|
||||
// The question the flag answers is "can this request come back as real
|
||||
// inference?". One real profile means yes, however many sim profiles sit
|
||||
// beside it, so a mixed model must not be labelled simulated.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
engines []string
|
||||
wantSim bool
|
||||
}{
|
||||
{"all simulated", []string{"sim", "sim"}, true},
|
||||
{"one real profile", []string{"sim", "llamacpp"}, false},
|
||||
{"no simulation", []string{"llamacpp"}, false},
|
||||
{"no profiles at all", nil, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var m Model
|
||||
for _, e := range tc.engines {
|
||||
m.Profiles = append(m.Profiles, Profile{Engine: e})
|
||||
}
|
||||
if got := m.Simulated(); got != tc.wantSim {
|
||||
t.Errorf("Simulated() = %v, want %v", got, tc.wantSim)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user