Offer a model that loads on demand in the chat picker
The picker listed what had an endpoint, and a model at min 0 has none until somebody asks for it. On this page a message is the asking, so an on-demand model could never be started from here: mistral-nemo-12b-instruct sat in the catalog, served by nothing, and absent from the one place someone went to try it. Such a model is now offered, marked "not loaded", with a note saying the first message fetches the weights and starts the engine, and that until it is up the reply is "the model is loading". A model stopped outright — max 0 — is still left out, because nothing would load it and offering it would only produce an error. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b56130ace3
commit
2cae92d113
4 files changed
+71
-1
No files matched your search
@@ -179,6 +179,23 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A model that loads on demand has no endpoint until somebody asks for it,
|
||||
// and sending it a message is how somebody asks. Listing only what already
|
||||
// runs left such a model unreachable from this page for good.
|
||||
for name, entry := range r.modelMap() {
|
||||
if name != entry.Name || !entry.Scaling.OnDemand() {
|
||||
continue // an alias, or a model that is placed without being asked
|
||||
}
|
||||
if _, up := seen[name]; up {
|
||||
continue
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
if r.permits(principal, name) {
|
||||
out = append(out, web.ChatModel{
|
||||
Name: name, Simulated: entry.Simulated(), OnDemand: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(out, func(a, b web.ChatModel) int {
|
||||
if a.Simulated != b.Simulated {
|
||||
if b.Simulated {
|
||||
|
||||
@@ -136,3 +136,43 @@ func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
|
||||
t.Errorf("a model allowing 1024 got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) {
|
||||
// An on-demand model has no endpoint until a request asks for it, and on
|
||||
// this page a message is that request. Listing only running models left
|
||||
// such a model impossible to start from the chat.
|
||||
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
||||
rt, user := chatRouter(t, e.srv.URL)
|
||||
cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
|
||||
c, err := catalog.New([]catalog.Model{
|
||||
{Name: "qwen2.5-coder-7b-instruct", Aliases: []string{"coder-7b", "coder"}, Source: "http://m/a.gguf", Profiles: cpu},
|
||||
{Name: "mistral-nemo-12b-instruct", Aliases: []string{"nemo"}, Source: "http://m/b.gguf", Profiles: cpu,
|
||||
Scaling: catalog.Scaling{Min: 0, Max: 1}},
|
||||
// Stopped outright: nothing will load it, so offering it would only
|
||||
// produce an error.
|
||||
{Name: "phi-4", Source: "http://m/c.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 0}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rt.SetCatalog(c)
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
|
||||
byName := map[string]bool{}
|
||||
for _, m := range got {
|
||||
byName[m.Name] = m.OnDemand
|
||||
}
|
||||
if onDemand, ok := byName["mistral-nemo-12b-instruct"]; !ok || !onDemand {
|
||||
t.Errorf("the on-demand model is not offered as one: %+v", got)
|
||||
}
|
||||
if _, ok := byName["nemo"]; ok {
|
||||
t.Errorf("the alias is offered as a second model: %+v", got)
|
||||
}
|
||||
if _, ok := byName["phi-4"]; ok {
|
||||
t.Errorf("a stopped model is offered: %+v", got)
|
||||
}
|
||||
if onDemand := byName["qwen2.5-coder-7b-instruct"]; onDemand {
|
||||
t.Errorf("a running model is marked as not loaded")
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,10 @@ type ChatModel struct {
|
||||
// inference output. The picker says so, because the two are
|
||||
// indistinguishable from the outside until the reply arrives.
|
||||
Simulated bool
|
||||
// OnDemand marks a model with nothing running that loads when asked. It is
|
||||
// offered anyway: the first message is what asks, so hiding it until it
|
||||
// runs meant it could never be started from here.
|
||||
OnDemand bool
|
||||
}
|
||||
|
||||
type chatData struct {
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
{{if .Models}}
|
||||
<div class="chat-bar">
|
||||
<label class="inline-label">Model
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
|
||||
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
||||
</label>
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
@@ -30,6 +30,13 @@
|
||||
replies are placeholder text, not inference.
|
||||
</p>
|
||||
|
||||
<p id="ondemand-note" class="note" hidden>
|
||||
This model is not running. The first message loads it onto a node, which
|
||||
means fetching its weights and starting the engine — minutes for a large
|
||||
model. Until it is up, a message is answered with "the model is loading";
|
||||
send it again once it is.
|
||||
</p>
|
||||
|
||||
<div id="log" class="chat-log" aria-live="polite"></div>
|
||||
|
||||
<form id="composer" class="chat-composer">
|
||||
@@ -70,9 +77,11 @@ access to it.</p>
|
||||
// The note follows the picker rather than the reply: someone should know what
|
||||
// they are about to get before they spend a minute waiting for it.
|
||||
const simNote = document.getElementById("sim-note");
|
||||
const onDemandNote = document.getElementById("ondemand-note");
|
||||
function showSimNote() {
|
||||
const opt = model.selectedOptions[0];
|
||||
simNote.hidden = !opt || opt.dataset.simulated !== "1";
|
||||
onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1";
|
||||
}
|
||||
model.addEventListener("change", showSimNote);
|
||||
showSimNote();
|
||||
|
||||
Reference in new issue
Block a user