From 2cae92d1135fb76d09401456fb6b2789ef74b264 Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Mon, 28 Sep 2026 10:13:42 +0200
Subject: [PATCH] Offer a model that loads on demand in the chat picker
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The picker listed what had an endpoint, and a model at min 0 has none until
somebody asks for it. On this page a message is the asking, so an on-demand
model could never be started from here: mistral-nemo-12b-instruct sat in the
catalog, served by nothing, and absent from the one place someone went to
try it.
Such a model is now offered, marked "not loaded", with a note saying the
first message fetches the weights and starts the engine, and that until it
is up the reply is "the model is loading". A model stopped outright — max 0 —
is still left out, because nothing would load it and offering it would only
produce an error.
Co-Authored-By: Claude Opus 5.5 (1M context)
---
internal/router/chat.go | 17 ++++++++++++
internal/router/chat_models_test.go | 40 +++++++++++++++++++++++++++++
internal/web/chat.go | 4 +++
internal/web/templates/chat.html | 11 +++++++-
4 files changed, 71 insertions(+), 1 deletion(-)
diff --git a/internal/router/chat.go b/internal/router/chat.go
index ea75d4a..bbf3cc9 100644
--- a/internal/router/chat.go
+++ b/internal/router/chat.go
@@ -179,6 +179,23 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel {
}
}
}
+ // A model that loads on demand has no endpoint until somebody asks for it,
+ // and sending it a message is how somebody asks. Listing only what already
+ // runs left such a model unreachable from this page for good.
+ for name, entry := range r.modelMap() {
+ if name != entry.Name || !entry.Scaling.OnDemand() {
+ continue // an alias, or a model that is placed without being asked
+ }
+ if _, up := seen[name]; up {
+ continue
+ }
+ seen[name] = struct{}{}
+ if r.permits(principal, name) {
+ out = append(out, web.ChatModel{
+ Name: name, Simulated: entry.Simulated(), OnDemand: true,
+ })
+ }
+ }
slices.SortFunc(out, func(a, b web.ChatModel) int {
if a.Simulated != b.Simulated {
if b.Simulated {
diff --git a/internal/router/chat_models_test.go b/internal/router/chat_models_test.go
index 66da593..1eccaff 100644
--- a/internal/router/chat_models_test.go
+++ b/internal/router/chat_models_test.go
@@ -136,3 +136,43 @@ func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) {
t.Errorf("a model allowing 1024 got %d", got)
}
}
+
+func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) {
+ // An on-demand model has no endpoint until a request asks for it, and on
+ // this page a message is that request. Listing only running models left
+ // such a model impossible to start from the chat.
+ e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
+ rt, user := chatRouter(t, e.srv.URL)
+ cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
+ c, err := catalog.New([]catalog.Model{
+ {Name: "qwen2.5-coder-7b-instruct", Aliases: []string{"coder-7b", "coder"}, Source: "http://m/a.gguf", Profiles: cpu},
+ {Name: "mistral-nemo-12b-instruct", Aliases: []string{"nemo"}, Source: "http://m/b.gguf", Profiles: cpu,
+ Scaling: catalog.Scaling{Min: 0, Max: 1}},
+ // Stopped outright: nothing will load it, so offering it would only
+ // produce an error.
+ {Name: "phi-4", Source: "http://m/c.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 0}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ rt.SetCatalog(c)
+ waitForEndpoint(t, rt)
+
+ got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user)
+ byName := map[string]bool{}
+ for _, m := range got {
+ byName[m.Name] = m.OnDemand
+ }
+ if onDemand, ok := byName["mistral-nemo-12b-instruct"]; !ok || !onDemand {
+ t.Errorf("the on-demand model is not offered as one: %+v", got)
+ }
+ if _, ok := byName["nemo"]; ok {
+ t.Errorf("the alias is offered as a second model: %+v", got)
+ }
+ if _, ok := byName["phi-4"]; ok {
+ t.Errorf("a stopped model is offered: %+v", got)
+ }
+ if onDemand := byName["qwen2.5-coder-7b-instruct"]; onDemand {
+ t.Errorf("a running model is marked as not loaded")
+ }
+}
diff --git a/internal/web/chat.go b/internal/web/chat.go
index f3f3051..b44edc7 100644
--- a/internal/web/chat.go
+++ b/internal/web/chat.go
@@ -14,6 +14,10 @@ type ChatModel struct {
// inference output. The picker says so, because the two are
// indistinguishable from the outside until the reply arrives.
Simulated bool
+ // OnDemand marks a model with nothing running that loads when asked. It is
+ // offered anyway: the first message is what asks, so hiding it until it
+ // runs meant it could never be started from here.
+ OnDemand bool
}
type chatData struct {
diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html
index c2572e6..d644645 100644
--- a/internal/web/templates/chat.html
+++ b/internal/web/templates/chat.html
@@ -20,7 +20,7 @@
{{if .Models}}
@@ -30,6 +30,13 @@
replies are placeholder text, not inference.
+
+ This model is not running. The first message loads it onto a node, which
+ means fetching its weights and starting the engine — minutes for a large
+ model. Until it is up, a message is answered with "the model is loading";
+ send it again once it is.
+