diff --git a/internal/router/chat.go b/internal/router/chat.go index ea75d4a9..bbf3cc95 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -179,6 +179,23 @@ func (r *Router) modelsFor(req *http.Request, user store.User) []web.ChatModel { } } } + // A model that loads on demand has no endpoint until somebody asks for it, + // and sending it a message is how somebody asks. Listing only what already + // runs left such a model unreachable from this page for good. + for name, entry := range r.modelMap() { + if name != entry.Name || !entry.Scaling.OnDemand() { + continue // an alias, or a model that is placed without being asked + } + if _, up := seen[name]; up { + continue + } + seen[name] = struct{}{} + if r.permits(principal, name) { + out = append(out, web.ChatModel{ + Name: name, Simulated: entry.Simulated(), OnDemand: true, + }) + } + } slices.SortFunc(out, func(a, b web.ChatModel) int { if a.Simulated != b.Simulated { if b.Simulated { diff --git a/internal/router/chat_models_test.go b/internal/router/chat_models_test.go index 66da5938..1eccaffd 100644 --- a/internal/router/chat_models_test.go +++ b/internal/router/chat_models_test.go @@ -136,3 +136,43 @@ func TestTheChatCapIsKeptWhereTheModelAllowsMore(t *testing.T) { t.Errorf("a model allowing 1024 got %d", got) } } + +func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) { + // An on-demand model has no endpoint until a request asks for it, and on + // this page a message is that request. Listing only running models left + // such a model impossible to start from the chat. + e := newFakeEngine(t, "qwen2.5-coder-7b-instruct") + rt, user := chatRouter(t, e.srv.URL) + cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}} + c, err := catalog.New([]catalog.Model{ + {Name: "qwen2.5-coder-7b-instruct", Aliases: []string{"coder-7b", "coder"}, Source: "http://m/a.gguf", Profiles: cpu}, + {Name: "mistral-nemo-12b-instruct", Aliases: []string{"nemo"}, Source: "http://m/b.gguf", Profiles: cpu, + Scaling: catalog.Scaling{Min: 0, Max: 1}}, + // Stopped outright: nothing will load it, so offering it would only + // produce an error. + {Name: "phi-4", Source: "http://m/c.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 0}}, + }) + if err != nil { + t.Fatal(err) + } + rt.SetCatalog(c) + waitForEndpoint(t, rt) + + got := rt.modelsFor(httptest.NewRequest("GET", "/chat", nil), user) + byName := map[string]bool{} + for _, m := range got { + byName[m.Name] = m.OnDemand + } + if onDemand, ok := byName["mistral-nemo-12b-instruct"]; !ok || !onDemand { + t.Errorf("the on-demand model is not offered as one: %+v", got) + } + if _, ok := byName["nemo"]; ok { + t.Errorf("the alias is offered as a second model: %+v", got) + } + if _, ok := byName["phi-4"]; ok { + t.Errorf("a stopped model is offered: %+v", got) + } + if onDemand := byName["qwen2.5-coder-7b-instruct"]; onDemand { + t.Errorf("a running model is marked as not loaded") + } +} diff --git a/internal/web/chat.go b/internal/web/chat.go index f3f3051b..b44edc7f 100644 --- a/internal/web/chat.go +++ b/internal/web/chat.go @@ -14,6 +14,10 @@ type ChatModel struct { // inference output. The picker says so, because the two are // indistinguishable from the outside until the reply arrives. Simulated bool + // OnDemand marks a model with nothing running that loads when asked. It is + // offered anyway: the first message is what asks, so hiding it until it + // runs meant it could never be started from here. + OnDemand bool } type chatData struct { diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index c2572e6d..d644645c 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -20,7 +20,7 @@ {{if .Models}}
@@ -30,6 +30,13 @@ replies are placeholder text, not inference.

+ +
@@ -70,9 +77,11 @@ access to it.

// The note follows the picker rather than the reply: someone should know what // they are about to get before they spend a minute waiting for it. const simNote = document.getElementById("sim-note"); + const onDemandNote = document.getElementById("ondemand-note"); function showSimNote() { const opt = model.selectedOptions[0]; simNote.hidden = !opt || opt.dataset.simulated !== "1"; + onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1"; } model.addEventListener("change", showSimNote); showSimNote();