From 5aa5675a9cd2a10dfeb52f98756b02be92213b79 Mon Sep 17 00:00:00 2001
From: Ilker Manap
Date: Mon, 28 Sep 2026 20:52:31 +0200
Subject: [PATCH] Show a model loading in the chat, and send once it is up
A message to a model that was not running waited 45 seconds and came
back as "the model is loading; retry shortly". The page now asks for
the model with /chat/wake, shows where it stands from
/chat/model-status - waiting for a node, downloading with bytes and a
percentage, starting the engine - and sends the message when it answers.
Also passes the nodes' llama.cpp build to the pages, which the mirror's
checks for loadable weights needed and were never given.
Co-Authored-By: Claude Opus 5.5 (1M context)
---
internal/router/chat.go | 4 +
internal/router/chat_load.go | 122 ++++++++++++++++++++++++++++++
internal/router/chat_load_test.go | 119 +++++++++++++++++++++++++++++
internal/router/handlers.go | 1 +
internal/router/router.go | 4 +
internal/web/static/base.css | 3 +
internal/web/templates/chat.html | 92 ++++++++++++++++++++--
7 files changed, 338 insertions(+), 7 deletions(-)
create mode 100644 internal/router/chat_load.go
create mode 100644 internal/router/chat_load_test.go
diff --git a/internal/router/chat.go b/internal/router/chat.go
index 99f06435..097af774 100644
--- a/internal/router/chat.go
+++ b/internal/router/chat.go
@@ -40,6 +40,10 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) {
web.Chat(user, r.modelsFor(req, user), contexts, current, version.Version)(w, req)
}))
mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend))
+ // Where a model stands while it loads, and a way to ask for it without
+ // holding a request open until it is up.
+ mux.HandleFunc("GET /chat/model-status", pages.Guard(r.handleChatStatus))
+ mux.HandleFunc("POST /chat/wake", pages.Guard(r.handleChatWake))
// Behind the same guard as the rest of the page: it takes a POST body and
// returns a file, so it wants a session and the CSRF check for the same
// reasons /chat/send does.
diff --git a/internal/router/chat_load.go b/internal/router/chat_load.go
new file mode 100644
index 00000000..afe0d4a9
--- /dev/null
+++ b/internal/router/chat_load.go
@@ -0,0 +1,122 @@
+package router
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "strings"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// modelStatus is how far a model is from answering, for the chat page's
+// progress bar. A person who asks for a model that is not loaded should see
+// it being fetched and started, not a spinner that ends in an error.
+type modelStatus struct {
+ // State is ready, downloading, starting, waiting, failed or stopped.
+ State string `json:"state"`
+ // Percent, Done and Total describe a download; Percent is absent when the
+ // size is unknown, rather than a number that stops moving.
+ Percent *int `json:"percent,omitempty"`
+ Done int64 `json:"done,omitempty"`
+ Total int64 `json:"total,omitempty"`
+ // Seconds is how long the current stage has run.
+ Seconds int64 `json:"seconds,omitempty"`
+ Message string `json:"message,omitempty"`
+}
+
+// loadStatus says where a model stands, from what the control plane reports.
+func (r *Router) loadStatus(model string) modelStatus {
+ if len(r.reg.Snapshots(model)) > 0 {
+ return modelStatus{State: "ready"}
+ }
+ if r.clusterView == nil {
+ return modelStatus{State: "stopped", Message: "nothing is serving this model"}
+ }
+ var best *modelStatus
+ for _, rep := range r.clusterView.Replicas(model) {
+ st := modelStatus{Seconds: rep.AgeSec}
+ switch rep.State {
+ case "ready":
+ // Up, and about to be published to the router.
+ return modelStatus{State: "starting", Message: "almost ready"}
+ case "failed":
+ st.State, st.Message = "failed", rep.Err
+ case "placing", "starting":
+ st.State = "starting"
+ for _, sh := range rep.Shards {
+ if sh.Fetch == nil {
+ continue
+ }
+ st.State = "downloading"
+ st.Done += sh.Fetch.BytesDone
+ st.Total += sh.Fetch.BytesTotal
+ }
+ if st.State == "downloading" && st.Total > 0 {
+ p := int(st.Done * 100 / st.Total)
+ st.Percent = &p
+ }
+ default:
+ continue
+ }
+ // A replica making progress says more than one that failed.
+ if best == nil || best.State == "failed" {
+ s := st
+ best = &s
+ }
+ }
+ if best != nil {
+ return *best
+ }
+ if _, m := r.resolve(model); m.Scaling.OnDemand() {
+ return modelStatus{State: "waiting", Message: "waiting for a node with room for it"}
+ }
+ return modelStatus{State: "stopped", Message: "this model is not running"}
+}
+
+// chatModel resolves the model a chat request names, checking the person may
+// use it in the context they chose. Empty when they may not, which answers the
+// same as a model that does not exist.
+func (r *Router) chatModel(req *http.Request, user store.User, asked, context string) string {
+ name, _ := r.resolve(asked)
+ if _, known := r.modelMap()[name]; !known && len(r.reg.Snapshots(name)) == 0 {
+ return ""
+ }
+ p, err := r.principalFor(req, user, context)
+ if err != nil || !r.permits(p, name) {
+ return ""
+ }
+ return name
+}
+
+func (r *Router) handleChatStatus(w http.ResponseWriter, req *http.Request, user store.User) {
+ name := r.chatModel(req, user, req.URL.Query().Get("model"), req.URL.Query().Get("context"))
+ if name == "" {
+ http.NotFound(w, req)
+ return
+ }
+ writeJSON(w, http.StatusOK, r.loadStatus(name))
+}
+
+// handleChatWake asks for an on-demand model and returns at once: the page
+// shows the load happening rather than holding a request open until it ends.
+func (r *Router) handleChatWake(w http.ResponseWriter, req *http.Request, user store.User) {
+ var in struct {
+ Model string `json:"model"`
+ Context string `json:"context"`
+ }
+ body, _ := io.ReadAll(io.LimitReader(req.Body, 4096))
+ if err := json.Unmarshal(body, &in); err != nil || strings.TrimSpace(in.Model) == "" {
+ http.Error(w, "which model?", http.StatusBadRequest)
+ return
+ }
+ name := r.chatModel(req, user, in.Model, in.Context)
+ if name == "" {
+ http.NotFound(w, req)
+ return
+ }
+ if r.activator != nil && len(r.reg.Snapshots(name)) == 0 {
+ r.activator.Want(name)
+ }
+ writeJSON(w, http.StatusOK, r.loadStatus(name))
+}
diff --git a/internal/router/chat_load_test.go b/internal/router/chat_load_test.go
new file mode 100644
index 00000000..a0399caa
--- /dev/null
+++ b/internal/router/chat_load_test.go
@@ -0,0 +1,119 @@
+package router
+
+import (
+ "encoding/json"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+)
+
+// replicaSource is the control plane's view with only replicas in it.
+type replicaSource struct {
+ replicas map[string][]adminapi.ReplicaView
+}
+
+func (s replicaSource) Nodes() []adminapi.NodeView { return nil }
+func (s replicaSource) Models() []adminapi.ModelView { return nil }
+func (s replicaSource) Replicas(model string) []adminapi.ReplicaView {
+ return s.replicas[model]
+}
+func (s replicaSource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
+func (s replicaSource) Summary() adminapi.Summary { return adminapi.Summary{} }
+
+func onDemandRouter(t *testing.T) (*Router, *fakeActivator) {
+ t.Helper()
+ e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
+ rt, _ := chatRouter(t, e.srv.URL)
+ cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
+ c, err := catalog.New([]catalog.Model{
+ {Name: "qwen2.5-coder-7b-instruct", Source: "http://m/a.gguf", Profiles: cpu},
+ {Name: "nemo", Source: "http://m/b.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 1}},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ rt.SetCatalog(c)
+ act := &fakeActivator{onDemand: map[string]bool{"nemo": true}}
+ rt.SetActivator(act)
+ waitForEndpoint(t, rt)
+ return rt, act
+}
+
+func TestALoadingModelReportsHowMuchOfItHasArrived(t *testing.T) {
+ // The chat page draws its bar from this. Two shards of one replica are one
+ // download as far as the person waiting is concerned.
+ rt, _ := onDemandRouter(t)
+ rt.SetClusterView(replicaSource{replicas: map[string][]adminapi.ReplicaView{
+ "nemo": {{State: "starting", AgeSec: 12, Shards: []adminapi.ShardView{
+ {Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}},
+ {Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}},
+ }}},
+ }}, nil)
+
+ st := rt.loadStatus("nemo")
+ if st.State != "downloading" || st.Percent == nil || *st.Percent != 25 {
+ t.Fatalf("got %+v, expected downloading at 25%%", st)
+ }
+ if st.Done != 200 || st.Total != 800 {
+ t.Errorf("bytes %d of %d, expected 200 of 800", st.Done, st.Total)
+ }
+}
+
+func TestAModelWithNothingPlacedYetIsWaitingNotStopped(t *testing.T) {
+ // Between the wake and the placement there is no replica at all. Calling
+ // that "stopped" would end the bar the moment it appeared.
+ rt, _ := onDemandRouter(t)
+ rt.SetClusterView(replicaSource{}, nil)
+ if st := rt.loadStatus("nemo"); st.State != "waiting" {
+ t.Errorf("got %+v, expected waiting", st)
+ }
+}
+
+func TestARunningModelIsReady(t *testing.T) {
+ rt, _ := onDemandRouter(t)
+ if st := rt.loadStatus("qwen2.5-coder-7b-instruct"); st.State != "ready" {
+ t.Errorf("got %+v, expected ready", st)
+ }
+}
+
+func TestWakingAModelAsksForItAndAnswersAtOnce(t *testing.T) {
+ rt, act := onDemandRouter(t)
+ rt.SetClusterView(replicaSource{}, nil)
+ _, user := chatRouter(t, "http://unused")
+ user.ProjectID = ""
+
+ req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nemo"}`))
+ rec := httptest.NewRecorder()
+ rt.handleChatWake(rec, req, user)
+ if rec.Code != 200 {
+ t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
+ }
+ var st modelStatus
+ if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil {
+ t.Fatal(err)
+ }
+ if st.State != "waiting" {
+ t.Errorf("got %+v, expected waiting", st)
+ }
+ if len(act.asked) != 1 || act.asked[0] != "nemo" {
+ t.Errorf("the activator was asked for %v", act.asked)
+ }
+}
+
+func TestWakingAnUnknownModelIsNotFound(t *testing.T) {
+ rt, act := onDemandRouter(t)
+ _, user := chatRouter(t, "http://unused")
+ user.ProjectID = ""
+ req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nope"}`))
+ rec := httptest.NewRecorder()
+ rt.handleChatWake(rec, req, user)
+ if rec.Code != 404 {
+ t.Errorf("status %d, expected 404", rec.Code)
+ }
+ if len(act.asked) != 0 {
+ t.Errorf("an unknown model was asked for: %v", act.asked)
+ }
+}
diff --git a/internal/router/handlers.go b/internal/router/handlers.go
index f9a5d4d9..ff372b71 100644
--- a/internal/router/handlers.go
+++ b/internal/router/handlers.go
@@ -146,6 +146,7 @@ func (r *Router) Handler() http.Handler {
// running, rather than a placeholder that gets sent as-is.
pages.ServedModels(r.exampleModels)
pages.SetPersonalOff(r.personalOff)
+ pages.SetLlamaCppVersion(r.llamaVersion)
pages.SetPersonalDefaults(r.personalDefaults)
pages.Register(mux)
r.registerChat(mux, pages)
diff --git a/internal/router/router.go b/internal/router/router.go
index 408108ea..9e3c40e1 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -93,6 +93,9 @@ type Router struct {
// personalDefaults is what personal use is held to when nobody wrote a
// limit for the person, for the quota page to say so.
personalDefaults *storepkg.Limits
+ // llamaVersion is the llama.cpp build the nodes install, so the mirror page
+ // can refuse weights that build cannot load before fetching them.
+ llamaVersion string
// ownsThreads is true when threads is not store and so must be closed here.
ownsThreads bool
recorder *accounting.Recorder
@@ -247,6 +250,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal))
r.personalOff = cfg.Auth.Personal.Disabled
r.personalDefaults = personalLimits(cfg.Auth.Personal)
+ r.llamaVersion = cfg.WantedEngines()["llamacpp"]
r.auth = auth.New(st, auth.Config{
TTL: cfg.Auth.CacheTTL,
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index 870406a4..59ff2eee 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -96,6 +96,9 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line);
.bubble.you { background: var(--accent); color: var(--bg); margin-left: auto; }
.bubble.assistant { background: var(--bg); border: 1px solid var(--line); }
.bubble.failed { color: var(--bad); border-color: var(--bad); }
+.bubble .loading { min-width: 16rem; }
+.bubble .loading .stage { font-size: .85rem; color: var(--muted); margin-bottom: .35rem; }
+.bubble .loading progress { width: 100%; height: .5rem; accent-color: var(--accent); }
.chat-composer { display: flex; gap: .6rem; margin-top: .75rem; align-items: flex-end; }
.chat-composer textarea { flex: 1; font: inherit; padding: .5rem .6rem; resize: vertical;
border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); }
diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html
index 0fb78815..b448f05e 100644
--- a/internal/web/templates/chat.html
+++ b/internal/web/templates/chat.html
@@ -40,10 +40,9 @@
- This model is not running. The first message loads it onto a node, which
- means fetching its weights and starting the engine — minutes for a large
- model. Until it is up, a message is answered with "the model is loading";
- send it again once it is.
+ This model is not running. The first message loads it onto a node — fetching
+ its weights and starting the engine, minutes for a large model — and is sent
+ once it is up.
@@ -168,6 +167,69 @@ access to it.
}
}
+ const ctxValue = () => (document.getElementById("context") || {}).value || "";
+ const stages = {
+ waiting: "Waiting for a node with room",
+ downloading: "Downloading weights",
+ starting: "Starting the engine",
+ ready: "Ready",
+ };
+ function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
+
+ // Loads a model that is not running and shows how far along it is, in the
+ // reply's own bubble. Resolves true once it answers, false if it cannot.
+ //
+ // Asked for with /chat/wake rather than by sending the message and waiting:
+ // that request gives up after 45 seconds, and a large model takes minutes.
+ async function load(name, out) {
+ const box = document.createElement("div");
+ box.className = "loading";
+ const label = document.createElement("div");
+ label.className = "stage";
+ const bar = document.createElement("progress");
+ bar.max = 100;
+ box.append(label, bar);
+ out.replaceChildren(box);
+ const started = Date.now();
+ let failures = 0;
+
+ let st;
+ try {
+ const resp = await fetch("/chat/wake", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
+ body: JSON.stringify({ model: name, context: ctxValue() }),
+ });
+ if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
+ st = await resp.json();
+ for (;;) {
+ const secs = Math.round((Date.now() - started) / 1000);
+ let text = (stages[st.state] || st.state) + " — " + secs + "s";
+ if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
+ label.textContent = text;
+ // Indeterminate (no value) unless there is a real number: a bar that
+ // sits at 0% for the engine start says it is stuck when it is not.
+ if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
+ if (st.state === "ready") return true;
+ // A replica that failed before the wake is still listed until the
+ // control plane places its replacement, so one sighting is not final.
+ failures = st.state === "failed" ? failures + 1 : 0;
+ if (st.state === "stopped" || failures >= 5) {
+ throw new Error(st.message || "the model could not be started");
+ }
+ await new Promise((r) => setTimeout(r, 1500));
+ const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
+ "&context=" + encodeURIComponent(ctxValue()));
+ if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
+ st = await r.json();
+ }
+ } catch (err) {
+ out.textContent = String(err.message || err);
+ out.classList.add("failed");
+ return false;
+ }
+ }
+
form.onsubmit = async (e) => {
e.preventDefault();
const text = input.value.trim();
@@ -181,13 +243,29 @@ access to it.
const out = bubble("assistant");
let reply = "";
let finish = "";
+ const opt = model.selectedOptions[0];
try {
- const resp = await fetch("/chat/send", {
+ if (opt && opt.dataset.ondemand === "1") {
+ if (!(await load(model.value, out))) { messages.pop(); return; }
+ // Up now: later messages go straight through.
+ opt.dataset.ondemand = "0";
+ opt.textContent = opt.value;
+ showSimNote();
+ out.textContent = "";
+ }
+ const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
- body: JSON.stringify({ model: model.value, messages: messages,
- context: (document.getElementById("context") || {}).value || "" }),
+ body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
});
+ let resp = await post();
+ // Stopped since the page was drawn, or scaled away by idleness: load it
+ // again and retry once, rather than telling the person to.
+ if (resp.status === 503 && resp.headers.get("Retry-After")) {
+ if (!(await load(model.value, out))) { messages.pop(); return; }
+ out.textContent = "";
+ resp = await post();
+ }
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");