Show a model loading in the chat, and send once it is up

A message to a model that was not running waited 45 seconds and came
back as "the model is loading; retry shortly". The page now asks for
the model with /chat/wake, shows where it stands from
/chat/model-status - waiting for a node, downloading with bytes and a
percentage, starting the engine - and sends the message when it answers.

Also passes the nodes' llama.cpp build to the pages, which the mirror's
checks for loadable weights needed and were never given.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 20:52:31 +02:00
1 parent f13b18149a
commit 5aa5675a9c
7 files changed
+338 -7

No files matched your search

+4
View File
@@ -40,6 +40,10 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) {
web.Chat(user, r.modelsFor(req, user), contexts, current, version.Version)(w, req)
}))
mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend))
// Where a model stands while it loads, and a way to ask for it without
// holding a request open until it is up.
mux.HandleFunc("GET /chat/model-status", pages.Guard(r.handleChatStatus))
mux.HandleFunc("POST /chat/wake", pages.Guard(r.handleChatWake))
// Behind the same guard as the rest of the page: it takes a POST body and
// returns a file, so it wants a session and the CSRF check for the same
// reasons /chat/send does.
+122
View File
@@ -0,0 +1,122 @@
package router
import (
"encoding/json"
"io"
"net/http"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// modelStatus is how far a model is from answering, for the chat page's
// progress bar. A person who asks for a model that is not loaded should see
// it being fetched and started, not a spinner that ends in an error.
type modelStatus struct {
// State is ready, downloading, starting, waiting, failed or stopped.
State string `json:"state"`
// Percent, Done and Total describe a download; Percent is absent when the
// size is unknown, rather than a number that stops moving.
Percent *int `json:"percent,omitempty"`
Done int64 `json:"done,omitempty"`
Total int64 `json:"total,omitempty"`
// Seconds is how long the current stage has run.
Seconds int64 `json:"seconds,omitempty"`
Message string `json:"message,omitempty"`
}
// loadStatus says where a model stands, from what the control plane reports.
func (r *Router) loadStatus(model string) modelStatus {
if len(r.reg.Snapshots(model)) > 0 {
return modelStatus{State: "ready"}
}
if r.clusterView == nil {
return modelStatus{State: "stopped", Message: "nothing is serving this model"}
}
var best *modelStatus
for _, rep := range r.clusterView.Replicas(model) {
st := modelStatus{Seconds: rep.AgeSec}
switch rep.State {
case "ready":
// Up, and about to be published to the router.
return modelStatus{State: "starting", Message: "almost ready"}
case "failed":
st.State, st.Message = "failed", rep.Err
case "placing", "starting":
st.State = "starting"
for _, sh := range rep.Shards {
if sh.Fetch == nil {
continue
}
st.State = "downloading"
st.Done += sh.Fetch.BytesDone
st.Total += sh.Fetch.BytesTotal
}
if st.State == "downloading" && st.Total > 0 {
p := int(st.Done * 100 / st.Total)
st.Percent = &p
}
default:
continue
}
// A replica making progress says more than one that failed.
if best == nil || best.State == "failed" {
s := st
best = &s
}
}
if best != nil {
return *best
}
if _, m := r.resolve(model); m.Scaling.OnDemand() {
return modelStatus{State: "waiting", Message: "waiting for a node with room for it"}
}
return modelStatus{State: "stopped", Message: "this model is not running"}
}
// chatModel resolves the model a chat request names, checking the person may
// use it in the context they chose. Empty when they may not, which answers the
// same as a model that does not exist.
func (r *Router) chatModel(req *http.Request, user store.User, asked, context string) string {
name, _ := r.resolve(asked)
if _, known := r.modelMap()[name]; !known && len(r.reg.Snapshots(name)) == 0 {
return ""
}
p, err := r.principalFor(req, user, context)
if err != nil || !r.permits(p, name) {
return ""
}
return name
}
func (r *Router) handleChatStatus(w http.ResponseWriter, req *http.Request, user store.User) {
name := r.chatModel(req, user, req.URL.Query().Get("model"), req.URL.Query().Get("context"))
if name == "" {
http.NotFound(w, req)
return
}
writeJSON(w, http.StatusOK, r.loadStatus(name))
}
// handleChatWake asks for an on-demand model and returns at once: the page
// shows the load happening rather than holding a request open until it ends.
func (r *Router) handleChatWake(w http.ResponseWriter, req *http.Request, user store.User) {
var in struct {
Model string `json:"model"`
Context string `json:"context"`
}
body, _ := io.ReadAll(io.LimitReader(req.Body, 4096))
if err := json.Unmarshal(body, &in); err != nil || strings.TrimSpace(in.Model) == "" {
http.Error(w, "which model?", http.StatusBadRequest)
return
}
name := r.chatModel(req, user, in.Model, in.Context)
if name == "" {
http.NotFound(w, req)
return
}
if r.activator != nil && len(r.reg.Snapshots(name)) == 0 {
r.activator.Want(name)
}
writeJSON(w, http.StatusOK, r.loadStatus(name))
}
+119
View File
@@ -0,0 +1,119 @@
package router
import (
"encoding/json"
"net/http/httptest"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// replicaSource is the control plane's view with only replicas in it.
type replicaSource struct {
replicas map[string][]adminapi.ReplicaView
}
func (s replicaSource) Nodes() []adminapi.NodeView { return nil }
func (s replicaSource) Models() []adminapi.ModelView { return nil }
func (s replicaSource) Replicas(model string) []adminapi.ReplicaView {
return s.replicas[model]
}
func (s replicaSource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} }
func (s replicaSource) Summary() adminapi.Summary { return adminapi.Summary{} }
func onDemandRouter(t *testing.T) (*Router, *fakeActivator) {
t.Helper()
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
rt, _ := chatRouter(t, e.srv.URL)
cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}
c, err := catalog.New([]catalog.Model{
{Name: "qwen2.5-coder-7b-instruct", Source: "http://m/a.gguf", Profiles: cpu},
{Name: "nemo", Source: "http://m/b.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 1}},
})
if err != nil {
t.Fatal(err)
}
rt.SetCatalog(c)
act := &fakeActivator{onDemand: map[string]bool{"nemo": true}}
rt.SetActivator(act)
waitForEndpoint(t, rt)
return rt, act
}
func TestALoadingModelReportsHowMuchOfItHasArrived(t *testing.T) {
// The chat page draws its bar from this. Two shards of one replica are one
// download as far as the person waiting is concerned.
rt, _ := onDemandRouter(t)
rt.SetClusterView(replicaSource{replicas: map[string][]adminapi.ReplicaView{
"nemo": {{State: "starting", AgeSec: 12, Shards: []adminapi.ShardView{
{Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}},
{Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}},
}}},
}}, nil)
st := rt.loadStatus("nemo")
if st.State != "downloading" || st.Percent == nil || *st.Percent != 25 {
t.Fatalf("got %+v, expected downloading at 25%%", st)
}
if st.Done != 200 || st.Total != 800 {
t.Errorf("bytes %d of %d, expected 200 of 800", st.Done, st.Total)
}
}
func TestAModelWithNothingPlacedYetIsWaitingNotStopped(t *testing.T) {
// Between the wake and the placement there is no replica at all. Calling
// that "stopped" would end the bar the moment it appeared.
rt, _ := onDemandRouter(t)
rt.SetClusterView(replicaSource{}, nil)
if st := rt.loadStatus("nemo"); st.State != "waiting" {
t.Errorf("got %+v, expected waiting", st)
}
}
func TestARunningModelIsReady(t *testing.T) {
rt, _ := onDemandRouter(t)
if st := rt.loadStatus("qwen2.5-coder-7b-instruct"); st.State != "ready" {
t.Errorf("got %+v, expected ready", st)
}
}
func TestWakingAModelAsksForItAndAnswersAtOnce(t *testing.T) {
rt, act := onDemandRouter(t)
rt.SetClusterView(replicaSource{}, nil)
_, user := chatRouter(t, "http://unused")
user.ProjectID = ""
req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nemo"}`))
rec := httptest.NewRecorder()
rt.handleChatWake(rec, req, user)
if rec.Code != 200 {
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
}
var st modelStatus
if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil {
t.Fatal(err)
}
if st.State != "waiting" {
t.Errorf("got %+v, expected waiting", st)
}
if len(act.asked) != 1 || act.asked[0] != "nemo" {
t.Errorf("the activator was asked for %v", act.asked)
}
}
func TestWakingAnUnknownModelIsNotFound(t *testing.T) {
rt, act := onDemandRouter(t)
_, user := chatRouter(t, "http://unused")
user.ProjectID = ""
req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nope"}`))
rec := httptest.NewRecorder()
rt.handleChatWake(rec, req, user)
if rec.Code != 404 {
t.Errorf("status %d, expected 404", rec.Code)
}
if len(act.asked) != 0 {
t.Errorf("an unknown model was asked for: %v", act.asked)
}
}
+1
View File
@@ -146,6 +146,7 @@ func (r *Router) Handler() http.Handler {
// running, rather than a placeholder that gets sent as-is.
pages.ServedModels(r.exampleModels)
pages.SetPersonalOff(r.personalOff)
pages.SetLlamaCppVersion(r.llamaVersion)
pages.SetPersonalDefaults(r.personalDefaults)
pages.Register(mux)
r.registerChat(mux, pages)
+4
View File
@@ -93,6 +93,9 @@ type Router struct {
// personalDefaults is what personal use is held to when nobody wrote a
// limit for the person, for the quota page to say so.
personalDefaults *storepkg.Limits
// llamaVersion is the llama.cpp build the nodes install, so the mirror page
// can refuse weights that build cannot load before fetching them.
llamaVersion string
// ownsThreads is true when threads is not store and so must be closed here.
ownsThreads bool
recorder *accounting.Recorder
@@ -247,6 +250,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal))
r.personalOff = cfg.Auth.Personal.Disabled
r.personalDefaults = personalLimits(cfg.Auth.Personal)
r.llamaVersion = cfg.WantedEngines()["llamacpp"]
r.auth = auth.New(st, auth.Config{
TTL: cfg.Auth.CacheTTL,
+3
View File
@@ -96,6 +96,9 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line);
.bubble.you { background: var(--accent); color: var(--bg); margin-left: auto; }
.bubble.assistant { background: var(--bg); border: 1px solid var(--line); }
.bubble.failed { color: var(--bad); border-color: var(--bad); }
.bubble .loading { min-width: 16rem; }
.bubble .loading .stage { font-size: .85rem; color: var(--muted); margin-bottom: .35rem; }
.bubble .loading progress { width: 100%; height: .5rem; accent-color: var(--accent); }
.chat-composer { display: flex; gap: .6rem; margin-top: .75rem; align-items: flex-end; }
.chat-composer textarea { flex: 1; font: inherit; padding: .5rem .6rem; resize: vertical;
border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); }
+85 -7
View File
@@ -40,10 +40,9 @@
</p>
<p id="ondemand-note" class="note" hidden>
This model is not running. The first message loads it onto a node, which
means fetching its weights and starting the engine — minutes for a large
model. Until it is up, a message is answered with "the model is loading";
send it again once it is.
This model is not running. The first message loads it onto a node — fetching
its weights and starting the engine, minutes for a large model — and is sent
once it is up.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
@@ -168,6 +167,69 @@ access to it.</p>
}
}
const ctxValue = () => (document.getElementById("context") || {}).value || "";
const stages = {
waiting: "Waiting for a node with room",
downloading: "Downloading weights",
starting: "Starting the engine",
ready: "Ready",
};
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
// Loads a model that is not running and shows how far along it is, in the
// reply's own bubble. Resolves true once it answers, false if it cannot.
//
// Asked for with /chat/wake rather than by sending the message and waiting:
// that request gives up after 45 seconds, and a large model takes minutes.
async function load(name, out) {
const box = document.createElement("div");
box.className = "loading";
const label = document.createElement("div");
label.className = "stage";
const bar = document.createElement("progress");
bar.max = 100;
box.append(label, bar);
out.replaceChildren(box);
const started = Date.now();
let failures = 0;
let st;
try {
const resp = await fetch("/chat/wake", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: name, context: ctxValue() }),
});
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
st = await resp.json();
for (;;) {
const secs = Math.round((Date.now() - started) / 1000);
let text = (stages[st.state] || st.state) + " — " + secs + "s";
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
label.textContent = text;
// Indeterminate (no value) unless there is a real number: a bar that
// sits at 0% for the engine start says it is stuck when it is not.
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
if (st.state === "ready") return true;
// A replica that failed before the wake is still listed until the
// control plane places its replacement, so one sighting is not final.
failures = st.state === "failed" ? failures + 1 : 0;
if (st.state === "stopped" || failures >= 5) {
throw new Error(st.message || "the model could not be started");
}
await new Promise((r) => setTimeout(r, 1500));
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
"&context=" + encodeURIComponent(ctxValue()));
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
st = await r.json();
}
} catch (err) {
out.textContent = String(err.message || err);
out.classList.add("failed");
return false;
}
}
form.onsubmit = async (e) => {
e.preventDefault();
const text = input.value.trim();
@@ -181,13 +243,29 @@ access to it.</p>
const out = bubble("assistant");
let reply = "";
let finish = "";
const opt = model.selectedOptions[0];
try {
const resp = await fetch("/chat/send", {
if (opt && opt.dataset.ondemand === "1") {
if (!(await load(model.value, out))) { messages.pop(); return; }
// Up now: later messages go straight through.
opt.dataset.ondemand = "0";
opt.textContent = opt.value;
showSimNote();
out.textContent = "";
}
const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, messages: messages,
context: (document.getElementById("context") || {}).value || "" }),
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
});
let resp = await post();
// Stopped since the page was drawn, or scaled away by idleness: load it
// again and retry once, rather than telling the person to.
if (resp.status === 503 && resp.headers.get("Retry-After")) {
if (!(await load(model.value, out))) { messages.pop(); return; }
out.textContent = "";
resp = await post();
}
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");