Show a model loading in the chat, and send once it is up

A message to a model that was not running waited 45 seconds and came
back as "the model is loading; retry shortly". The page now asks for
the model with /chat/wake, shows where it stands from
/chat/model-status - waiting for a node, downloading with bytes and a
percentage, starting the engine - and sends the message when it answers.

Also passes the nodes' llama.cpp build to the pages, which the mirror's
checks for loadable weights needed and were never given.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-28 20:52:31 +02:00
1 parent f13b18149a
commit 5aa5675a9c
7 files changed
+338 -7

No files matched your search

+85 -7
View File
@@ -40,10 +40,9 @@
</p>
<p id="ondemand-note" class="note" hidden>
This model is not running. The first message loads it onto a node, which
means fetching its weights and starting the engine — minutes for a large
model. Until it is up, a message is answered with "the model is loading";
send it again once it is.
This model is not running. The first message loads it onto a node — fetching
its weights and starting the engine, minutes for a large model — and is sent
once it is up.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
@@ -168,6 +167,69 @@ access to it.</p>
}
}
const ctxValue = () => (document.getElementById("context") || {}).value || "";
const stages = {
waiting: "Waiting for a node with room",
downloading: "Downloading weights",
starting: "Starting the engine",
ready: "Ready",
};
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
// Loads a model that is not running and shows how far along it is, in the
// reply's own bubble. Resolves true once it answers, false if it cannot.
//
// Asked for with /chat/wake rather than by sending the message and waiting:
// that request gives up after 45 seconds, and a large model takes minutes.
async function load(name, out) {
const box = document.createElement("div");
box.className = "loading";
const label = document.createElement("div");
label.className = "stage";
const bar = document.createElement("progress");
bar.max = 100;
box.append(label, bar);
out.replaceChildren(box);
const started = Date.now();
let failures = 0;
let st;
try {
const resp = await fetch("/chat/wake", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: name, context: ctxValue() }),
});
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
st = await resp.json();
for (;;) {
const secs = Math.round((Date.now() - started) / 1000);
let text = (stages[st.state] || st.state) + " — " + secs + "s";
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
label.textContent = text;
// Indeterminate (no value) unless there is a real number: a bar that
// sits at 0% for the engine start says it is stuck when it is not.
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
if (st.state === "ready") return true;
// A replica that failed before the wake is still listed until the
// control plane places its replacement, so one sighting is not final.
failures = st.state === "failed" ? failures + 1 : 0;
if (st.state === "stopped" || failures >= 5) {
throw new Error(st.message || "the model could not be started");
}
await new Promise((r) => setTimeout(r, 1500));
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
"&context=" + encodeURIComponent(ctxValue()));
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
st = await r.json();
}
} catch (err) {
out.textContent = String(err.message || err);
out.classList.add("failed");
return false;
}
}
form.onsubmit = async (e) => {
e.preventDefault();
const text = input.value.trim();
@@ -181,13 +243,29 @@ access to it.</p>
const out = bubble("assistant");
let reply = "";
let finish = "";
const opt = model.selectedOptions[0];
try {
const resp = await fetch("/chat/send", {
if (opt && opt.dataset.ondemand === "1") {
if (!(await load(model.value, out))) { messages.pop(); return; }
// Up now: later messages go straight through.
opt.dataset.ondemand = "0";
opt.textContent = opt.value;
showSimNote();
out.textContent = "";
}
const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, messages: messages,
context: (document.getElementById("context") || {}).value || "" }),
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
});
let resp = await post();
// Stopped since the page was drawn, or scaled away by idleness: load it
// again and retry once, rather than telling the person to.
if (resp.status === 503 && resp.headers.get("Retry-After")) {
if (!(await load(model.value, out))) { messages.pop(); return; }
out.textContent = "";
resp = await post();
}
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");