Show a model loading in the chat, and send once it is up
A message to a model that was not running waited 45 seconds and came back as "the model is loading; retry shortly". The page now asks for the model with /chat/wake, shows where it stands from /chat/model-status - waiting for a node, downloading with bytes and a percentage, starting the engine - and sends the message when it answers. Also passes the nodes' llama.cpp build to the pages, which the mirror's checks for loadable weights needed and were never given. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f13b18149a
commit
5aa5675a9c
7 files changed
+338
-7
No files matched your search
@@ -40,10 +40,9 @@
|
||||
</p>
|
||||
|
||||
<p id="ondemand-note" class="note" hidden>
|
||||
This model is not running. The first message loads it onto a node, which
|
||||
means fetching its weights and starting the engine — minutes for a large
|
||||
model. Until it is up, a message is answered with "the model is loading";
|
||||
send it again once it is.
|
||||
This model is not running. The first message loads it onto a node — fetching
|
||||
its weights and starting the engine, minutes for a large model — and is sent
|
||||
once it is up.
|
||||
</p>
|
||||
|
||||
<div id="log" class="chat-log" aria-live="polite"></div>
|
||||
@@ -168,6 +167,69 @@ access to it.</p>
|
||||
}
|
||||
}
|
||||
|
||||
const ctxValue = () => (document.getElementById("context") || {}).value || "";
|
||||
const stages = {
|
||||
waiting: "Waiting for a node with room",
|
||||
downloading: "Downloading weights",
|
||||
starting: "Starting the engine",
|
||||
ready: "Ready",
|
||||
};
|
||||
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
|
||||
|
||||
// Loads a model that is not running and shows how far along it is, in the
|
||||
// reply's own bubble. Resolves true once it answers, false if it cannot.
|
||||
//
|
||||
// Asked for with /chat/wake rather than by sending the message and waiting:
|
||||
// that request gives up after 45 seconds, and a large model takes minutes.
|
||||
async function load(name, out) {
|
||||
const box = document.createElement("div");
|
||||
box.className = "loading";
|
||||
const label = document.createElement("div");
|
||||
label.className = "stage";
|
||||
const bar = document.createElement("progress");
|
||||
bar.max = 100;
|
||||
box.append(label, bar);
|
||||
out.replaceChildren(box);
|
||||
const started = Date.now();
|
||||
let failures = 0;
|
||||
|
||||
let st;
|
||||
try {
|
||||
const resp = await fetch("/chat/wake", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
||||
body: JSON.stringify({ model: name, context: ctxValue() }),
|
||||
});
|
||||
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
|
||||
st = await resp.json();
|
||||
for (;;) {
|
||||
const secs = Math.round((Date.now() - started) / 1000);
|
||||
let text = (stages[st.state] || st.state) + " — " + secs + "s";
|
||||
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
|
||||
label.textContent = text;
|
||||
// Indeterminate (no value) unless there is a real number: a bar that
|
||||
// sits at 0% for the engine start says it is stuck when it is not.
|
||||
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
|
||||
if (st.state === "ready") return true;
|
||||
// A replica that failed before the wake is still listed until the
|
||||
// control plane places its replacement, so one sighting is not final.
|
||||
failures = st.state === "failed" ? failures + 1 : 0;
|
||||
if (st.state === "stopped" || failures >= 5) {
|
||||
throw new Error(st.message || "the model could not be started");
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
|
||||
"&context=" + encodeURIComponent(ctxValue()));
|
||||
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
|
||||
st = await r.json();
|
||||
}
|
||||
} catch (err) {
|
||||
out.textContent = String(err.message || err);
|
||||
out.classList.add("failed");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
form.onsubmit = async (e) => {
|
||||
e.preventDefault();
|
||||
const text = input.value.trim();
|
||||
@@ -181,13 +243,29 @@ access to it.</p>
|
||||
const out = bubble("assistant");
|
||||
let reply = "";
|
||||
let finish = "";
|
||||
const opt = model.selectedOptions[0];
|
||||
try {
|
||||
const resp = await fetch("/chat/send", {
|
||||
if (opt && opt.dataset.ondemand === "1") {
|
||||
if (!(await load(model.value, out))) { messages.pop(); return; }
|
||||
// Up now: later messages go straight through.
|
||||
opt.dataset.ondemand = "0";
|
||||
opt.textContent = opt.value;
|
||||
showSimNote();
|
||||
out.textContent = "";
|
||||
}
|
||||
const post = () => fetch("/chat/send", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
||||
body: JSON.stringify({ model: model.value, messages: messages,
|
||||
context: (document.getElementById("context") || {}).value || "" }),
|
||||
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
|
||||
});
|
||||
let resp = await post();
|
||||
// Stopped since the page was drawn, or scaled away by idleness: load it
|
||||
// again and retry once, rather than telling the person to.
|
||||
if (resp.status === 503 && resp.headers.get("Retry-After")) {
|
||||
if (!(await load(model.value, out))) { messages.pop(); return; }
|
||||
out.textContent = "";
|
||||
resp = await post();
|
||||
}
|
||||
if (!resp.ok) {
|
||||
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
|
||||
out.classList.add("failed");
|
||||
|
||||
Reference in new issue
Block a user