A message to a model that was not running waited 45 seconds and came back as "the model is loading; retry shortly". The page now asks for the model with /chat/wake, shows where it stands from /chat/model-status - waiting for a node, downloading with bytes and a percentage, starting the engine - and sends the message when it answers. Also passes the nodes' llama.cpp build to the pages, which the mirror's checks for loadable weights needed and were never given. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
342 lines
14 KiB
HTML
342 lines
14 KiB
HTML
{{define "chat"}}<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Chat · llmgw</title><link rel="stylesheet" href="/_static/base.css">
|
|
<link rel="stylesheet" href="/_static/katex/katex.min.css">
|
|
<script defer src="/_static/katex/katex.min.js"></script>
|
|
<script defer src="/_static/katex/auto-render.min.js"></script>
|
|
<meta name="csrf" content="{{.CSRF}}">
|
|
</head><body><div class="wrap">
|
|
<header>
|
|
<h1>Chat</h1>
|
|
<p class="sub">{{.User.Email}}</p>
|
|
{{template "nav" .Nav}}
|
|
</header>
|
|
|
|
{{if gt (len .Contexts) 1}}
|
|
{{/* What this conversation spends against. A plain GET, so the model list
|
|
below is the one this choice allows; the server checks it again on every
|
|
message rather than trusting the page. */}}
|
|
<form method="get" action="/chat" class="chat-context">
|
|
<label class="inline-label">Using
|
|
<select id="context" name="context" onchange="this.form.submit()">
|
|
{{range .Contexts}}<option value="{{.Value}}"{{if eq .Value $.Context}} selected{{end}}>{{.Label}}</option>{{end}}
|
|
</select></label>
|
|
<noscript><button type="submit" class="link">switch</button></noscript>
|
|
</form>
|
|
{{else if .Contexts}}{{with index .Contexts 0}}<input type="hidden" id="context" value="{{.Value}}">{{end}}{{end}}
|
|
|
|
{{if .Models}}
|
|
<div class="chat-bar">
|
|
<label class="inline-label">Model
|
|
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
|
</label>
|
|
<button id="reset" type="button" class="link">New conversation</button>
|
|
</div>
|
|
|
|
<p id="sim-note" class="note" hidden>
|
|
This model is simulated: it exercises scheduling and reports timings, and its
|
|
replies are placeholder text, not inference.
|
|
</p>
|
|
|
|
<p id="ondemand-note" class="note" hidden>
|
|
This model is not running. The first message loads it onto a node — fetching
|
|
its weights and starting the engine, minutes for a large model — and is sent
|
|
once it is up.
|
|
</p>
|
|
|
|
<div id="log" class="chat-log" aria-live="polite"></div>
|
|
|
|
<form id="composer" class="chat-composer">
|
|
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
|
|
<button type="submit" id="send">Send</button>
|
|
</form>
|
|
<p class="note">Nothing here is stored. The conversation lives in this page and
|
|
is gone when you close it — what is recorded is the same usage any request
|
|
records: tokens, cost and latency.</p>
|
|
{{else}}
|
|
<p class="empty">No models are available to you right now.</p>
|
|
<p class="note">Either nothing is placed yet, or your project's allowlist does
|
|
not include anything that is. <code>/admin</code> shows the cluster if you have
|
|
access to it.</p>
|
|
{{end}}
|
|
|
|
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
|
|
</div>
|
|
|
|
<script>
|
|
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
|
|
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
|
|
// paragraph feels broken, and on CPU inference ten seconds is normal.
|
|
(function () {
|
|
const log = document.getElementById("log");
|
|
if (!log) return;
|
|
const form = document.getElementById("composer");
|
|
const input = document.getElementById("input");
|
|
const send = document.getElementById("send");
|
|
const model = document.getElementById("model");
|
|
// From a meta tag rather than interpolated into the script, so the token
|
|
// travels through the same HTML escaping as everything else on the page.
|
|
const csrf = document.querySelector('meta[name="csrf"]').content;
|
|
let messages = [];
|
|
|
|
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
|
|
|
|
// The note follows the picker rather than the reply: someone should know what
|
|
// they are about to get before they spend a minute waiting for it.
|
|
const simNote = document.getElementById("sim-note");
|
|
const onDemandNote = document.getElementById("ondemand-note");
|
|
function showSimNote() {
|
|
const opt = model.selectedOptions[0];
|
|
simNote.hidden = !opt || opt.dataset.simulated !== "1";
|
|
onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1";
|
|
}
|
|
model.addEventListener("change", showSimNote);
|
|
showSimNote();
|
|
|
|
function bubble(role) {
|
|
const el = document.createElement("div");
|
|
el.className = "bubble " + role;
|
|
// textContent throughout: the reply is model output and the prompt is user
|
|
// input, and neither has any business being parsed as markup.
|
|
log.appendChild(el);
|
|
log.scrollTop = log.scrollHeight;
|
|
return el;
|
|
}
|
|
|
|
// Maths is rendered after the text is already in place as text.
|
|
//
|
|
// That order is the whole safety argument. renderMathInElement walks the text
|
|
// nodes of an element whose content was set with textContent, so nothing the
|
|
// model wrote is ever parsed as HTML — only the spans KaTeX recognises become
|
|
// markup, and that markup is KaTeX's, not the model's.
|
|
//
|
|
// trust stays false, which is KaTeX's default and worth stating: it is what
|
|
// keeps \href, \url and \includegraphics out of a reply.
|
|
function renderMath(el) {
|
|
if (typeof renderMathInElement !== "function") return;
|
|
try {
|
|
renderMathInElement(el, {
|
|
delimiters: [
|
|
{ left: "$$", right: "$$", display: true },
|
|
{ left: "\\[", right: "\\]", display: true },
|
|
{ left: "\\(", right: "\\)", display: false },
|
|
// Single $ last: a model writing about prices produces "$5 and $10",
|
|
// and treating that as maths turns a sentence into gibberish. It is
|
|
// still enabled because models overwhelmingly use it for maths, and
|
|
// ignoreOnError leaves anything that does not parse as plain text.
|
|
{ left: "$", right: "$", display: false },
|
|
],
|
|
throwOnError: false,
|
|
errorColor: "var(--bad)",
|
|
trust: false,
|
|
});
|
|
} catch (e) { /* a reply without maths is better than no reply */ }
|
|
}
|
|
|
|
// Fetched rather than linked, because the archive is built from a POST body
|
|
// and there is no URL that names it. The blob is revoked immediately after
|
|
// the click: it is the whole reply, and leaving it alive keeps it addressable
|
|
// for as long as the tab is open.
|
|
async function downloadCode(reply, btn) {
|
|
const was = btn.textContent;
|
|
btn.disabled = true;
|
|
btn.textContent = "Preparing…";
|
|
try {
|
|
const resp = await fetch("/chat/download", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ content: reply, name: "code" }),
|
|
});
|
|
if (!resp.ok) {
|
|
btn.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 120);
|
|
return;
|
|
}
|
|
const url = URL.createObjectURL(await resp.blob());
|
|
const a = document.createElement("a");
|
|
a.href = url;
|
|
a.download = (resp.headers.get("Content-Disposition") || "")
|
|
.match(/filename="?([^"]+)"?/)?.[1] || "code.zip";
|
|
a.click();
|
|
URL.revokeObjectURL(url);
|
|
btn.textContent = was;
|
|
} catch (err) {
|
|
btn.textContent = "download failed";
|
|
} finally {
|
|
btn.disabled = false;
|
|
}
|
|
}
|
|
|
|
const ctxValue = () => (document.getElementById("context") || {}).value || "";
|
|
const stages = {
|
|
waiting: "Waiting for a node with room",
|
|
downloading: "Downloading weights",
|
|
starting: "Starting the engine",
|
|
ready: "Ready",
|
|
};
|
|
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
|
|
|
|
// Loads a model that is not running and shows how far along it is, in the
|
|
// reply's own bubble. Resolves true once it answers, false if it cannot.
|
|
//
|
|
// Asked for with /chat/wake rather than by sending the message and waiting:
|
|
// that request gives up after 45 seconds, and a large model takes minutes.
|
|
async function load(name, out) {
|
|
const box = document.createElement("div");
|
|
box.className = "loading";
|
|
const label = document.createElement("div");
|
|
label.className = "stage";
|
|
const bar = document.createElement("progress");
|
|
bar.max = 100;
|
|
box.append(label, bar);
|
|
out.replaceChildren(box);
|
|
const started = Date.now();
|
|
let failures = 0;
|
|
|
|
let st;
|
|
try {
|
|
const resp = await fetch("/chat/wake", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ model: name, context: ctxValue() }),
|
|
});
|
|
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
|
|
st = await resp.json();
|
|
for (;;) {
|
|
const secs = Math.round((Date.now() - started) / 1000);
|
|
let text = (stages[st.state] || st.state) + " — " + secs + "s";
|
|
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
|
|
label.textContent = text;
|
|
// Indeterminate (no value) unless there is a real number: a bar that
|
|
// sits at 0% for the engine start says it is stuck when it is not.
|
|
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
|
|
if (st.state === "ready") return true;
|
|
// A replica that failed before the wake is still listed until the
|
|
// control plane places its replacement, so one sighting is not final.
|
|
failures = st.state === "failed" ? failures + 1 : 0;
|
|
if (st.state === "stopped" || failures >= 5) {
|
|
throw new Error(st.message || "the model could not be started");
|
|
}
|
|
await new Promise((r) => setTimeout(r, 1500));
|
|
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
|
|
"&context=" + encodeURIComponent(ctxValue()));
|
|
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
|
|
st = await r.json();
|
|
}
|
|
} catch (err) {
|
|
out.textContent = String(err.message || err);
|
|
out.classList.add("failed");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
form.onsubmit = async (e) => {
|
|
e.preventDefault();
|
|
const text = input.value.trim();
|
|
if (!text) return;
|
|
|
|
input.value = "";
|
|
send.disabled = true;
|
|
bubble("you").textContent = text;
|
|
messages.push({ role: "user", content: text });
|
|
|
|
const out = bubble("assistant");
|
|
let reply = "";
|
|
let finish = "";
|
|
const opt = model.selectedOptions[0];
|
|
try {
|
|
if (opt && opt.dataset.ondemand === "1") {
|
|
if (!(await load(model.value, out))) { messages.pop(); return; }
|
|
// Up now: later messages go straight through.
|
|
opt.dataset.ondemand = "0";
|
|
opt.textContent = opt.value;
|
|
showSimNote();
|
|
out.textContent = "";
|
|
}
|
|
const post = () => fetch("/chat/send", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
|
|
});
|
|
let resp = await post();
|
|
// Stopped since the page was drawn, or scaled away by idleness: load it
|
|
// again and retry once, rather than telling the person to.
|
|
if (resp.status === 503 && resp.headers.get("Retry-After")) {
|
|
if (!(await load(model.value, out))) { messages.pop(); return; }
|
|
out.textContent = "";
|
|
resp = await post();
|
|
}
|
|
if (!resp.ok) {
|
|
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
|
|
out.classList.add("failed");
|
|
send.disabled = false;
|
|
return;
|
|
}
|
|
|
|
const reader = resp.body.getReader();
|
|
const dec = new TextDecoder();
|
|
let buf = "";
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
buf += dec.decode(value, { stream: true });
|
|
// Server-sent events arrive in whatever chunks the network felt like;
|
|
// only whole lines can be parsed.
|
|
let nl;
|
|
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
const line = buf.slice(0, nl).trim();
|
|
buf = buf.slice(nl + 1);
|
|
if (!line.startsWith("data:")) continue;
|
|
const payload = line.slice(5).trim();
|
|
if (payload === "[DONE]") continue;
|
|
try {
|
|
const chunk = JSON.parse(payload);
|
|
const piece = chunk.choices?.[0]?.delta?.content;
|
|
if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; }
|
|
// Why the model stopped. Without this a reply that hit the token
|
|
// cap is indistinguishable from one that finished, and the only
|
|
// clue is that it ended mid-word.
|
|
const why = chunk.choices?.[0]?.finish_reason;
|
|
if (why) finish = why;
|
|
} catch (_) { /* a partial frame; the next read completes it */ }
|
|
}
|
|
}
|
|
if (reply) {
|
|
messages.push({ role: "assistant", content: reply });
|
|
// A reply with fenced code gets a download. The button carries the
|
|
// reply text; the server decides what the files are called, because
|
|
// those names come from the model and sanitising them is not a job
|
|
// for string handling in a page.
|
|
if (/^\s*(```|~~~)/m.test(reply)) {
|
|
const dl = document.createElement("button");
|
|
dl.type = "button";
|
|
dl.className = "link download";
|
|
dl.textContent = "Download as .zip";
|
|
dl.onclick = () => downloadCode(reply, dl);
|
|
out.after(dl);
|
|
}
|
|
if (finish === "length") {
|
|
// Appended as a sibling, never into the reply: out holds model text
|
|
// placed with textContent, and this is ours.
|
|
const note = document.createElement("p");
|
|
note.className = "note truncated";
|
|
note.textContent = "Cut off at the length limit — ask it to continue.";
|
|
out.after(note);
|
|
}
|
|
// Once, at the end. Re-rendering every chunk would try to typeset half
|
|
// an expression on each token and flicker its way through the reply.
|
|
renderMath(out);
|
|
}
|
|
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
|
|
} catch (err) {
|
|
out.textContent = "(" + err + ")";
|
|
out.classList.add("failed");
|
|
} finally {
|
|
send.disabled = false;
|
|
input.focus();
|
|
}
|
|
};
|
|
})();
|
|
</script>
|
|
</body></html>{{end}}
|