Files
llmgw/internal/web/templates/chat.html
T
Ilker ManapandClaude Opus 5.5 1b89d5cdea Say on the chat page whether a GPU or a CPU answered
A model with replicas on a card and on CPUs answered at either speed with no
way to tell which. The picker names where each model runs, each reply names
the hardware that served it, and every answer carries x-llmgw-hardware.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-03 18:57:50 +02:00

483 lines
21 KiB
HTML

{{define "chat"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Chat · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
<link rel="stylesheet" href="/_static/katex/katex.min.css">
<script defer src="/_static/katex/katex.min.js"></script>
<script defer src="/_static/katex/auto-render.min.js"></script>
<meta name="csrf" content="{{.CSRF}}">
</head><body><div class="wrap">
<header>
<h1>Chat</h1>
<p class="sub">{{.User.Email}}</p>
{{template "nav" .Nav}}
</header>
{{if gt (len .Contexts) 1}}
{{/* What this conversation spends against. A plain GET, so the model list
below is the one this choice allows; the server checks it again on every
message rather than trusting the page. */}}
<form method="get" action="/chat" class="chat-context">
<label class="inline-label">Using
<select id="context" name="context" onchange="this.form.submit()">
{{range .Contexts}}<option value="{{.Value}}"{{if eq .Value $.Context}} selected{{end}}>{{.Label}}</option>{{end}}
</select></label>
<noscript><button type="submit" class="link">switch</button></noscript>
</form>
{{else if .Contexts}}{{with index .Contexts 0}}<input type="hidden" id="context" value="{{.Value}}">{{end}}{{end}}
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
<details class="chat-settings" id="settings">
<summary>Settings</summary>
<label>System prompt <span class="opt">how the model should behave, sent before every turn</span>
<textarea id="system" rows="2" placeholder="You are a concise assistant. Answer in the language of the question."></textarea></label>
<label class="temp">Temperature <output id="temp-out">0.7</output>
<input type="range" id="temperature" min="0" max="2" step="0.1" value="0.7">
<span class="opt">lower is steadier, higher more varied</span></label>
</details>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
</p>
<p id="ondemand-note" class="note" hidden>
This model is not running. The first message loads it onto a node — fetching
its weights and starting the engine, minutes for a large model — and is sent
once it is up.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
<form id="composer" class="chat-composer">
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
<button type="submit" id="send">Send</button>
</form>
<p class="note">The conversation is kept in this browser, so a reload does not lose it;
"New conversation" clears it. Nothing of it is stored on the gateway — what is
recorded there is the same usage any request records: tokens, cost and latency.</p>
{{else}}
<p class="empty">No models are available to you right now.</p>
<p class="note">Either nothing is placed yet, or your project's allowlist does
not include anything that is. <code>/admin</code> shows the cluster if you have
access to it.</p>
{{end}}
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div>
<script>
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
// paragraph feels broken, and on CPU inference ten seconds is normal.
(function () {
const log = document.getElementById("log");
if (!log) return;
const form = document.getElementById("composer");
const input = document.getElementById("input");
const send = document.getElementById("send");
const model = document.getElementById("model");
// From a meta tag rather than interpolated into the script, so the token
// travels through the same HTML escaping as everything else on the page.
const csrf = document.querySelector('meta[name="csrf"]').content;
let messages = [];
// --- settings and the kept conversation --------------------------------
//
// In this browser's storage, and only there: the gateway keeps no
// conversation. Every read and write is guarded — a private window, or a
// browser that refuses storage, simply starts empty each time.
const sys = document.getElementById("system");
const temp = document.getElementById("temperature");
const tempOut = document.getElementById("temp-out");
const KEY = "llmgw.chat.v1";
const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } };
const save = () => {
try {
localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60),
system: sys.value, temperature: temp.value }));
} catch (_) { /* no storage: nothing kept */ }
};
temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); });
sys.addEventListener("change", save);
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
const simNote = document.getElementById("sim-note");
const onDemandNote = document.getElementById("ondemand-note");
function showSimNote() {
const opt = model.selectedOptions[0];
simNote.hidden = !opt || opt.dataset.simulated !== "1";
onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1";
}
model.addEventListener("change", showSimNote);
showSimNote();
function bubble(role) {
const el = document.createElement("div");
el.className = "bubble " + role;
// textContent throughout: the reply is model output and the prompt is user
// input, and neither has any business being parsed as markup.
log.appendChild(el);
log.scrollTop = log.scrollHeight;
return el;
}
// Maths is rendered after the text is already in place as text.
//
// That order is the whole safety argument. renderMathInElement walks the text
// nodes of an element whose content was set with textContent, so nothing the
// model wrote is ever parsed as HTML — only the spans KaTeX recognises become
// markup, and that markup is KaTeX's, not the model's.
//
// trust stays false, which is KaTeX's default and worth stating: it is what
// keeps \href, \url and \includegraphics out of a reply.
function renderMath(el) {
// A rendered reply says where its maths is: the server set each span
// aside before the markdown and marked it, so exactly those are typeset.
// KaTeX's own search is the fallback for a reply that stayed plain text,
// and it takes "$5 and $10" for a formula, which the marks do not.
const marked = el.querySelectorAll("span.math");
if (marked.length && typeof katex === "object") {
for (const m of marked) {
try {
katex.render(m.textContent, m, {
displayMode: m.dataset.display === "1", throwOnError: false, trust: false,
});
} catch (e) { m.textContent = m.dataset.src || m.textContent; }
}
return;
}
if (el.classList.contains("md")) return;
if (typeof renderMathInElement !== "function") return;
try {
renderMathInElement(el, {
delimiters: [
{ left: "$$", right: "$$", display: true },
{ left: "\\[", right: "\\]", display: true },
{ left: "\\(", right: "\\)", display: false },
// Single $ last: a model writing about prices produces "$5 and $10",
// and treating that as maths turns a sentence into gibberish. It is
// still enabled because models overwhelmingly use it for maths, and
// ignoreOnError leaves anything that does not parse as plain text.
{ left: "$", right: "$", display: false },
],
throwOnError: false,
errorColor: "var(--bad)",
trust: false,
});
} catch (e) { /* a reply without maths is better than no reply */ }
}
// The finished reply as markdown: headings, lists, tables, code blocks.
//
// Rendered by the server, under the rules the mirror's model cards use —
// raw HTML dropped, dangerous links emptied, images as links — so what
// comes back is safe to place. If that fails the reply stays as the plain
// text it streamed as, which is still the whole answer.
async function renderReply(out, reply) {
try {
const resp = await fetch("/chat/render", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ content: reply }),
});
if (!resp.ok) return;
out.innerHTML = await resp.text();
out.classList.add("md");
for (const a of out.querySelectorAll("a[href]")) {
a.target = "_blank";
a.rel = "noopener noreferrer";
}
} catch (_) { /* plain text it is */ }
}
// Fetched rather than linked, because the archive is built from a POST body
// and there is no URL that names it. The blob is revoked immediately after
// the click: it is the whole reply, and leaving it alive keeps it addressable
// for as long as the tab is open.
async function downloadCode(reply, btn) {
const was = btn.textContent;
btn.disabled = true;
btn.textContent = "Preparing…";
try {
const resp = await fetch("/chat/download", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ content: reply, name: "code" }),
});
if (!resp.ok) {
btn.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 120);
return;
}
const url = URL.createObjectURL(await resp.blob());
const a = document.createElement("a");
a.href = url;
a.download = (resp.headers.get("Content-Disposition") || "")
.match(/filename="?([^"]+)"?/)?.[1] || "code.zip";
a.click();
URL.revokeObjectURL(url);
btn.textContent = was;
} catch (err) {
btn.textContent = "download failed";
} finally {
btn.disabled = false;
}
}
const ctxValue = () => (document.getElementById("context") || {}).value || "";
const stages = {
waiting: "Waiting for a node with room",
downloading: "Downloading weights",
starting: "Starting the engine",
ready: "Ready",
};
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
// Loads a model that is not running and shows how far along it is, in the
// reply's own bubble. Resolves true once it answers, false if it cannot.
//
// Asked for with /chat/wake rather than by sending the message and waiting:
// that request gives up after 45 seconds, and a large model takes minutes.
async function load(name, out) {
const box = document.createElement("div");
box.className = "loading";
const label = document.createElement("div");
label.className = "stage";
const bar = document.createElement("progress");
bar.max = 100;
box.append(label, bar);
out.replaceChildren(box);
const started = Date.now();
let failures = 0;
let st;
try {
const resp = await fetch("/chat/wake", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: name, context: ctxValue() }),
});
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
st = await resp.json();
for (;;) {
const secs = Math.round((Date.now() - started) / 1000);
let text = (stages[st.state] || st.state) + " — " + secs + "s";
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
label.textContent = text;
// Indeterminate (no value) unless there is a real number: a bar that
// sits at 0% for the engine start says it is stuck when it is not.
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
if (st.state === "ready") return true;
// A replica that failed before the wake is still listed until the
// control plane places its replacement, so one sighting is not final.
failures = st.state === "failed" ? failures + 1 : 0;
if (st.state === "stopped" || failures >= 5) {
throw new Error(st.message || "the model could not be started");
}
await new Promise((r) => setTimeout(r, 1500));
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
"&context=" + encodeURIComponent(ctxValue()));
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
st = await r.json();
}
} catch (err) {
out.textContent = String(err.message || err);
out.classList.add("failed");
return false;
}
}
// The turn in progress, so the button can stop it.
let busy = null;
// A line under a reply: how long it took and how fast it came.
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
const p = document.createElement("p");
p.className = "note chat-stats";
const parts = [];
// What answered: a model with replicas on a card and on CPUs is served
// by either, and the speed below means nothing without knowing which.
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
if (tokens) parts.push(tokens + " tokens");
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
parts.push(((tEnd - t0) / 1000).toFixed(1) + " s");
if (stopped) parts.push("stopped");
p.textContent = parts.join(" · ");
out.after(p);
}
form.onsubmit = async (e) => {
e.preventDefault();
// A second press while a reply streams is Stop.
if (busy) { busy.abort(); return; }
const text = input.value.trim();
if (!text) return;
input.value = "";
bubble("you").textContent = text;
messages.push({ role: "user", content: text });
const out = bubble("assistant");
let reply = "";
let finish = "";
let tokens = 0, pieces = 0, stopped = false;
const opt = model.selectedOptions[0];
busy = new AbortController();
send.textContent = "Stop";
send.classList.add("stop");
const t0 = performance.now();
let tFirst = 0;
try {
if (opt && opt.dataset.ondemand === "1") {
if (!(await load(model.value, out))) { messages.pop(); return; }
// Up now: later messages go straight through.
opt.dataset.ondemand = "0";
opt.textContent = opt.value;
showSimNote();
out.textContent = "";
}
const system = sys.value.trim();
const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value,
messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }),
signal: busy.signal,
});
let resp = await post();
// Stopped since the page was drawn, or scaled away by idleness: load it
// again and retry once, rather than telling the person to.
if (resp.status === 503 && resp.headers.get("Retry-After")) {
if (!(await load(model.value, out))) { messages.pop(); return; }
out.textContent = "";
resp = await post();
}
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");
messages.pop();
return;
}
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = "";
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
// Server-sent events arrive in whatever chunks the network felt
// like; only whole lines can be parsed.
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (payload === "[DONE]") continue;
try {
const chunk = JSON.parse(payload);
const piece = chunk.choices?.[0]?.delta?.content;
if (piece) {
if (!tFirst) tFirst = performance.now();
pieces++;
reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight;
}
// Why the model stopped. Without this a reply that hit the
// token cap is indistinguishable from one that finished, and
// the only clue is that it ended mid-word.
const why = chunk.choices?.[0]?.finish_reason;
if (why) finish = why;
// The engine's own count, on the last chunk.
if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens;
} catch (_) { /* a partial frame; the next read completes it */ }
}
}
} catch (err) {
if (err.name !== "AbortError") throw err;
stopped = true;
}
if (reply) {
messages.push({ role: "assistant", content: reply });
// A reply with fenced code gets a download. The button carries the
// reply text; the server decides what the files are called, because
// those names come from the model and sanitising them is not a job
// for string handling in a page.
if (/^\s*(```|~~~)/m.test(reply)) {
const dl = document.createElement("button");
dl.type = "button";
dl.className = "link download";
dl.textContent = "Download as .zip";
dl.onclick = () => downloadCode(reply, dl);
out.after(dl);
}
if (finish === "length") {
// Appended as a sibling, never into the reply: out holds model text
// placed with textContent, and this is ours.
const note = document.createElement("p");
note.className = "note truncated";
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
await renderReply(out, reply);
renderMath(out);
}
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
} catch (err) {
if (err.name === "AbortError") {
out.textContent = "(stopped before anything came back)";
messages.pop();
} else {
out.textContent = "(" + err + ")";
}
out.classList.add("failed");
} finally {
busy = null;
send.textContent = "Send";
send.classList.remove("stop");
save();
input.focus();
}
};
// A conversation kept from before, put back as it was: the person's turns
// as text, the model's rendered again by the server, as when they came.
(async () => {
const kept = load_();
if (!kept) return;
if (kept.system) sys.value = kept.system;
if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); }
// Open when there is a prompt in it, so it is not a hidden surprise.
document.getElementById("settings").open = !!kept.system;
if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); }
for (const m of kept.messages || []) {
if (m.role === "user") bubble("you").textContent = m.content;
else if (m.role === "assistant") {
const out = bubble("assistant");
out.textContent = m.content;
await renderReply(out, m.content);
renderMath(out);
}
messages.push(m);
}
})();
})();
</script>
</body></html>{{end}}