Files
llmgw/internal/web/templates/chat.html
T

484 lines
21 KiB
HTML

{{define "chat"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Chat · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
<link rel="stylesheet" href="/_static/katex/katex.min.css">
<script defer src="/_static/katex/katex.min.js"></script>
<script defer src="/_static/katex/auto-render.min.js"></script>
<script defer src="/_static/pages.js"></script>
<meta name="csrf" content="{{.CSRF}}">
</head><body><div class="wrap">
<header>
<h1>Chat</h1>
<p class="sub">{{.User.Email}}</p>
{{template "nav" .Nav}}
</header>
{{if gt (len .Contexts) 1}}
{{/* What this conversation spends against. A plain GET, so the model list
below is the one this choice allows; the server checks it again on every
message rather than trusting the page. */}}
<form method="get" action="/chat" class="chat-context">
<label class="inline-label">Using
<select id="context" name="context" data-autosubmit>
{{range .Contexts}}<option value="{{.Value}}"{{if eq .Value $.Context}} selected{{end}}>{{.Label}}</option>{{end}}
</select></label>
<noscript><button type="submit" class="link">switch</button></noscript>
</form>
{{else if .Contexts}}{{with index .Contexts 0}}<input type="hidden" id="context" value="{{.Value}}">{{end}}{{end}}
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{with .Runs}} · {{.}}{{end}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
<details class="chat-settings" id="settings">
<summary>Settings</summary>
<label>System prompt <span class="opt">how the model should behave, sent before every turn</span>
<textarea id="system" rows="2" placeholder="You are a concise assistant. Answer in the language of the question."></textarea></label>
<label class="temp">Temperature <output id="temp-out">0.7</output>
<input type="range" id="temperature" min="0" max="2" step="0.1" value="0.7">
<span class="opt">lower is steadier, higher more varied</span></label>
</details>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
</p>
<p id="ondemand-note" class="note" hidden>
This model is not running. The first message loads it onto a node — fetching
its weights and starting the engine, minutes for a large model — and is sent
once it is up.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
<form id="composer" class="chat-composer">
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
<button type="submit" id="send">Send</button>
</form>
<p class="note">The conversation is kept in this browser, so a reload does not lose it;
"New conversation" clears it. Nothing of it is stored on the gateway — what is
recorded there is the same usage any request records: tokens, cost and latency.</p>
{{else}}
<p class="empty">No models are available to you right now.</p>
<p class="note">Either nothing is placed yet, or your project's allowlist does
not include anything that is. <code>/admin</code> shows the cluster if you have
access to it.</p>
{{end}}
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div>
<script>
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
// paragraph feels broken, and on CPU inference ten seconds is normal.
(function () {
const log = document.getElementById("log");
if (!log) return;
const form = document.getElementById("composer");
const input = document.getElementById("input");
const send = document.getElementById("send");
const model = document.getElementById("model");
// From a meta tag rather than interpolated into the script, so the token
// travels through the same HTML escaping as everything else on the page.
const csrf = document.querySelector('meta[name="csrf"]').content;
let messages = [];
// --- settings and the kept conversation --------------------------------
//
// In this browser's storage, and only there: the gateway keeps no
// conversation. Every read and write is guarded — a private window, or a
// browser that refuses storage, simply starts empty each time.
const sys = document.getElementById("system");
const temp = document.getElementById("temperature");
const tempOut = document.getElementById("temp-out");
const KEY = "llmgw.chat.v1";
const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } };
const save = () => {
try {
localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60),
system: sys.value, temperature: temp.value }));
} catch (_) { /* no storage: nothing kept */ }
};
temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); });
sys.addEventListener("change", save);
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
const simNote = document.getElementById("sim-note");
const onDemandNote = document.getElementById("ondemand-note");
function showSimNote() {
const opt = model.selectedOptions[0];
simNote.hidden = !opt || opt.dataset.simulated !== "1";
onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1";
}
model.addEventListener("change", showSimNote);
showSimNote();
function bubble(role) {
const el = document.createElement("div");
el.className = "bubble " + role;
// textContent throughout: the reply is model output and the prompt is user
// input, and neither has any business being parsed as markup.
log.appendChild(el);
log.scrollTop = log.scrollHeight;
return el;
}
// Maths is rendered after the text is already in place as text.
//
// That order is the whole safety argument. renderMathInElement walks the text
// nodes of an element whose content was set with textContent, so nothing the
// model wrote is ever parsed as HTML — only the spans KaTeX recognises become
// markup, and that markup is KaTeX's, not the model's.
//
// trust stays false, which is KaTeX's default and worth stating: it is what
// keeps \href, \url and \includegraphics out of a reply.
function renderMath(el) {
// A rendered reply says where its maths is: the server set each span
// aside before the markdown and marked it, so exactly those are typeset.
// KaTeX's own search is the fallback for a reply that stayed plain text,
// and it takes "$5 and $10" for a formula, which the marks do not.
const marked = el.querySelectorAll("span.math");
if (marked.length && typeof katex === "object") {
for (const m of marked) {
try {
katex.render(m.textContent, m, {
displayMode: m.dataset.display === "1", throwOnError: false, trust: false,
});
} catch (e) { m.textContent = m.dataset.src || m.textContent; }
}
return;
}
if (el.classList.contains("md")) return;
if (typeof renderMathInElement !== "function") return;
try {
renderMathInElement(el, {
delimiters: [
{ left: "$$", right: "$$", display: true },
{ left: "\\[", right: "\\]", display: true },
{ left: "\\(", right: "\\)", display: false },
// Single $ last: a model writing about prices produces "$5 and $10",
// and treating that as maths turns a sentence into gibberish. It is
// still enabled because models overwhelmingly use it for maths, and
// ignoreOnError leaves anything that does not parse as plain text.
{ left: "$", right: "$", display: false },
],
throwOnError: false,
errorColor: "var(--bad)",
trust: false,
});
} catch (e) { /* a reply without maths is better than no reply */ }
}
// The finished reply as markdown: headings, lists, tables, code blocks.
//
// Rendered by the server, under the rules the mirror's model cards use —
// raw HTML dropped, dangerous links emptied, images as links — so what
// comes back is safe to place. If that fails the reply stays as the plain
// text it streamed as, which is still the whole answer.
async function renderReply(out, reply) {
try {
const resp = await fetch("/chat/render", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ content: reply }),
});
if (!resp.ok) return;
out.innerHTML = await resp.text();
out.classList.add("md");
for (const a of out.querySelectorAll("a[href]")) {
a.target = "_blank";
a.rel = "noopener noreferrer";
}
} catch (_) { /* plain text it is */ }
}
// Fetched rather than linked, because the archive is built from a POST body
// and there is no URL that names it. The blob is revoked immediately after
// the click: it is the whole reply, and leaving it alive keeps it addressable
// for as long as the tab is open.
async function downloadCode(reply, btn) {
const was = btn.textContent;
btn.disabled = true;
btn.textContent = "Preparing…";
try {
const resp = await fetch("/chat/download", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ content: reply, name: "code" }),
});
if (!resp.ok) {
btn.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 120);
return;
}
const url = URL.createObjectURL(await resp.blob());
const a = document.createElement("a");
a.href = url;
a.download = (resp.headers.get("Content-Disposition") || "")
.match(/filename="?([^"]+)"?/)?.[1] || "code.zip";
a.click();
URL.revokeObjectURL(url);
btn.textContent = was;
} catch (err) {
btn.textContent = "download failed";
} finally {
btn.disabled = false;
}
}
const ctxValue = () => (document.getElementById("context") || {}).value || "";
const stages = {
waiting: "Waiting for a node with room",
downloading: "Downloading weights",
starting: "Starting the engine",
ready: "Ready",
};
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
// Loads a model that is not running and shows how far along it is, in the
// reply's own bubble. Resolves true once it answers, false if it cannot.
//
// Asked for with /chat/wake rather than by sending the message and waiting:
// that request gives up after 45 seconds, and a large model takes minutes.
async function load(name, out) {
const box = document.createElement("div");
box.className = "loading";
const label = document.createElement("div");
label.className = "stage";
const bar = document.createElement("progress");
bar.max = 100;
box.append(label, bar);
out.replaceChildren(box);
const started = Date.now();
let failures = 0;
let st;
try {
const resp = await fetch("/chat/wake", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: name, context: ctxValue() }),
});
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
st = await resp.json();
for (;;) {
const secs = Math.round((Date.now() - started) / 1000);
let text = (stages[st.state] || st.state) + " — " + secs + "s";
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
label.textContent = text;
// Indeterminate (no value) unless there is a real number: a bar that
// sits at 0% for the engine start says it is stuck when it is not.
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
if (st.state === "ready") return true;
// A replica that failed before the wake is still listed until the
// control plane places its replacement, so one sighting is not final.
failures = st.state === "failed" ? failures + 1 : 0;
if (st.state === "stopped" || failures >= 5) {
throw new Error(st.message || "the model could not be started");
}
await new Promise((r) => setTimeout(r, 1500));
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
"&context=" + encodeURIComponent(ctxValue()));
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
st = await r.json();
}
} catch (err) {
out.textContent = String(err.message || err);
out.classList.add("failed");
return false;
}
}
// The turn in progress, so the button can stop it.
let busy = null;
// A line under a reply: how long it took and how fast it came.
function stats(out, tokens, t0, tFirst, tEnd, stopped, hw) {
const p = document.createElement("p");
p.className = "note chat-stats";
const parts = [];
// What answered: a model with replicas on a card and on CPUs is served
// by either, and the speed below means nothing without knowing which.
if (hw) parts.push(hw === "cpu" ? "CPU" : "GPU " + hw);
if (tokens) parts.push(tokens + " tokens");
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
parts.push(((tEnd - t0) / 1000).toFixed(1) + " s");
if (stopped) parts.push("stopped");
p.textContent = parts.join(" · ");
out.after(p);
}
form.onsubmit = async (e) => {
e.preventDefault();
// A second press while a reply streams is Stop.
if (busy) { busy.abort(); return; }
const text = input.value.trim();
if (!text) return;
input.value = "";
bubble("you").textContent = text;
messages.push({ role: "user", content: text });
const out = bubble("assistant");
let reply = "";
let finish = "";
let tokens = 0, pieces = 0, stopped = false;
const opt = model.selectedOptions[0];
busy = new AbortController();
send.textContent = "Stop";
send.classList.add("stop");
const t0 = performance.now();
let tFirst = 0;
try {
if (opt && opt.dataset.ondemand === "1") {
if (!(await load(model.value, out))) { messages.pop(); return; }
// Up now: later messages go straight through.
opt.dataset.ondemand = "0";
opt.textContent = opt.value;
showSimNote();
out.textContent = "";
}
const system = sys.value.trim();
const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value,
messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }),
signal: busy.signal,
});
let resp = await post();
// Stopped since the page was drawn, or scaled away by idleness: load it
// again and retry once, rather than telling the person to.
if (resp.status === 503 && resp.headers.get("Retry-After")) {
if (!(await load(model.value, out))) { messages.pop(); return; }
out.textContent = "";
resp = await post();
}
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");
messages.pop();
return;
}
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = "";
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
// Server-sent events arrive in whatever chunks the network felt
// like; only whole lines can be parsed.
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (payload === "[DONE]") continue;
try {
const chunk = JSON.parse(payload);
const piece = chunk.choices?.[0]?.delta?.content;
if (piece) {
if (!tFirst) tFirst = performance.now();
pieces++;
reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight;
}
// Why the model stopped. Without this a reply that hit the
// token cap is indistinguishable from one that finished, and
// the only clue is that it ended mid-word.
const why = chunk.choices?.[0]?.finish_reason;
if (why) finish = why;
// The engine's own count, on the last chunk.
if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens;
} catch (_) { /* a partial frame; the next read completes it */ }
}
}
} catch (err) {
if (err.name !== "AbortError") throw err;
stopped = true;
}
if (reply) {
messages.push({ role: "assistant", content: reply });
// A reply with fenced code gets a download. The button carries the
// reply text; the server decides what the files are called, because
// those names come from the model and sanitising them is not a job
// for string handling in a page.
if (/^\s*(```|~~~)/m.test(reply)) {
const dl = document.createElement("button");
dl.type = "button";
dl.className = "link download";
dl.textContent = "Download as .zip";
dl.onclick = () => downloadCode(reply, dl);
out.after(dl);
}
if (finish === "length") {
// Appended as a sibling, never into the reply: out holds model text
// placed with textContent, and this is ours.
const note = document.createElement("p");
note.className = "note truncated";
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped, resp.headers.get("x-llmgw-hardware"));
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
await renderReply(out, reply);
renderMath(out);
}
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
} catch (err) {
if (err.name === "AbortError") {
out.textContent = "(stopped before anything came back)";
messages.pop();
} else {
out.textContent = "(" + err + ")";
}
out.classList.add("failed");
} finally {
busy = null;
send.textContent = "Send";
send.classList.remove("stop");
save();
input.focus();
}
};
// A conversation kept from before, put back as it was: the person's turns
// as text, the model's rendered again by the server, as when they came.
(async () => {
const kept = load_();
if (!kept) return;
if (kept.system) sys.value = kept.system;
if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); }
// Open when there is a prompt in it, so it is not a hidden surprise.
document.getElementById("settings").open = !!kept.system;
if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); }
for (const m of kept.messages || []) {
if (m.role === "user") bubble("you").textContent = m.content;
else if (m.role === "assistant") {
const out = bubble("assistant");
out.textContent = m.content;
await renderReply(out, m.content);
renderMath(out);
}
messages.push(m);
}
})();
})();
</script>
</body></html>{{end}}