While a reply streams the Send button is Stop, which aborts the request - and with it the one to the engine - keeping what had arrived. Under each reply: its tokens, from the engine's own count (the page's turn now asks for include_usage), tokens a second, time to the first token and in all. A settings fold takes a system prompt, sent before every turn, and a temperature, which the gateway holds to 0-2 whatever the page sends. The conversation, the prompt and the temperature are kept in the browser's storage, so a reload puts them back - the model's turns rendered again by the server as when they came - and "New conversation" clears them. The gateway still keeps no conversation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
480 lines
20 KiB
HTML
480 lines
20 KiB
HTML
{{define "chat"}}<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Chat · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
|
|
<link rel="stylesheet" href="/_static/katex/katex.min.css">
|
|
<script defer src="/_static/katex/katex.min.js"></script>
|
|
<script defer src="/_static/katex/auto-render.min.js"></script>
|
|
<meta name="csrf" content="{{.CSRF}}">
|
|
</head><body><div class="wrap">
|
|
<header>
|
|
<h1>Chat</h1>
|
|
<p class="sub">{{.User.Email}}</p>
|
|
{{template "nav" .Nav}}
|
|
</header>
|
|
|
|
{{if gt (len .Contexts) 1}}
|
|
{{/* What this conversation spends against. A plain GET, so the model list
|
|
below is the one this choice allows; the server checks it again on every
|
|
message rather than trusting the page. */}}
|
|
<form method="get" action="/chat" class="chat-context">
|
|
<label class="inline-label">Using
|
|
<select id="context" name="context" onchange="this.form.submit()">
|
|
{{range .Contexts}}<option value="{{.Value}}"{{if eq .Value $.Context}} selected{{end}}>{{.Label}}</option>{{end}}
|
|
</select></label>
|
|
<noscript><button type="submit" class="link">switch</button></noscript>
|
|
</form>
|
|
{{else if .Contexts}}{{with index .Contexts 0}}<input type="hidden" id="context" value="{{.Value}}">{{end}}{{end}}
|
|
|
|
{{if .Models}}
|
|
<div class="chat-bar">
|
|
<label class="inline-label">Model
|
|
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}" data-ondemand="{{if .OnDemand}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}{{if .OnDemand}} (not loaded){{end}}</option>{{end}}</select>
|
|
</label>
|
|
<button id="reset" type="button" class="link">New conversation</button>
|
|
</div>
|
|
|
|
<details class="chat-settings" id="settings">
|
|
<summary>Settings</summary>
|
|
<label>System prompt <span class="opt">how the model should behave, sent before every turn</span>
|
|
<textarea id="system" rows="2" placeholder="You are a concise assistant. Answer in the language of the question."></textarea></label>
|
|
<label class="temp">Temperature <output id="temp-out">0.7</output>
|
|
<input type="range" id="temperature" min="0" max="2" step="0.1" value="0.7">
|
|
<span class="opt">lower is steadier, higher more varied</span></label>
|
|
</details>
|
|
|
|
<p id="sim-note" class="note" hidden>
|
|
This model is simulated: it exercises scheduling and reports timings, and its
|
|
replies are placeholder text, not inference.
|
|
</p>
|
|
|
|
<p id="ondemand-note" class="note" hidden>
|
|
This model is not running. The first message loads it onto a node — fetching
|
|
its weights and starting the engine, minutes for a large model — and is sent
|
|
once it is up.
|
|
</p>
|
|
|
|
<div id="log" class="chat-log" aria-live="polite"></div>
|
|
|
|
<form id="composer" class="chat-composer">
|
|
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
|
|
<button type="submit" id="send">Send</button>
|
|
</form>
|
|
<p class="note">The conversation is kept in this browser, so a reload does not lose it;
|
|
"New conversation" clears it. Nothing of it is stored on the gateway — what is
|
|
recorded there is the same usage any request records: tokens, cost and latency.</p>
|
|
{{else}}
|
|
<p class="empty">No models are available to you right now.</p>
|
|
<p class="note">Either nothing is placed yet, or your project's allowlist does
|
|
not include anything that is. <code>/admin</code> shows the cluster if you have
|
|
access to it.</p>
|
|
{{end}}
|
|
|
|
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
|
|
</div>
|
|
|
|
<script>
|
|
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
|
|
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
|
|
// paragraph feels broken, and on CPU inference ten seconds is normal.
|
|
(function () {
|
|
const log = document.getElementById("log");
|
|
if (!log) return;
|
|
const form = document.getElementById("composer");
|
|
const input = document.getElementById("input");
|
|
const send = document.getElementById("send");
|
|
const model = document.getElementById("model");
|
|
// From a meta tag rather than interpolated into the script, so the token
|
|
// travels through the same HTML escaping as everything else on the page.
|
|
const csrf = document.querySelector('meta[name="csrf"]').content;
|
|
let messages = [];
|
|
|
|
// --- settings and the kept conversation --------------------------------
|
|
//
|
|
// In this browser's storage, and only there: the gateway keeps no
|
|
// conversation. Every read and write is guarded — a private window, or a
|
|
// browser that refuses storage, simply starts empty each time.
|
|
const sys = document.getElementById("system");
|
|
const temp = document.getElementById("temperature");
|
|
const tempOut = document.getElementById("temp-out");
|
|
const KEY = "llmgw.chat.v1";
|
|
const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } };
|
|
const save = () => {
|
|
try {
|
|
localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60),
|
|
system: sys.value, temperature: temp.value }));
|
|
} catch (_) { /* no storage: nothing kept */ }
|
|
};
|
|
temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); });
|
|
sys.addEventListener("change", save);
|
|
|
|
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); };
|
|
|
|
// The note follows the picker rather than the reply: someone should know what
|
|
// they are about to get before they spend a minute waiting for it.
|
|
const simNote = document.getElementById("sim-note");
|
|
const onDemandNote = document.getElementById("ondemand-note");
|
|
function showSimNote() {
|
|
const opt = model.selectedOptions[0];
|
|
simNote.hidden = !opt || opt.dataset.simulated !== "1";
|
|
onDemandNote.hidden = !opt || opt.dataset.ondemand !== "1";
|
|
}
|
|
model.addEventListener("change", showSimNote);
|
|
showSimNote();
|
|
|
|
function bubble(role) {
|
|
const el = document.createElement("div");
|
|
el.className = "bubble " + role;
|
|
// textContent throughout: the reply is model output and the prompt is user
|
|
// input, and neither has any business being parsed as markup.
|
|
log.appendChild(el);
|
|
log.scrollTop = log.scrollHeight;
|
|
return el;
|
|
}
|
|
|
|
// Maths is rendered after the text is already in place as text.
|
|
//
|
|
// That order is the whole safety argument. renderMathInElement walks the text
|
|
// nodes of an element whose content was set with textContent, so nothing the
|
|
// model wrote is ever parsed as HTML — only the spans KaTeX recognises become
|
|
// markup, and that markup is KaTeX's, not the model's.
|
|
//
|
|
// trust stays false, which is KaTeX's default and worth stating: it is what
|
|
// keeps \href, \url and \includegraphics out of a reply.
|
|
function renderMath(el) {
|
|
// A rendered reply says where its maths is: the server set each span
|
|
// aside before the markdown and marked it, so exactly those are typeset.
|
|
// KaTeX's own search is the fallback for a reply that stayed plain text,
|
|
// and it takes "$5 and $10" for a formula, which the marks do not.
|
|
const marked = el.querySelectorAll("span.math");
|
|
if (marked.length && typeof katex === "object") {
|
|
for (const m of marked) {
|
|
try {
|
|
katex.render(m.textContent, m, {
|
|
displayMode: m.dataset.display === "1", throwOnError: false, trust: false,
|
|
});
|
|
} catch (e) { m.textContent = m.dataset.src || m.textContent; }
|
|
}
|
|
return;
|
|
}
|
|
if (el.classList.contains("md")) return;
|
|
if (typeof renderMathInElement !== "function") return;
|
|
try {
|
|
renderMathInElement(el, {
|
|
delimiters: [
|
|
{ left: "$$", right: "$$", display: true },
|
|
{ left: "\\[", right: "\\]", display: true },
|
|
{ left: "\\(", right: "\\)", display: false },
|
|
// Single $ last: a model writing about prices produces "$5 and $10",
|
|
// and treating that as maths turns a sentence into gibberish. It is
|
|
// still enabled because models overwhelmingly use it for maths, and
|
|
// ignoreOnError leaves anything that does not parse as plain text.
|
|
{ left: "$", right: "$", display: false },
|
|
],
|
|
throwOnError: false,
|
|
errorColor: "var(--bad)",
|
|
trust: false,
|
|
});
|
|
} catch (e) { /* a reply without maths is better than no reply */ }
|
|
}
|
|
|
|
// The finished reply as markdown: headings, lists, tables, code blocks.
|
|
//
|
|
// Rendered by the server, under the rules the mirror's model cards use —
|
|
// raw HTML dropped, dangerous links emptied, images as links — so what
|
|
// comes back is safe to place. If that fails the reply stays as the plain
|
|
// text it streamed as, which is still the whole answer.
|
|
async function renderReply(out, reply) {
|
|
try {
|
|
const resp = await fetch("/chat/render", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ content: reply }),
|
|
});
|
|
if (!resp.ok) return;
|
|
out.innerHTML = await resp.text();
|
|
out.classList.add("md");
|
|
for (const a of out.querySelectorAll("a[href]")) {
|
|
a.target = "_blank";
|
|
a.rel = "noopener noreferrer";
|
|
}
|
|
} catch (_) { /* plain text it is */ }
|
|
}
|
|
|
|
// Fetched rather than linked, because the archive is built from a POST body
|
|
// and there is no URL that names it. The blob is revoked immediately after
|
|
// the click: it is the whole reply, and leaving it alive keeps it addressable
|
|
// for as long as the tab is open.
|
|
async function downloadCode(reply, btn) {
|
|
const was = btn.textContent;
|
|
btn.disabled = true;
|
|
btn.textContent = "Preparing…";
|
|
try {
|
|
const resp = await fetch("/chat/download", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ content: reply, name: "code" }),
|
|
});
|
|
if (!resp.ok) {
|
|
btn.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 120);
|
|
return;
|
|
}
|
|
const url = URL.createObjectURL(await resp.blob());
|
|
const a = document.createElement("a");
|
|
a.href = url;
|
|
a.download = (resp.headers.get("Content-Disposition") || "")
|
|
.match(/filename="?([^"]+)"?/)?.[1] || "code.zip";
|
|
a.click();
|
|
URL.revokeObjectURL(url);
|
|
btn.textContent = was;
|
|
} catch (err) {
|
|
btn.textContent = "download failed";
|
|
} finally {
|
|
btn.disabled = false;
|
|
}
|
|
}
|
|
|
|
const ctxValue = () => (document.getElementById("context") || {}).value || "";
|
|
const stages = {
|
|
waiting: "Waiting for a node with room",
|
|
downloading: "Downloading weights",
|
|
starting: "Starting the engine",
|
|
ready: "Ready",
|
|
};
|
|
function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; }
|
|
|
|
// Loads a model that is not running and shows how far along it is, in the
|
|
// reply's own bubble. Resolves true once it answers, false if it cannot.
|
|
//
|
|
// Asked for with /chat/wake rather than by sending the message and waiting:
|
|
// that request gives up after 45 seconds, and a large model takes minutes.
|
|
async function load(name, out) {
|
|
const box = document.createElement("div");
|
|
box.className = "loading";
|
|
const label = document.createElement("div");
|
|
label.className = "stage";
|
|
const bar = document.createElement("progress");
|
|
bar.max = 100;
|
|
box.append(label, bar);
|
|
out.replaceChildren(box);
|
|
const started = Date.now();
|
|
let failures = 0;
|
|
|
|
let st;
|
|
try {
|
|
const resp = await fetch("/chat/wake", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ model: name, context: ctxValue() }),
|
|
});
|
|
if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200));
|
|
st = await resp.json();
|
|
for (;;) {
|
|
const secs = Math.round((Date.now() - started) / 1000);
|
|
let text = (stages[st.state] || st.state) + " — " + secs + "s";
|
|
if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total);
|
|
label.textContent = text;
|
|
// Indeterminate (no value) unless there is a real number: a bar that
|
|
// sits at 0% for the engine start says it is stuck when it is not.
|
|
if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value");
|
|
if (st.state === "ready") return true;
|
|
// A replica that failed before the wake is still listed until the
|
|
// control plane places its replacement, so one sighting is not final.
|
|
failures = st.state === "failed" ? failures + 1 : 0;
|
|
if (st.state === "stopped" || failures >= 5) {
|
|
throw new Error(st.message || "the model could not be started");
|
|
}
|
|
await new Promise((r) => setTimeout(r, 1500));
|
|
const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) +
|
|
"&context=" + encodeURIComponent(ctxValue()));
|
|
if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200));
|
|
st = await r.json();
|
|
}
|
|
} catch (err) {
|
|
out.textContent = String(err.message || err);
|
|
out.classList.add("failed");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// The turn in progress, so the button can stop it.
|
|
let busy = null;
|
|
|
|
// A line under a reply: how long it took and how fast it came.
|
|
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
|
|
const p = document.createElement("p");
|
|
p.className = "note chat-stats";
|
|
const parts = [];
|
|
if (tokens) parts.push(tokens + " tokens");
|
|
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
|
|
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
|
|
parts.push(((tEnd - t0) / 1000).toFixed(1) + " s");
|
|
if (stopped) parts.push("stopped");
|
|
p.textContent = parts.join(" · ");
|
|
out.after(p);
|
|
}
|
|
|
|
form.onsubmit = async (e) => {
|
|
e.preventDefault();
|
|
// A second press while a reply streams is Stop.
|
|
if (busy) { busy.abort(); return; }
|
|
const text = input.value.trim();
|
|
if (!text) return;
|
|
|
|
input.value = "";
|
|
bubble("you").textContent = text;
|
|
messages.push({ role: "user", content: text });
|
|
|
|
const out = bubble("assistant");
|
|
let reply = "";
|
|
let finish = "";
|
|
let tokens = 0, pieces = 0, stopped = false;
|
|
const opt = model.selectedOptions[0];
|
|
busy = new AbortController();
|
|
send.textContent = "Stop";
|
|
send.classList.add("stop");
|
|
const t0 = performance.now();
|
|
let tFirst = 0;
|
|
try {
|
|
if (opt && opt.dataset.ondemand === "1") {
|
|
if (!(await load(model.value, out))) { messages.pop(); return; }
|
|
// Up now: later messages go straight through.
|
|
opt.dataset.ondemand = "0";
|
|
opt.textContent = opt.value;
|
|
showSimNote();
|
|
out.textContent = "";
|
|
}
|
|
const system = sys.value.trim();
|
|
const post = () => fetch("/chat/send", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
|
body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value,
|
|
messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }),
|
|
signal: busy.signal,
|
|
});
|
|
let resp = await post();
|
|
// Stopped since the page was drawn, or scaled away by idleness: load it
|
|
// again and retry once, rather than telling the person to.
|
|
if (resp.status === 503 && resp.headers.get("Retry-After")) {
|
|
if (!(await load(model.value, out))) { messages.pop(); return; }
|
|
out.textContent = "";
|
|
resp = await post();
|
|
}
|
|
if (!resp.ok) {
|
|
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
|
|
out.classList.add("failed");
|
|
messages.pop();
|
|
return;
|
|
}
|
|
|
|
const reader = resp.body.getReader();
|
|
const dec = new TextDecoder();
|
|
let buf = "";
|
|
try {
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
buf += dec.decode(value, { stream: true });
|
|
// Server-sent events arrive in whatever chunks the network felt
|
|
// like; only whole lines can be parsed.
|
|
let nl;
|
|
while ((nl = buf.indexOf("\n")) >= 0) {
|
|
const line = buf.slice(0, nl).trim();
|
|
buf = buf.slice(nl + 1);
|
|
if (!line.startsWith("data:")) continue;
|
|
const payload = line.slice(5).trim();
|
|
if (payload === "[DONE]") continue;
|
|
try {
|
|
const chunk = JSON.parse(payload);
|
|
const piece = chunk.choices?.[0]?.delta?.content;
|
|
if (piece) {
|
|
if (!tFirst) tFirst = performance.now();
|
|
pieces++;
|
|
reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight;
|
|
}
|
|
// Why the model stopped. Without this a reply that hit the
|
|
// token cap is indistinguishable from one that finished, and
|
|
// the only clue is that it ended mid-word.
|
|
const why = chunk.choices?.[0]?.finish_reason;
|
|
if (why) finish = why;
|
|
// The engine's own count, on the last chunk.
|
|
if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens;
|
|
} catch (_) { /* a partial frame; the next read completes it */ }
|
|
}
|
|
}
|
|
} catch (err) {
|
|
if (err.name !== "AbortError") throw err;
|
|
stopped = true;
|
|
}
|
|
if (reply) {
|
|
messages.push({ role: "assistant", content: reply });
|
|
// A reply with fenced code gets a download. The button carries the
|
|
// reply text; the server decides what the files are called, because
|
|
// those names come from the model and sanitising them is not a job
|
|
// for string handling in a page.
|
|
if (/^\s*(```|~~~)/m.test(reply)) {
|
|
const dl = document.createElement("button");
|
|
dl.type = "button";
|
|
dl.className = "link download";
|
|
dl.textContent = "Download as .zip";
|
|
dl.onclick = () => downloadCode(reply, dl);
|
|
out.after(dl);
|
|
}
|
|
if (finish === "length") {
|
|
// Appended as a sibling, never into the reply: out holds model text
|
|
// placed with textContent, and this is ours.
|
|
const note = document.createElement("p");
|
|
note.className = "note truncated";
|
|
note.textContent = "Cut off at the length limit — ask it to continue.";
|
|
out.after(note);
|
|
}
|
|
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
|
|
// Once, at the end. Re-rendering every chunk would try to typeset half
|
|
// an expression on each token and flicker its way through the reply.
|
|
await renderReply(out, reply);
|
|
renderMath(out);
|
|
}
|
|
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
|
|
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
|
|
} catch (err) {
|
|
if (err.name === "AbortError") {
|
|
out.textContent = "(stopped before anything came back)";
|
|
messages.pop();
|
|
} else {
|
|
out.textContent = "(" + err + ")";
|
|
}
|
|
out.classList.add("failed");
|
|
} finally {
|
|
busy = null;
|
|
send.textContent = "Send";
|
|
send.classList.remove("stop");
|
|
save();
|
|
input.focus();
|
|
}
|
|
};
|
|
|
|
// A conversation kept from before, put back as it was: the person's turns
|
|
// as text, the model's rendered again by the server, as when they came.
|
|
(async () => {
|
|
const kept = load_();
|
|
if (!kept) return;
|
|
if (kept.system) sys.value = kept.system;
|
|
if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); }
|
|
// Open when there is a prompt in it, so it is not a hidden surprise.
|
|
document.getElementById("settings").open = !!kept.system;
|
|
if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); }
|
|
for (const m of kept.messages || []) {
|
|
if (m.role === "user") bubble("you").textContent = m.content;
|
|
else if (m.role === "assistant") {
|
|
const out = bubble("assistant");
|
|
out.textContent = m.content;
|
|
await renderReply(out, m.content);
|
|
renderMath(out);
|
|
}
|
|
messages.push(m);
|
|
}
|
|
})();
|
|
})();
|
|
</script>
|
|
</body></html>{{end}}
|