Files
llmgw/internal/web/templates/chat.html
T
Ilker ManapandClaude Opus 5 0a98d01361
CI / test (push) Successful in 1m20s
Say when a chat reply was cut off, and stop cutting it so early
Reported as the model not completing a request and stopping. It did
complete — it ran out of room. The chat page caps a turn at max_tokens and
then never looked at finish_reason, so a reply that hit the cap ended
mid-word and looked exactly like one that was finished.

The silence was the real defect. A reader has no way to tell "the model had
nothing more to say" from "the page stopped it", and the only clue is that
the last sentence ends abruptly.

512 was also too low. It was chosen when every model here was a 0.5B
answering in a paragraph; it is about a screenful of code, so a coding
model gets cut off inside the first function it writes. 2048 now, which is
still a bound — the reason for having one has not changed.

The notice is appended beside the reply, never into it: that element holds
model output placed with textContent, and building markup in the same node
is how the careful handling everywhere else gets undone.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 21:03:07 +02:00

200 lines
7.9 KiB
HTML

{{define "chat"}}<!doctype html>
<html lang="en"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Chat · llmgw</title><link rel="stylesheet" href="/_static/base.css">
<link rel="stylesheet" href="/_static/katex/katex.min.css">
<script defer src="/_static/katex/katex.min.js"></script>
<script defer src="/_static/katex/auto-render.min.js"></script>
<meta name="csrf" content="{{.CSRF}}">
</head><body><div class="wrap">
<header>
<h1>Chat</h1>
<p class="sub">{{.User.Email}} · project {{.User.ProjectID}}</p>
<nav>
<a href="/profile">your keys</a>
{{if .User.IsAdmin}}<a href="/admin">cluster</a>{{end}}
<a href="/">about this gateway</a>
</nav>
</header>
{{if .Models}}
<div class="chat-bar">
<label class="inline-label">Model
<select id="model">{{range .Models}}<option value="{{.Name}}" data-simulated="{{if .Simulated}}1{{else}}0{{end}}">{{.Name}}{{if .Simulated}} (simulated){{end}}</option>{{end}}</select>
</label>
<button id="reset" type="button" class="link">New conversation</button>
</div>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
</p>
<div id="log" class="chat-log" aria-live="polite"></div>
<form id="composer" class="chat-composer">
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
<button type="submit" id="send">Send</button>
</form>
<p class="note">Nothing here is stored. The conversation lives in this page and
is gone when you close it — what is recorded is the same usage any request
records: tokens, cost and latency.</p>
{{else}}
<p class="empty">No models are available to you right now.</p>
<p class="note">Either nothing is placed yet, or your project's allowlist does
not include anything that is. <code>/admin</code> shows the cluster if you have
access to it.</p>
{{end}}
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
</div>
<script>
// Vanilla, inline, no build step — the same reason the charts are hand-drawn
// SVG. It streams because a chat that hangs for ten seconds and then dumps a
// paragraph feels broken, and on CPU inference ten seconds is normal.
(function () {
const log = document.getElementById("log");
if (!log) return;
const form = document.getElementById("composer");
const input = document.getElementById("input");
const send = document.getElementById("send");
const model = document.getElementById("model");
// From a meta tag rather than interpolated into the script, so the token
// travels through the same HTML escaping as everything else on the page.
const csrf = document.querySelector('meta[name="csrf"]').content;
let messages = [];
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
const simNote = document.getElementById("sim-note");
function showSimNote() {
const opt = model.selectedOptions[0];
simNote.hidden = !opt || opt.dataset.simulated !== "1";
}
model.addEventListener("change", showSimNote);
showSimNote();
function bubble(role) {
const el = document.createElement("div");
el.className = "bubble " + role;
// textContent throughout: the reply is model output and the prompt is user
// input, and neither has any business being parsed as markup.
log.appendChild(el);
log.scrollTop = log.scrollHeight;
return el;
}
// Maths is rendered after the text is already in place as text.
//
// That order is the whole safety argument. renderMathInElement walks the text
// nodes of an element whose content was set with textContent, so nothing the
// model wrote is ever parsed as HTML — only the spans KaTeX recognises become
// markup, and that markup is KaTeX's, not the model's.
//
// trust stays false, which is KaTeX's default and worth stating: it is what
// keeps \href, \url and \includegraphics out of a reply.
function renderMath(el) {
if (typeof renderMathInElement !== "function") return;
try {
renderMathInElement(el, {
delimiters: [
{ left: "$$", right: "$$", display: true },
{ left: "\\[", right: "\\]", display: true },
{ left: "\\(", right: "\\)", display: false },
// Single $ last: a model writing about prices produces "$5 and $10",
// and treating that as maths turns a sentence into gibberish. It is
// still enabled because models overwhelmingly use it for maths, and
// ignoreOnError leaves anything that does not parse as plain text.
{ left: "$", right: "$", display: false },
],
throwOnError: false,
errorColor: "var(--bad)",
trust: false,
});
} catch (e) { /* a reply without maths is better than no reply */ }
}
form.onsubmit = async (e) => {
e.preventDefault();
const text = input.value.trim();
if (!text) return;
input.value = "";
send.disabled = true;
bubble("you").textContent = text;
messages.push({ role: "user", content: text });
const out = bubble("assistant");
let reply = "";
let finish = "";
try {
const resp = await fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, messages: messages }),
});
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");
send.disabled = false;
return;
}
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = "";
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
// Server-sent events arrive in whatever chunks the network felt like;
// only whole lines can be parsed.
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (payload === "[DONE]") continue;
try {
const chunk = JSON.parse(payload);
const piece = chunk.choices?.[0]?.delta?.content;
if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; }
// Why the model stopped. Without this a reply that hit the token
// cap is indistinguishable from one that finished, and the only
// clue is that it ended mid-word.
const why = chunk.choices?.[0]?.finish_reason;
if (why) finish = why;
} catch (_) { /* a partial frame; the next read completes it */ }
}
}
if (reply) {
messages.push({ role: "assistant", content: reply });
if (finish === "length") {
// Appended as a sibling, never into the reply: out holds model text
// placed with textContent, and this is ours.
const note = document.createElement("p");
note.className = "note truncated";
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
renderMath(out);
}
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
} catch (err) {
out.textContent = "(" + err + ")";
out.classList.add("failed");
} finally {
send.disabled = false;
input.focus();
}
};
})();
</script>
</body></html>{{end}}