Let the chat be stopped, timed, tuned and kept across a reload

While a reply streams the Send button is Stop, which aborts the request
- and with it the one to the engine - keeping what had arrived. Under
each reply: its tokens, from the engine's own count (the page's turn now
asks for include_usage), tokens a second, time to the first token and in
all. A settings fold takes a system prompt, sent before every turn, and
a temperature, which the gateway holds to 0-2 whatever the page sends.

The conversation, the prompt and the temperature are kept in the
browser's storage, so a reload puts them back - the model's turns
rendered again by the server as when they came - and "New conversation"
clears them. The gateway still keeps no conversation.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-29 09:54:23 +02:00
1 parent 0823e5d743
commit b7e8e1a3a1
5 files changed
+190 -35

No files matched your search

+12 -2
View File
@@ -72,6 +72,9 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
// Context is what this turn spends against: a project id, or
// "personal". Checked here, not trusted from the page.
Context string `json:"context"`
// Temperature is the page's one sampling setting, held to the range
// every engine takes.
Temperature *float64 `json:"temperature"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 {
http.Error(w, "a model and at least one message are required", http.StatusBadRequest)
@@ -87,12 +90,19 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
// business setting — a stream:false that would hang the reader, a
// max_tokens large enough to occupy a replica for an hour.
_, model := r.resolve(in.Model)
upstream, err := json.Marshal(map[string]any{
req2 := map[string]any{
"model": in.Model,
"messages": in.Messages,
"stream": true,
"max_tokens": chatTokenCap(model),
})
// So the page can say how many tokens the answer took, and how fast
// they came, from the engine's own count.
"stream_options": map[string]any{"include_usage": true},
}
if t := in.Temperature; t != nil {
req2["temperature"] = max(0, min(*t, 2))
}
upstream, err := json.Marshal(req2)
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
+31
View File
@@ -1,6 +1,7 @@
package router
import (
"encoding/json"
"net/http/httptest"
"path/filepath"
"strings"
@@ -176,3 +177,33 @@ func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) {
t.Errorf("a running model is marked as not loaded")
}
}
func TestAChatTurnCarriesItsTemperatureAndAsksForUsage(t *testing.T) {
// The page shows how many tokens an answer took and how fast, from the
// engine's own count, and lets a person turn the temperature — held to
// the range every engine takes, whatever the page sends.
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
rt, user := chatRouter(t, e.srv.URL)
waitForEndpoint(t, rt)
body := `{"model":"qwen2.5-coder-7b-instruct","temperature":5,"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}]}`
rec := httptest.NewRecorder()
rt.chatSend(rec, httptest.NewRequest("POST", "/chat/send", strings.NewReader(body)), user)
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
var sent struct {
Temperature float64 `json:"temperature"`
StreamOptions struct {
IncludeUsage bool `json:"include_usage"`
} `json:"stream_options"`
Messages []map[string]string `json:"messages"`
}
raw, _ := e.last.Load().([]byte)
if err := json.Unmarshal(raw, &sent); err != nil {
t.Fatal(err, string(raw))
}
if sent.Temperature != 2 || !sent.StreamOptions.IncludeUsage || sent.Messages[0]["role"] != "system" {
t.Errorf("sent %s", raw)
}
}
+6 -1
View File
@@ -33,6 +33,8 @@ type fakeEngine struct {
calls atomic.Int64
// status, when non-zero, is returned instead of a completion.
status atomic.Int64
// last is the body of the last completion asked for.
last atomic.Value
}
func newFakeEngine(t *testing.T, model string) *fakeEngine {
@@ -44,8 +46,11 @@ func newFakeEngine(t *testing.T, model string) *fakeEngine {
fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n")
fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n")
})
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) {
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
f.calls.Add(1)
if b, err := io.ReadAll(r.Body); err == nil {
f.last.Store(b)
}
if code := int(f.status.Load()); code != 0 {
http.Error(w, "unavailable", code)
return
+11
View File
@@ -784,3 +784,14 @@ details.add-inline form.add { margin-top: .3rem; }
.explore-search input[type=search] { flex-basis: 100%; }
form.add-to select { max-width: 9rem; }
}
/* Chat: the settings, the stop button, and a reply's timing. */
details.chat-settings { margin: .5rem 0 .25rem; }
details.chat-settings > summary { cursor: pointer; font-size: .85rem; color: var(--muted); }
details.chat-settings label { display: block; margin: .5rem 0; font-size: .85rem; }
details.chat-settings textarea { width: 100%; font: inherit; font-size: .88rem; padding: .45rem .55rem; margin-top: .3rem;
border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); resize: vertical; }
details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: middle; margin: 0 .5rem; accent-color: var(--accent); }
details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; }
#send.stop { background: var(--bad); border-color: var(--bad); }
.chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; }
+130 -32
View File
@@ -34,6 +34,15 @@
<button id="reset" type="button" class="link">New conversation</button>
</div>
<details class="chat-settings" id="settings">
<summary>Settings</summary>
<label>System prompt <span class="opt">how the model should behave, sent before every turn</span>
<textarea id="system" rows="2" placeholder="You are a concise assistant. Answer in the language of the question."></textarea></label>
<label class="temp">Temperature <output id="temp-out">0.7</output>
<input type="range" id="temperature" min="0" max="2" step="0.1" value="0.7">
<span class="opt">lower is steadier, higher more varied</span></label>
</details>
<p id="sim-note" class="note" hidden>
This model is simulated: it exercises scheduling and reports timings, and its
replies are placeholder text, not inference.
@@ -51,9 +60,9 @@
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
<button type="submit" id="send">Send</button>
</form>
<p class="note">Nothing here is stored. The conversation lives in this page and
is gone when you close it — what is recorded is the same usage any request
records: tokens, cost and latency.</p>
<p class="note">The conversation is kept in this browser, so a reload does not lose it;
"New conversation" clears it. Nothing of it is stored on the gateway — what is
recorded there is the same usage any request records: tokens, cost and latency.</p>
{{else}}
<p class="empty">No models are available to you right now.</p>
<p class="note">Either nothing is placed yet, or your project's allowlist does
@@ -80,7 +89,26 @@ access to it.</p>
const csrf = document.querySelector('meta[name="csrf"]').content;
let messages = [];
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
// --- settings and the kept conversation --------------------------------
//
// In this browser's storage, and only there: the gateway keeps no
// conversation. Every read and write is guarded — a private window, or a
// browser that refuses storage, simply starts empty each time.
const sys = document.getElementById("system");
const temp = document.getElementById("temperature");
const tempOut = document.getElementById("temp-out");
const KEY = "llmgw.chat.v1";
const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } };
const save = () => {
try {
localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60),
system: sys.value, temperature: temp.value }));
} catch (_) { /* no storage: nothing kept */ }
};
temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); });
sys.addEventListener("change", save);
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); };
// The note follows the picker rather than the reply: someone should know what
// they are about to get before they spend a minute waiting for it.
@@ -269,20 +297,44 @@ access to it.</p>
}
}
// The turn in progress, so the button can stop it.
let busy = null;
// A line under a reply: how long it took and how fast it came.
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
const p = document.createElement("p");
p.className = "note chat-stats";
const parts = [];
if (tokens) parts.push(tokens + " tokens");
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
parts.push(((tEnd - t0) / 1000).toFixed(1) + " s");
if (stopped) parts.push("stopped");
p.textContent = parts.join(" · ");
out.after(p);
}
form.onsubmit = async (e) => {
e.preventDefault();
// A second press while a reply streams is Stop.
if (busy) { busy.abort(); return; }
const text = input.value.trim();
if (!text) return;
input.value = "";
send.disabled = true;
bubble("you").textContent = text;
messages.push({ role: "user", content: text });
const out = bubble("assistant");
let reply = "";
let finish = "";
let tokens = 0, pieces = 0, stopped = false;
const opt = model.selectedOptions[0];
busy = new AbortController();
send.textContent = "Stop";
send.classList.add("stop");
const t0 = performance.now();
let tFirst = 0;
try {
if (opt && opt.dataset.ondemand === "1") {
if (!(await load(model.value, out))) { messages.pop(); return; }
@@ -292,10 +344,13 @@ access to it.</p>
showSimNote();
out.textContent = "";
}
const system = sys.value.trim();
const post = () => fetch("/chat/send", {
method: "POST",
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value,
messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }),
signal: busy.signal,
});
let resp = await post();
// Stopped since the page was drawn, or scaled away by idleness: load it
@@ -308,37 +363,48 @@ access to it.</p>
if (!resp.ok) {
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
out.classList.add("failed");
send.disabled = false;
messages.pop();
return;
}
const reader = resp.body.getReader();
const dec = new TextDecoder();
let buf = "";
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
// Server-sent events arrive in whatever chunks the network felt like;
// only whole lines can be parsed.
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (payload === "[DONE]") continue;
try {
const chunk = JSON.parse(payload);
const piece = chunk.choices?.[0]?.delta?.content;
if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; }
// Why the model stopped. Without this a reply that hit the token
// cap is indistinguishable from one that finished, and the only
// clue is that it ended mid-word.
const why = chunk.choices?.[0]?.finish_reason;
if (why) finish = why;
} catch (_) { /* a partial frame; the next read completes it */ }
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
buf += dec.decode(value, { stream: true });
// Server-sent events arrive in whatever chunks the network felt
// like; only whole lines can be parsed.
let nl;
while ((nl = buf.indexOf("\n")) >= 0) {
const line = buf.slice(0, nl).trim();
buf = buf.slice(nl + 1);
if (!line.startsWith("data:")) continue;
const payload = line.slice(5).trim();
if (payload === "[DONE]") continue;
try {
const chunk = JSON.parse(payload);
const piece = chunk.choices?.[0]?.delta?.content;
if (piece) {
if (!tFirst) tFirst = performance.now();
pieces++;
reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight;
}
// Why the model stopped. Without this a reply that hit the
// token cap is indistinguishable from one that finished, and
// the only clue is that it ended mid-word.
const why = chunk.choices?.[0]?.finish_reason;
if (why) finish = why;
// The engine's own count, on the last chunk.
if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens;
} catch (_) { /* a partial frame; the next read completes it */ }
}
}
} catch (err) {
if (err.name !== "AbortError") throw err;
stopped = true;
}
if (reply) {
messages.push({ role: "assistant", content: reply });
@@ -362,20 +428,52 @@ access to it.</p>
note.textContent = "Cut off at the length limit — ask it to continue.";
out.after(note);
}
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
// Once, at the end. Re-rendering every chunk would try to typeset half
// an expression on each token and flicker its way through the reply.
await renderReply(out, reply);
renderMath(out);
}
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
} catch (err) {
out.textContent = "(" + err + ")";
if (err.name === "AbortError") {
out.textContent = "(stopped before anything came back)";
messages.pop();
} else {
out.textContent = "(" + err + ")";
}
out.classList.add("failed");
} finally {
send.disabled = false;
busy = null;
send.textContent = "Send";
send.classList.remove("stop");
save();
input.focus();
}
};
// A conversation kept from before, put back as it was: the person's turns
// as text, the model's rendered again by the server, as when they came.
(async () => {
const kept = load_();
if (!kept) return;
if (kept.system) sys.value = kept.system;
if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); }
// Open when there is a prompt in it, so it is not a hidden surprise.
document.getElementById("settings").open = !!kept.system;
if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); }
for (const m of kept.messages || []) {
if (m.role === "user") bubble("you").textContent = m.content;
else if (m.role === "assistant") {
const out = bubble("assistant");
out.textContent = m.content;
await renderReply(out, m.content);
renderMath(out);
}
messages.push(m);
}
})();
})();
</script>
</body></html>{{end}}