diff --git a/internal/router/chat.go b/internal/router/chat.go
index 810ddb9..88c641e 100644
--- a/internal/router/chat.go
+++ b/internal/router/chat.go
@@ -72,6 +72,9 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
// Context is what this turn spends against: a project id, or
// "personal". Checked here, not trusted from the page.
Context string `json:"context"`
+ // Temperature is the page's one sampling setting, held to the range
+ // every engine takes.
+ Temperature *float64 `json:"temperature"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 {
http.Error(w, "a model and at least one message are required", http.StatusBadRequest)
@@ -87,12 +90,19 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
// business setting — a stream:false that would hang the reader, a
// max_tokens large enough to occupy a replica for an hour.
_, model := r.resolve(in.Model)
- upstream, err := json.Marshal(map[string]any{
+ req2 := map[string]any{
"model": in.Model,
"messages": in.Messages,
"stream": true,
"max_tokens": chatTokenCap(model),
- })
+ // So the page can say how many tokens the answer took, and how fast
+ // they came, from the engine's own count.
+ "stream_options": map[string]any{"include_usage": true},
+ }
+ if t := in.Temperature; t != nil {
+ req2["temperature"] = max(0, min(*t, 2))
+ }
+ upstream, err := json.Marshal(req2)
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
diff --git a/internal/router/chat_models_test.go b/internal/router/chat_models_test.go
index 1eccaff..833040d 100644
--- a/internal/router/chat_models_test.go
+++ b/internal/router/chat_models_test.go
@@ -1,6 +1,7 @@
package router
import (
+ "encoding/json"
"net/http/httptest"
"path/filepath"
"strings"
@@ -176,3 +177,33 @@ func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) {
t.Errorf("a running model is marked as not loaded")
}
}
+
+func TestAChatTurnCarriesItsTemperatureAndAsksForUsage(t *testing.T) {
+ // The page shows how many tokens an answer took and how fast, from the
+ // engine's own count, and lets a person turn the temperature — held to
+ // the range every engine takes, whatever the page sends.
+ e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
+ rt, user := chatRouter(t, e.srv.URL)
+ waitForEndpoint(t, rt)
+
+ body := `{"model":"qwen2.5-coder-7b-instruct","temperature":5,"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}]}`
+ rec := httptest.NewRecorder()
+ rt.chatSend(rec, httptest.NewRequest("POST", "/chat/send", strings.NewReader(body)), user)
+ if rec.Code != 200 {
+ t.Fatalf("%d %s", rec.Code, rec.Body.String())
+ }
+ var sent struct {
+ Temperature float64 `json:"temperature"`
+ StreamOptions struct {
+ IncludeUsage bool `json:"include_usage"`
+ } `json:"stream_options"`
+ Messages []map[string]string `json:"messages"`
+ }
+ raw, _ := e.last.Load().([]byte)
+ if err := json.Unmarshal(raw, &sent); err != nil {
+ t.Fatal(err, string(raw))
+ }
+ if sent.Temperature != 2 || !sent.StreamOptions.IncludeUsage || sent.Messages[0]["role"] != "system" {
+ t.Errorf("sent %s", raw)
+ }
+}
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index c85c965..b6a9f94 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -33,6 +33,8 @@ type fakeEngine struct {
calls atomic.Int64
// status, when non-zero, is returned instead of a completion.
status atomic.Int64
+ // last is the body of the last completion asked for.
+ last atomic.Value
}
func newFakeEngine(t *testing.T, model string) *fakeEngine {
@@ -44,8 +46,11 @@ func newFakeEngine(t *testing.T, model string) *fakeEngine {
fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n")
fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n")
})
- mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) {
+ mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
f.calls.Add(1)
+ if b, err := io.ReadAll(r.Body); err == nil {
+ f.last.Store(b)
+ }
if code := int(f.status.Load()); code != 0 {
http.Error(w, "unavailable", code)
return
diff --git a/internal/web/static/base.css b/internal/web/static/base.css
index 5bc4fcd..e14088f 100644
--- a/internal/web/static/base.css
+++ b/internal/web/static/base.css
@@ -784,3 +784,14 @@ details.add-inline form.add { margin-top: .3rem; }
.explore-search input[type=search] { flex-basis: 100%; }
form.add-to select { max-width: 9rem; }
}
+
+/* Chat: the settings, the stop button, and a reply's timing. */
+details.chat-settings { margin: .5rem 0 .25rem; }
+details.chat-settings > summary { cursor: pointer; font-size: .85rem; color: var(--muted); }
+details.chat-settings label { display: block; margin: .5rem 0; font-size: .85rem; }
+details.chat-settings textarea { width: 100%; font: inherit; font-size: .88rem; padding: .45rem .55rem; margin-top: .3rem;
+ border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); resize: vertical; }
+details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: middle; margin: 0 .5rem; accent-color: var(--accent); }
+details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; }
+#send.stop { background: var(--bad); border-color: var(--bad); }
+.chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; }
diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html
index 344c169..f82b6f5 100644
--- a/internal/web/templates/chat.html
+++ b/internal/web/templates/chat.html
@@ -34,6 +34,15 @@
+Settings
+
+
+
This model is simulated: it exercises scheduling and reports timings, and its replies are placeholder text, not inference. @@ -51,9 +60,9 @@ -
Nothing here is stored. The conversation lives in this page and -is gone when you close it — what is recorded is the same usage any request -records: tokens, cost and latency.
+The conversation is kept in this browser, so a reload does not lose it; +"New conversation" clears it. Nothing of it is stored on the gateway — what is +recorded there is the same usage any request records: tokens, cost and latency.
{{else}}No models are available to you right now.
Either nothing is placed yet, or your project's allowlist does @@ -80,7 +89,26 @@ access to it.
const csrf = document.querySelector('meta[name="csrf"]').content; let messages = []; - document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; }; + // --- settings and the kept conversation -------------------------------- + // + // In this browser's storage, and only there: the gateway keeps no + // conversation. Every read and write is guarded — a private window, or a + // browser that refuses storage, simply starts empty each time. + const sys = document.getElementById("system"); + const temp = document.getElementById("temperature"); + const tempOut = document.getElementById("temp-out"); + const KEY = "llmgw.chat.v1"; + const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } }; + const save = () => { + try { + localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60), + system: sys.value, temperature: temp.value })); + } catch (_) { /* no storage: nothing kept */ } + }; + temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); }); + sys.addEventListener("change", save); + + document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); }; // The note follows the picker rather than the reply: someone should know what // they are about to get before they spend a minute waiting for it. @@ -269,20 +297,44 @@ access to it. } } + // The turn in progress, so the button can stop it. + let busy = null; + + // A line under a reply: how long it took and how fast it came. + function stats(out, tokens, t0, tFirst, tEnd, stopped) { + const p = document.createElement("p"); + p.className = "note chat-stats"; + const parts = []; + if (tokens) parts.push(tokens + " tokens"); + if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s"); + if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s"); + parts.push(((tEnd - t0) / 1000).toFixed(1) + " s"); + if (stopped) parts.push("stopped"); + p.textContent = parts.join(" · "); + out.after(p); + } + form.onsubmit = async (e) => { e.preventDefault(); + // A second press while a reply streams is Stop. + if (busy) { busy.abort(); return; } const text = input.value.trim(); if (!text) return; input.value = ""; - send.disabled = true; bubble("you").textContent = text; messages.push({ role: "user", content: text }); const out = bubble("assistant"); let reply = ""; let finish = ""; + let tokens = 0, pieces = 0, stopped = false; const opt = model.selectedOptions[0]; + busy = new AbortController(); + send.textContent = "Stop"; + send.classList.add("stop"); + const t0 = performance.now(); + let tFirst = 0; try { if (opt && opt.dataset.ondemand === "1") { if (!(await load(model.value, out))) { messages.pop(); return; } @@ -292,10 +344,13 @@ access to it. showSimNote(); out.textContent = ""; } + const system = sys.value.trim(); const post = () => fetch("/chat/send", { method: "POST", headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, - body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }), + body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value, + messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }), + signal: busy.signal, }); let resp = await post(); // Stopped since the page was drawn, or scaled away by idleness: load it @@ -308,37 +363,48 @@ access to it. if (!resp.ok) { out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400); out.classList.add("failed"); - send.disabled = false; + messages.pop(); return; } const reader = resp.body.getReader(); const dec = new TextDecoder(); let buf = ""; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - buf += dec.decode(value, { stream: true }); - // Server-sent events arrive in whatever chunks the network felt like; - // only whole lines can be parsed. - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - const line = buf.slice(0, nl).trim(); - buf = buf.slice(nl + 1); - if (!line.startsWith("data:")) continue; - const payload = line.slice(5).trim(); - if (payload === "[DONE]") continue; - try { - const chunk = JSON.parse(payload); - const piece = chunk.choices?.[0]?.delta?.content; - if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; } - // Why the model stopped. Without this a reply that hit the token - // cap is indistinguishable from one that finished, and the only - // clue is that it ended mid-word. - const why = chunk.choices?.[0]?.finish_reason; - if (why) finish = why; - } catch (_) { /* a partial frame; the next read completes it */ } + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buf += dec.decode(value, { stream: true }); + // Server-sent events arrive in whatever chunks the network felt + // like; only whole lines can be parsed. + let nl; + while ((nl = buf.indexOf("\n")) >= 0) { + const line = buf.slice(0, nl).trim(); + buf = buf.slice(nl + 1); + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (payload === "[DONE]") continue; + try { + const chunk = JSON.parse(payload); + const piece = chunk.choices?.[0]?.delta?.content; + if (piece) { + if (!tFirst) tFirst = performance.now(); + pieces++; + reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; + } + // Why the model stopped. Without this a reply that hit the + // token cap is indistinguishable from one that finished, and + // the only clue is that it ended mid-word. + const why = chunk.choices?.[0]?.finish_reason; + if (why) finish = why; + // The engine's own count, on the last chunk. + if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens; + } catch (_) { /* a partial frame; the next read completes it */ } + } } + } catch (err) { + if (err.name !== "AbortError") throw err; + stopped = true; } if (reply) { messages.push({ role: "assistant", content: reply }); @@ -362,20 +428,52 @@ access to it. note.textContent = "Cut off at the length limit — ask it to continue."; out.after(note); } + stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped); // Once, at the end. Re-rendering every chunk would try to typeset half // an expression on each token and flicker its way through the reply. await renderReply(out, reply); renderMath(out); } + else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); } else { out.textContent = "(no content came back)"; out.classList.add("failed"); } } catch (err) { - out.textContent = "(" + err + ")"; + if (err.name === "AbortError") { + out.textContent = "(stopped before anything came back)"; + messages.pop(); + } else { + out.textContent = "(" + err + ")"; + } out.classList.add("failed"); } finally { - send.disabled = false; + busy = null; + send.textContent = "Send"; + send.classList.remove("stop"); + save(); input.focus(); } }; + + // A conversation kept from before, put back as it was: the person's turns + // as text, the model's rendered again by the server, as when they came. + (async () => { + const kept = load_(); + if (!kept) return; + if (kept.system) sys.value = kept.system; + if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); } + // Open when there is a prompt in it, so it is not a hidden surprise. + document.getElementById("settings").open = !!kept.system; + if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); } + for (const m of kept.messages || []) { + if (m.role === "user") bubble("you").textContent = m.content; + else if (m.role === "assistant") { + const out = bubble("assistant"); + out.textContent = m.content; + await renderReply(out, m.content); + renderMath(out); + } + messages.push(m); + } + })(); })();