diff --git a/internal/router/chat.go b/internal/router/chat.go index 810ddb9..88c641e 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -72,6 +72,9 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U // Context is what this turn spends against: a project id, or // "personal". Checked here, not trusted from the page. Context string `json:"context"` + // Temperature is the page's one sampling setting, held to the range + // every engine takes. + Temperature *float64 `json:"temperature"` } if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 { http.Error(w, "a model and at least one message are required", http.StatusBadRequest) @@ -87,12 +90,19 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U // business setting — a stream:false that would hang the reader, a // max_tokens large enough to occupy a replica for an hour. _, model := r.resolve(in.Model) - upstream, err := json.Marshal(map[string]any{ + req2 := map[string]any{ "model": in.Model, "messages": in.Messages, "stream": true, "max_tokens": chatTokenCap(model), - }) + // So the page can say how many tokens the answer took, and how fast + // they came, from the engine's own count. + "stream_options": map[string]any{"include_usage": true}, + } + if t := in.Temperature; t != nil { + req2["temperature"] = max(0, min(*t, 2)) + } + upstream, err := json.Marshal(req2) if err != nil { http.Error(w, "could not build the request", http.StatusInternalServerError) return diff --git a/internal/router/chat_models_test.go b/internal/router/chat_models_test.go index 1eccaff..833040d 100644 --- a/internal/router/chat_models_test.go +++ b/internal/router/chat_models_test.go @@ -1,6 +1,7 @@ package router import ( + "encoding/json" "net/http/httptest" "path/filepath" "strings" @@ -176,3 +177,33 @@ func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) { t.Errorf("a running model is marked as not loaded") } } + +func TestAChatTurnCarriesItsTemperatureAndAsksForUsage(t *testing.T) { + // The page shows how many tokens an answer took and how fast, from the + // engine's own count, and lets a person turn the temperature — held to + // the range every engine takes, whatever the page sends. + e := newFakeEngine(t, "qwen2.5-coder-7b-instruct") + rt, user := chatRouter(t, e.srv.URL) + waitForEndpoint(t, rt) + + body := `{"model":"qwen2.5-coder-7b-instruct","temperature":5,"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}]}` + rec := httptest.NewRecorder() + rt.chatSend(rec, httptest.NewRequest("POST", "/chat/send", strings.NewReader(body)), user) + if rec.Code != 200 { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var sent struct { + Temperature float64 `json:"temperature"` + StreamOptions struct { + IncludeUsage bool `json:"include_usage"` + } `json:"stream_options"` + Messages []map[string]string `json:"messages"` + } + raw, _ := e.last.Load().([]byte) + if err := json.Unmarshal(raw, &sent); err != nil { + t.Fatal(err, string(raw)) + } + if sent.Temperature != 2 || !sent.StreamOptions.IncludeUsage || sent.Messages[0]["role"] != "system" { + t.Errorf("sent %s", raw) + } +} diff --git a/internal/router/router_test.go b/internal/router/router_test.go index c85c965..b6a9f94 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -33,6 +33,8 @@ type fakeEngine struct { calls atomic.Int64 // status, when non-zero, is returned instead of a completion. status atomic.Int64 + // last is the body of the last completion asked for. + last atomic.Value } func newFakeEngine(t *testing.T, model string) *fakeEngine { @@ -44,8 +46,11 @@ func newFakeEngine(t *testing.T, model string) *fakeEngine { fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n") fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n") }) - mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) { + mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) { f.calls.Add(1) + if b, err := io.ReadAll(r.Body); err == nil { + f.last.Store(b) + } if code := int(f.status.Load()); code != 0 { http.Error(w, "unavailable", code) return diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 5bc4fcd..e14088f 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -784,3 +784,14 @@ details.add-inline form.add { margin-top: .3rem; } .explore-search input[type=search] { flex-basis: 100%; } form.add-to select { max-width: 9rem; } } + +/* Chat: the settings, the stop button, and a reply's timing. */ +details.chat-settings { margin: .5rem 0 .25rem; } +details.chat-settings > summary { cursor: pointer; font-size: .85rem; color: var(--muted); } +details.chat-settings label { display: block; margin: .5rem 0; font-size: .85rem; } +details.chat-settings textarea { width: 100%; font: inherit; font-size: .88rem; padding: .45rem .55rem; margin-top: .3rem; + border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); resize: vertical; } +details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: middle; margin: 0 .5rem; accent-color: var(--accent); } +details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; } +#send.stop { background: var(--bad); border-color: var(--bad); } +.chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; } diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index 344c169..f82b6f5 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -34,6 +34,15 @@ +
+ Settings + + +
+

Nothing here is stored. The conversation lives in this page and -is gone when you close it — what is recorded is the same usage any request -records: tokens, cost and latency.

+

The conversation is kept in this browser, so a reload does not lose it; +"New conversation" clears it. Nothing of it is stored on the gateway — what is +recorded there is the same usage any request records: tokens, cost and latency.

{{else}}

No models are available to you right now.

Either nothing is placed yet, or your project's allowlist does @@ -80,7 +89,26 @@ access to it.

const csrf = document.querySelector('meta[name="csrf"]').content; let messages = []; - document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; }; + // --- settings and the kept conversation -------------------------------- + // + // In this browser's storage, and only there: the gateway keeps no + // conversation. Every read and write is guarded — a private window, or a + // browser that refuses storage, simply starts empty each time. + const sys = document.getElementById("system"); + const temp = document.getElementById("temperature"); + const tempOut = document.getElementById("temp-out"); + const KEY = "llmgw.chat.v1"; + const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } }; + const save = () => { + try { + localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60), + system: sys.value, temperature: temp.value })); + } catch (_) { /* no storage: nothing kept */ } + }; + temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); }); + sys.addEventListener("change", save); + + document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); }; // The note follows the picker rather than the reply: someone should know what // they are about to get before they spend a minute waiting for it. @@ -269,20 +297,44 @@ access to it.

} } + // The turn in progress, so the button can stop it. + let busy = null; + + // A line under a reply: how long it took and how fast it came. + function stats(out, tokens, t0, tFirst, tEnd, stopped) { + const p = document.createElement("p"); + p.className = "note chat-stats"; + const parts = []; + if (tokens) parts.push(tokens + " tokens"); + if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s"); + if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s"); + parts.push(((tEnd - t0) / 1000).toFixed(1) + " s"); + if (stopped) parts.push("stopped"); + p.textContent = parts.join(" · "); + out.after(p); + } + form.onsubmit = async (e) => { e.preventDefault(); + // A second press while a reply streams is Stop. + if (busy) { busy.abort(); return; } const text = input.value.trim(); if (!text) return; input.value = ""; - send.disabled = true; bubble("you").textContent = text; messages.push({ role: "user", content: text }); const out = bubble("assistant"); let reply = ""; let finish = ""; + let tokens = 0, pieces = 0, stopped = false; const opt = model.selectedOptions[0]; + busy = new AbortController(); + send.textContent = "Stop"; + send.classList.add("stop"); + const t0 = performance.now(); + let tFirst = 0; try { if (opt && opt.dataset.ondemand === "1") { if (!(await load(model.value, out))) { messages.pop(); return; } @@ -292,10 +344,13 @@ access to it.

showSimNote(); out.textContent = ""; } + const system = sys.value.trim(); const post = () => fetch("/chat/send", { method: "POST", headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, - body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }), + body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value, + messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }), + signal: busy.signal, }); let resp = await post(); // Stopped since the page was drawn, or scaled away by idleness: load it @@ -308,37 +363,48 @@ access to it.

if (!resp.ok) { out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400); out.classList.add("failed"); - send.disabled = false; + messages.pop(); return; } const reader = resp.body.getReader(); const dec = new TextDecoder(); let buf = ""; - for (;;) { - const { done, value } = await reader.read(); - if (done) break; - buf += dec.decode(value, { stream: true }); - // Server-sent events arrive in whatever chunks the network felt like; - // only whole lines can be parsed. - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - const line = buf.slice(0, nl).trim(); - buf = buf.slice(nl + 1); - if (!line.startsWith("data:")) continue; - const payload = line.slice(5).trim(); - if (payload === "[DONE]") continue; - try { - const chunk = JSON.parse(payload); - const piece = chunk.choices?.[0]?.delta?.content; - if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; } - // Why the model stopped. Without this a reply that hit the token - // cap is indistinguishable from one that finished, and the only - // clue is that it ended mid-word. - const why = chunk.choices?.[0]?.finish_reason; - if (why) finish = why; - } catch (_) { /* a partial frame; the next read completes it */ } + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + buf += dec.decode(value, { stream: true }); + // Server-sent events arrive in whatever chunks the network felt + // like; only whole lines can be parsed. + let nl; + while ((nl = buf.indexOf("\n")) >= 0) { + const line = buf.slice(0, nl).trim(); + buf = buf.slice(nl + 1); + if (!line.startsWith("data:")) continue; + const payload = line.slice(5).trim(); + if (payload === "[DONE]") continue; + try { + const chunk = JSON.parse(payload); + const piece = chunk.choices?.[0]?.delta?.content; + if (piece) { + if (!tFirst) tFirst = performance.now(); + pieces++; + reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; + } + // Why the model stopped. Without this a reply that hit the + // token cap is indistinguishable from one that finished, and + // the only clue is that it ended mid-word. + const why = chunk.choices?.[0]?.finish_reason; + if (why) finish = why; + // The engine's own count, on the last chunk. + if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens; + } catch (_) { /* a partial frame; the next read completes it */ } + } } + } catch (err) { + if (err.name !== "AbortError") throw err; + stopped = true; } if (reply) { messages.push({ role: "assistant", content: reply }); @@ -362,20 +428,52 @@ access to it.

note.textContent = "Cut off at the length limit — ask it to continue."; out.after(note); } + stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped); // Once, at the end. Re-rendering every chunk would try to typeset half // an expression on each token and flicker its way through the reply. await renderReply(out, reply); renderMath(out); } + else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); } else { out.textContent = "(no content came back)"; out.classList.add("failed"); } } catch (err) { - out.textContent = "(" + err + ")"; + if (err.name === "AbortError") { + out.textContent = "(stopped before anything came back)"; + messages.pop(); + } else { + out.textContent = "(" + err + ")"; + } out.classList.add("failed"); } finally { - send.disabled = false; + busy = null; + send.textContent = "Send"; + send.classList.remove("stop"); + save(); input.focus(); } }; + + // A conversation kept from before, put back as it was: the person's turns + // as text, the model's rendered again by the server, as when they came. + (async () => { + const kept = load_(); + if (!kept) return; + if (kept.system) sys.value = kept.system; + if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); } + // Open when there is a prompt in it, so it is not a hidden surprise. + document.getElementById("settings").open = !!kept.system; + if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); } + for (const m of kept.messages || []) { + if (m.role === "user") bubble("you").textContent = m.content; + else if (m.role === "assistant") { + const out = bubble("assistant"); + out.textContent = m.content; + await renderReply(out, m.content); + renderMath(out); + } + messages.push(m); + } + })(); })(); {{end}}