Let the chat be stopped, timed, tuned and kept across a reload
While a reply streams the Send button is Stop, which aborts the request - and with it the one to the engine - keeping what had arrived. Under each reply: its tokens, from the engine's own count (the page's turn now asks for include_usage), tokens a second, time to the first token and in all. A settings fold takes a system prompt, sent before every turn, and a temperature, which the gateway holds to 0-2 whatever the page sends. The conversation, the prompt and the temperature are kept in the browser's storage, so a reload puts them back - the model's turns rendered again by the server as when they came - and "New conversation" clears them. The gateway still keeps no conversation. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0823e5d743
commit
b7e8e1a3a1
5 files changed
+190
-35
No files matched your search
+12
-2
@@ -72,6 +72,9 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
|
||||
// Context is what this turn spends against: a project id, or
|
||||
// "personal". Checked here, not trusted from the page.
|
||||
Context string `json:"context"`
|
||||
// Temperature is the page's one sampling setting, held to the range
|
||||
// every engine takes.
|
||||
Temperature *float64 `json:"temperature"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" || len(in.Messages) == 0 {
|
||||
http.Error(w, "a model and at least one message are required", http.StatusBadRequest)
|
||||
@@ -87,12 +90,19 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
|
||||
// business setting — a stream:false that would hang the reader, a
|
||||
// max_tokens large enough to occupy a replica for an hour.
|
||||
_, model := r.resolve(in.Model)
|
||||
upstream, err := json.Marshal(map[string]any{
|
||||
req2 := map[string]any{
|
||||
"model": in.Model,
|
||||
"messages": in.Messages,
|
||||
"stream": true,
|
||||
"max_tokens": chatTokenCap(model),
|
||||
})
|
||||
// So the page can say how many tokens the answer took, and how fast
|
||||
// they came, from the engine's own count.
|
||||
"stream_options": map[string]any{"include_usage": true},
|
||||
}
|
||||
if t := in.Temperature; t != nil {
|
||||
req2["temperature"] = max(0, min(*t, 2))
|
||||
}
|
||||
upstream, err := json.Marshal(req2)
|
||||
if err != nil {
|
||||
http.Error(w, "could not build the request", http.StatusInternalServerError)
|
||||
return
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -176,3 +177,33 @@ func TestTheChatPickerOffersAModelThatLoadsWhenAsked(t *testing.T) {
|
||||
t.Errorf("a running model is marked as not loaded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAChatTurnCarriesItsTemperatureAndAsksForUsage(t *testing.T) {
|
||||
// The page shows how many tokens an answer took and how fast, from the
|
||||
// engine's own count, and lets a person turn the temperature — held to
|
||||
// the range every engine takes, whatever the page sends.
|
||||
e := newFakeEngine(t, "qwen2.5-coder-7b-instruct")
|
||||
rt, user := chatRouter(t, e.srv.URL)
|
||||
waitForEndpoint(t, rt)
|
||||
|
||||
body := `{"model":"qwen2.5-coder-7b-instruct","temperature":5,"messages":[{"role":"system","content":"be brief"},{"role":"user","content":"hi"}]}`
|
||||
rec := httptest.NewRecorder()
|
||||
rt.chatSend(rec, httptest.NewRequest("POST", "/chat/send", strings.NewReader(body)), user)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var sent struct {
|
||||
Temperature float64 `json:"temperature"`
|
||||
StreamOptions struct {
|
||||
IncludeUsage bool `json:"include_usage"`
|
||||
} `json:"stream_options"`
|
||||
Messages []map[string]string `json:"messages"`
|
||||
}
|
||||
raw, _ := e.last.Load().([]byte)
|
||||
if err := json.Unmarshal(raw, &sent); err != nil {
|
||||
t.Fatal(err, string(raw))
|
||||
}
|
||||
if sent.Temperature != 2 || !sent.StreamOptions.IncludeUsage || sent.Messages[0]["role"] != "system" {
|
||||
t.Errorf("sent %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,8 @@ type fakeEngine struct {
|
||||
calls atomic.Int64
|
||||
// status, when non-zero, is returned instead of a completion.
|
||||
status atomic.Int64
|
||||
// last is the body of the last completion asked for.
|
||||
last atomic.Value
|
||||
}
|
||||
|
||||
func newFakeEngine(t *testing.T, model string) *fakeEngine {
|
||||
@@ -44,8 +46,11 @@ func newFakeEngine(t *testing.T, model string) *fakeEngine {
|
||||
fmt.Fprintf(w, "# TYPE vllm:num_requests_waiting gauge\nvllm:num_requests_waiting 0\n")
|
||||
fmt.Fprintf(w, "# TYPE vllm:kv_cache_usage_perc gauge\nvllm:kv_cache_usage_perc 0.1\n")
|
||||
})
|
||||
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, _ *http.Request) {
|
||||
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
||||
f.calls.Add(1)
|
||||
if b, err := io.ReadAll(r.Body); err == nil {
|
||||
f.last.Store(b)
|
||||
}
|
||||
if code := int(f.status.Load()); code != 0 {
|
||||
http.Error(w, "unavailable", code)
|
||||
return
|
||||
|
||||
@@ -784,3 +784,14 @@ details.add-inline form.add { margin-top: .3rem; }
|
||||
.explore-search input[type=search] { flex-basis: 100%; }
|
||||
form.add-to select { max-width: 9rem; }
|
||||
}
|
||||
|
||||
/* Chat: the settings, the stop button, and a reply's timing. */
|
||||
details.chat-settings { margin: .5rem 0 .25rem; }
|
||||
details.chat-settings > summary { cursor: pointer; font-size: .85rem; color: var(--muted); }
|
||||
details.chat-settings label { display: block; margin: .5rem 0; font-size: .85rem; }
|
||||
details.chat-settings textarea { width: 100%; font: inherit; font-size: .88rem; padding: .45rem .55rem; margin-top: .3rem;
|
||||
border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); resize: vertical; }
|
||||
details.chat-settings .temp input[type=range] { width: 14rem; vertical-align: middle; margin: 0 .5rem; accent-color: var(--accent); }
|
||||
details.chat-settings output { font-variant-numeric: tabular-nums; font-weight: 700; margin-left: .3rem; }
|
||||
#send.stop { background: var(--bad); border-color: var(--bad); }
|
||||
.chat-stats { margin: -.2rem 0 .6rem; font-size: .72rem; font-variant-numeric: tabular-nums; }
|
||||
@@ -34,6 +34,15 @@
|
||||
<button id="reset" type="button" class="link">New conversation</button>
|
||||
</div>
|
||||
|
||||
<details class="chat-settings" id="settings">
|
||||
<summary>Settings</summary>
|
||||
<label>System prompt <span class="opt">how the model should behave, sent before every turn</span>
|
||||
<textarea id="system" rows="2" placeholder="You are a concise assistant. Answer in the language of the question."></textarea></label>
|
||||
<label class="temp">Temperature <output id="temp-out">0.7</output>
|
||||
<input type="range" id="temperature" min="0" max="2" step="0.1" value="0.7">
|
||||
<span class="opt">lower is steadier, higher more varied</span></label>
|
||||
</details>
|
||||
|
||||
<p id="sim-note" class="note" hidden>
|
||||
This model is simulated: it exercises scheduling and reports timings, and its
|
||||
replies are placeholder text, not inference.
|
||||
@@ -51,9 +60,9 @@
|
||||
<textarea id="input" rows="3" placeholder="Ask something…" required></textarea>
|
||||
<button type="submit" id="send">Send</button>
|
||||
</form>
|
||||
<p class="note">Nothing here is stored. The conversation lives in this page and
|
||||
is gone when you close it — what is recorded is the same usage any request
|
||||
records: tokens, cost and latency.</p>
|
||||
<p class="note">The conversation is kept in this browser, so a reload does not lose it;
|
||||
"New conversation" clears it. Nothing of it is stored on the gateway — what is
|
||||
recorded there is the same usage any request records: tokens, cost and latency.</p>
|
||||
{{else}}
|
||||
<p class="empty">No models are available to you right now.</p>
|
||||
<p class="note">Either nothing is placed yet, or your project's allowlist does
|
||||
@@ -80,7 +89,26 @@ access to it.</p>
|
||||
const csrf = document.querySelector('meta[name="csrf"]').content;
|
||||
let messages = [];
|
||||
|
||||
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; };
|
||||
// --- settings and the kept conversation --------------------------------
|
||||
//
|
||||
// In this browser's storage, and only there: the gateway keeps no
|
||||
// conversation. Every read and write is guarded — a private window, or a
|
||||
// browser that refuses storage, simply starts empty each time.
|
||||
const sys = document.getElementById("system");
|
||||
const temp = document.getElementById("temperature");
|
||||
const tempOut = document.getElementById("temp-out");
|
||||
const KEY = "llmgw.chat.v1";
|
||||
const load_ = () => { try { return JSON.parse(localStorage.getItem(KEY) || "null"); } catch (_) { return null; } };
|
||||
const save = () => {
|
||||
try {
|
||||
localStorage.setItem(KEY, JSON.stringify({ model: model.value, messages: messages.slice(-60),
|
||||
system: sys.value, temperature: temp.value }));
|
||||
} catch (_) { /* no storage: nothing kept */ }
|
||||
};
|
||||
temp.addEventListener("input", () => { tempOut.textContent = (+temp.value).toFixed(1); save(); });
|
||||
sys.addEventListener("change", save);
|
||||
|
||||
document.getElementById("reset").onclick = () => { messages = []; log.textContent = ""; save(); };
|
||||
|
||||
// The note follows the picker rather than the reply: someone should know what
|
||||
// they are about to get before they spend a minute waiting for it.
|
||||
@@ -269,20 +297,44 @@ access to it.</p>
|
||||
}
|
||||
}
|
||||
|
||||
// The turn in progress, so the button can stop it.
|
||||
let busy = null;
|
||||
|
||||
// A line under a reply: how long it took and how fast it came.
|
||||
function stats(out, tokens, t0, tFirst, tEnd, stopped) {
|
||||
const p = document.createElement("p");
|
||||
p.className = "note chat-stats";
|
||||
const parts = [];
|
||||
if (tokens) parts.push(tokens + " tokens");
|
||||
if (tokens && tFirst && tEnd > tFirst) parts.push((tokens / ((tEnd - tFirst) / 1000)).toFixed(1) + " tok/s");
|
||||
if (tFirst) parts.push("first token " + ((tFirst - t0) / 1000).toFixed(2) + " s");
|
||||
parts.push(((tEnd - t0) / 1000).toFixed(1) + " s");
|
||||
if (stopped) parts.push("stopped");
|
||||
p.textContent = parts.join(" · ");
|
||||
out.after(p);
|
||||
}
|
||||
|
||||
form.onsubmit = async (e) => {
|
||||
e.preventDefault();
|
||||
// A second press while a reply streams is Stop.
|
||||
if (busy) { busy.abort(); return; }
|
||||
const text = input.value.trim();
|
||||
if (!text) return;
|
||||
|
||||
input.value = "";
|
||||
send.disabled = true;
|
||||
bubble("you").textContent = text;
|
||||
messages.push({ role: "user", content: text });
|
||||
|
||||
const out = bubble("assistant");
|
||||
let reply = "";
|
||||
let finish = "";
|
||||
let tokens = 0, pieces = 0, stopped = false;
|
||||
const opt = model.selectedOptions[0];
|
||||
busy = new AbortController();
|
||||
send.textContent = "Stop";
|
||||
send.classList.add("stop");
|
||||
const t0 = performance.now();
|
||||
let tFirst = 0;
|
||||
try {
|
||||
if (opt && opt.dataset.ondemand === "1") {
|
||||
if (!(await load(model.value, out))) { messages.pop(); return; }
|
||||
@@ -292,10 +344,13 @@ access to it.</p>
|
||||
showSimNote();
|
||||
out.textContent = "";
|
||||
}
|
||||
const system = sys.value.trim();
|
||||
const post = () => fetch("/chat/send", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf },
|
||||
body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }),
|
||||
body: JSON.stringify({ model: model.value, context: ctxValue(), temperature: +temp.value,
|
||||
messages: (system ? [{ role: "system", content: system }] : []).concat(messages) }),
|
||||
signal: busy.signal,
|
||||
});
|
||||
let resp = await post();
|
||||
// Stopped since the page was drawn, or scaled away by idleness: load it
|
||||
@@ -308,37 +363,48 @@ access to it.</p>
|
||||
if (!resp.ok) {
|
||||
out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400);
|
||||
out.classList.add("failed");
|
||||
send.disabled = false;
|
||||
messages.pop();
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = resp.body.getReader();
|
||||
const dec = new TextDecoder();
|
||||
let buf = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += dec.decode(value, { stream: true });
|
||||
// Server-sent events arrive in whatever chunks the network felt like;
|
||||
// only whole lines can be parsed.
|
||||
let nl;
|
||||
while ((nl = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, nl).trim();
|
||||
buf = buf.slice(nl + 1);
|
||||
if (!line.startsWith("data:")) continue;
|
||||
const payload = line.slice(5).trim();
|
||||
if (payload === "[DONE]") continue;
|
||||
try {
|
||||
const chunk = JSON.parse(payload);
|
||||
const piece = chunk.choices?.[0]?.delta?.content;
|
||||
if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; }
|
||||
// Why the model stopped. Without this a reply that hit the token
|
||||
// cap is indistinguishable from one that finished, and the only
|
||||
// clue is that it ended mid-word.
|
||||
const why = chunk.choices?.[0]?.finish_reason;
|
||||
if (why) finish = why;
|
||||
} catch (_) { /* a partial frame; the next read completes it */ }
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += dec.decode(value, { stream: true });
|
||||
// Server-sent events arrive in whatever chunks the network felt
|
||||
// like; only whole lines can be parsed.
|
||||
let nl;
|
||||
while ((nl = buf.indexOf("\n")) >= 0) {
|
||||
const line = buf.slice(0, nl).trim();
|
||||
buf = buf.slice(nl + 1);
|
||||
if (!line.startsWith("data:")) continue;
|
||||
const payload = line.slice(5).trim();
|
||||
if (payload === "[DONE]") continue;
|
||||
try {
|
||||
const chunk = JSON.parse(payload);
|
||||
const piece = chunk.choices?.[0]?.delta?.content;
|
||||
if (piece) {
|
||||
if (!tFirst) tFirst = performance.now();
|
||||
pieces++;
|
||||
reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight;
|
||||
}
|
||||
// Why the model stopped. Without this a reply that hit the
|
||||
// token cap is indistinguishable from one that finished, and
|
||||
// the only clue is that it ended mid-word.
|
||||
const why = chunk.choices?.[0]?.finish_reason;
|
||||
if (why) finish = why;
|
||||
// The engine's own count, on the last chunk.
|
||||
if (chunk.usage && chunk.usage.completion_tokens) tokens = chunk.usage.completion_tokens;
|
||||
} catch (_) { /* a partial frame; the next read completes it */ }
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err.name !== "AbortError") throw err;
|
||||
stopped = true;
|
||||
}
|
||||
if (reply) {
|
||||
messages.push({ role: "assistant", content: reply });
|
||||
@@ -362,20 +428,52 @@ access to it.</p>
|
||||
note.textContent = "Cut off at the length limit — ask it to continue.";
|
||||
out.after(note);
|
||||
}
|
||||
stats(out, tokens || pieces, t0, tFirst, performance.now(), stopped);
|
||||
// Once, at the end. Re-rendering every chunk would try to typeset half
|
||||
// an expression on each token and flicker its way through the reply.
|
||||
await renderReply(out, reply);
|
||||
renderMath(out);
|
||||
}
|
||||
else if (stopped) { out.textContent = "(stopped before anything came back)"; out.classList.add("failed"); messages.pop(); }
|
||||
else { out.textContent = "(no content came back)"; out.classList.add("failed"); }
|
||||
} catch (err) {
|
||||
out.textContent = "(" + err + ")";
|
||||
if (err.name === "AbortError") {
|
||||
out.textContent = "(stopped before anything came back)";
|
||||
messages.pop();
|
||||
} else {
|
||||
out.textContent = "(" + err + ")";
|
||||
}
|
||||
out.classList.add("failed");
|
||||
} finally {
|
||||
send.disabled = false;
|
||||
busy = null;
|
||||
send.textContent = "Send";
|
||||
send.classList.remove("stop");
|
||||
save();
|
||||
input.focus();
|
||||
}
|
||||
};
|
||||
|
||||
// A conversation kept from before, put back as it was: the person's turns
|
||||
// as text, the model's rendered again by the server, as when they came.
|
||||
(async () => {
|
||||
const kept = load_();
|
||||
if (!kept) return;
|
||||
if (kept.system) sys.value = kept.system;
|
||||
if (kept.temperature != null) { temp.value = kept.temperature; tempOut.textContent = (+temp.value).toFixed(1); }
|
||||
// Open when there is a prompt in it, so it is not a hidden surprise.
|
||||
document.getElementById("settings").open = !!kept.system;
|
||||
if (kept.model && [...model.options].some((o) => o.value === kept.model)) { model.value = kept.model; showSimNote(); }
|
||||
for (const m of kept.messages || []) {
|
||||
if (m.role === "user") bubble("you").textContent = m.content;
|
||||
else if (m.role === "assistant") {
|
||||
const out = bubble("assistant");
|
||||
out.textContent = m.content;
|
||||
await renderReply(out, m.content);
|
||||
renderMath(out);
|
||||
}
|
||||
messages.push(m);
|
||||
}
|
||||
})();
|
||||
})();
|
||||
</script>
|
||||
</body></html>{{end}}
|
||||
Reference in new issue
Block a user