diff --git a/internal/router/chat.go b/internal/router/chat.go index 5a6880e..5cc39b3 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -83,7 +83,13 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U // // A playground should not be the thing that ties up a replica for ten minutes, // and someone who wants a longer answer has an API key for that. -const chatMaxTokens = 512 +// +// 512 was chosen when every model here was a 0.5B answering questions in a +// paragraph. It is about a screenful of code, so a coding model cut off +// mid-function on the first real request — and the page said nothing, because +// it never read finish_reason. The cap being too low was a nuisance; the reply +// ending silently was the actual defect, and both are fixed. +const chatMaxTokens = 2048 // principalFor builds the identity a signed-in person acts under. // diff --git a/internal/web/chat_test.go b/internal/web/chat_test.go index 7616369..c143330 100644 --- a/internal/web/chat_test.go +++ b/internal/web/chat_test.go @@ -149,3 +149,32 @@ func TestTheFirstOfferedModelAnswers(t *testing.T) { t.Error("a simulated model is preselected; the default reply would be placeholder text") } } + +func TestATruncatedReplySaysSo(t *testing.T) { + // A reply that hit the token cap looks exactly like one that finished: the + // stream ends and the text stops. Reported as the model not completing the + // request, which is a fair reading when nothing says otherwise. + page := chatPage(t, "m") + if !strings.Contains(page, "finish_reason") { + t.Error("the page never reads why the model stopped") + } + if !strings.Contains(page, `finish === "length"`) { + t.Error("the page does not distinguish running out of room from finishing") + } + if !strings.Contains(page, "Cut off at the length limit") { + t.Error("nothing tells the reader the answer was cut short") + } +} + +func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) { + // out holds model output placed with textContent. The notice is ours, so + // it goes beside that element — putting it inside would mean building + // markup in the same node the model writes into. + page := chatPage(t, "m") + if !strings.Contains(page, "out.after(note)") { + t.Error("the notice is not appended as a sibling of the reply") + } + if strings.Contains(page, "out.innerHTML") { + t.Error("the reply element is being written as HTML") + } +} diff --git a/internal/web/static/base.css b/internal/web/static/base.css index bd1ce5f..419394b 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -106,3 +106,5 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line); .bubble .katex-display { white-space: normal; margin: .6rem 0; overflow-x: auto; } .bubble .katex { white-space: normal; } .bubble.you .katex { color: inherit; } +/* A reply that stopped because it ran out of room, not because it was done. */ +.note.truncated { color: var(--warn, #a15c00); margin: .2rem 0 .8rem; } diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index 317c831..15b22fd 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -129,6 +129,7 @@ access to it.

const out = bubble("assistant"); let reply = ""; + let finish = ""; try { const resp = await fetch("/chat/send", { method: "POST", @@ -162,11 +163,24 @@ access to it.

const chunk = JSON.parse(payload); const piece = chunk.choices?.[0]?.delta?.content; if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; } + // Why the model stopped. Without this a reply that hit the token + // cap is indistinguishable from one that finished, and the only + // clue is that it ended mid-word. + const why = chunk.choices?.[0]?.finish_reason; + if (why) finish = why; } catch (_) { /* a partial frame; the next read completes it */ } } } if (reply) { messages.push({ role: "assistant", content: reply }); + if (finish === "length") { + // Appended as a sibling, never into the reply: out holds model text + // placed with textContent, and this is ours. + const note = document.createElement("p"); + note.className = "note truncated"; + note.textContent = "Cut off at the length limit — ask it to continue."; + out.after(note); + } // Once, at the end. Re-rendering every chunk would try to typeset half // an expression on each token and flicker its way through the reply. renderMath(out);