Say when a chat reply was cut off, and stop cutting it so early
CI / test (push) Successful in 1m20s
CI / test (push) Successful in 1m20s
Reported as the model not completing a request and stopping. It did complete — it ran out of room. The chat page caps a turn at max_tokens and then never looked at finish_reason, so a reply that hit the cap ended mid-word and looked exactly like one that was finished. The silence was the real defect. A reader has no way to tell "the model had nothing more to say" from "the page stopped it", and the only clue is that the last sentence ends abruptly. 512 was also too low. It was chosen when every model here was a 0.5B answering in a paragraph; it is about a screenful of code, so a coding model gets cut off inside the first function it writes. 2048 now, which is still a bound — the reason for having one has not changed. The notice is appended beside the reply, never into it: that element holds model output placed with textContent, and building markup in the same node is how the careful handling everywhere else gets undone. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
634f87ba15
commit
0a98d01361
4 files changed
+52
-1
No files matched your search
@@ -83,7 +83,13 @@ func (r *Router) chatSend(w http.ResponseWriter, req *http.Request, user store.U
|
||||
//
|
||||
// A playground should not be the thing that ties up a replica for ten minutes,
|
||||
// and someone who wants a longer answer has an API key for that.
|
||||
const chatMaxTokens = 512
|
||||
//
|
||||
// 512 was chosen when every model here was a 0.5B answering questions in a
|
||||
// paragraph. It is about a screenful of code, so a coding model cut off
|
||||
// mid-function on the first real request — and the page said nothing, because
|
||||
// it never read finish_reason. The cap being too low was a nuisance; the reply
|
||||
// ending silently was the actual defect, and both are fixed.
|
||||
const chatMaxTokens = 2048
|
||||
|
||||
// principalFor builds the identity a signed-in person acts under.
|
||||
//
|
||||
|
||||
@@ -149,3 +149,32 @@ func TestTheFirstOfferedModelAnswers(t *testing.T) {
|
||||
t.Error("a simulated model is preselected; the default reply would be placeholder text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATruncatedReplySaysSo(t *testing.T) {
|
||||
// A reply that hit the token cap looks exactly like one that finished: the
|
||||
// stream ends and the text stops. Reported as the model not completing the
|
||||
// request, which is a fair reading when nothing says otherwise.
|
||||
page := chatPage(t, "m")
|
||||
if !strings.Contains(page, "finish_reason") {
|
||||
t.Error("the page never reads why the model stopped")
|
||||
}
|
||||
if !strings.Contains(page, `finish === "length"`) {
|
||||
t.Error("the page does not distinguish running out of room from finishing")
|
||||
}
|
||||
if !strings.Contains(page, "Cut off at the length limit") {
|
||||
t.Error("nothing tells the reader the answer was cut short")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheTruncationNoticeIsNotPutInsideTheReply(t *testing.T) {
|
||||
// out holds model output placed with textContent. The notice is ours, so
|
||||
// it goes beside that element — putting it inside would mean building
|
||||
// markup in the same node the model writes into.
|
||||
page := chatPage(t, "m")
|
||||
if !strings.Contains(page, "out.after(note)") {
|
||||
t.Error("the notice is not appended as a sibling of the reply")
|
||||
}
|
||||
if strings.Contains(page, "out.innerHTML") {
|
||||
t.Error("the reply element is being written as HTML")
|
||||
}
|
||||
}
|
||||
@@ -106,3 +106,5 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line);
|
||||
.bubble .katex-display { white-space: normal; margin: .6rem 0; overflow-x: auto; }
|
||||
.bubble .katex { white-space: normal; }
|
||||
.bubble.you .katex { color: inherit; }
|
||||
/* A reply that stopped because it ran out of room, not because it was done. */
|
||||
.note.truncated { color: var(--warn, #a15c00); margin: .2rem 0 .8rem; }
|
||||
@@ -129,6 +129,7 @@ access to it.</p>
|
||||
|
||||
const out = bubble("assistant");
|
||||
let reply = "";
|
||||
let finish = "";
|
||||
try {
|
||||
const resp = await fetch("/chat/send", {
|
||||
method: "POST",
|
||||
@@ -162,11 +163,24 @@ access to it.</p>
|
||||
const chunk = JSON.parse(payload);
|
||||
const piece = chunk.choices?.[0]?.delta?.content;
|
||||
if (piece) { reply += piece; out.textContent = reply; log.scrollTop = log.scrollHeight; }
|
||||
// Why the model stopped. Without this a reply that hit the token
|
||||
// cap is indistinguishable from one that finished, and the only
|
||||
// clue is that it ended mid-word.
|
||||
const why = chunk.choices?.[0]?.finish_reason;
|
||||
if (why) finish = why;
|
||||
} catch (_) { /* a partial frame; the next read completes it */ }
|
||||
}
|
||||
}
|
||||
if (reply) {
|
||||
messages.push({ role: "assistant", content: reply });
|
||||
if (finish === "length") {
|
||||
// Appended as a sibling, never into the reply: out holds model text
|
||||
// placed with textContent, and this is ours.
|
||||
const note = document.createElement("p");
|
||||
note.className = "note truncated";
|
||||
note.textContent = "Cut off at the length limit — ask it to continue.";
|
||||
out.after(note);
|
||||
}
|
||||
// Once, at the end. Re-rendering every chunk would try to typeset half
|
||||
// an expression on each token and flicker its way through the reply.
|
||||
renderMath(out);
|
||||
|
||||
Reference in new issue
Block a user