Let a conversation continue on a different model than it started on
The request path half. A caller sets x-llmgw-thread and sends only the new turn; llmgw puts the stored history in front of it, sends that to whichever model is serving, and stores the exchange afterwards. A header rather than a body field because some SDKs drop fields they do not know and all of them let a caller set a header. Capturing the reply is opt-in and bounded. This is the hottest path in the system, so a request without a thread accumulates nothing at all, and one with a thread stops accumulating at a cap rather than growing to it — a gateway that kept every reply in memory would be one long answer away from trouble. A reply is truncated on a rune boundary rather than dropped: a shortened turn replays as a shortened turn, while nothing at all replays as the assistant having said nothing, and invalid UTF-8 would break the next turn's body for good rather than for one turn. The streamed path sets the captured reply on every exit, not only the one that looks like the end. A stream can also finish with the client gone or the engine stalled, and what the assistant said up to that point is still what it said. A thread on a project that keeps no conversations is refused. The e2e test asserted that and got a cheerful 200: the request was served, nothing was stored, and the caller would have discovered it only when the model had no idea what they were referring to — which reads as the model being broken. It now asks whether storage is on before doing anything else. History is spliced after the model is finally settled, because a budget fallback may have changed it and trimming is measured against the context of the model that will actually serve. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cebf9906d2
commit
9eb321ae3e
7 files changed
+595
-3
No files matched your search
@@ -154,6 +154,14 @@ type Request struct {
|
||||
// still recording it. Set when the router asked for usage the client did
|
||||
// not request.
|
||||
SuppressUsageChunk bool
|
||||
// CaptureReply asks for the assistant's text back in the Outcome, up to
|
||||
// this many bytes. Zero means capture nothing, which is every request that
|
||||
// is not continuing a stored conversation.
|
||||
//
|
||||
// Opt-in and bounded on purpose. This is the hottest path in the system,
|
||||
// and a gateway that accumulated every reply in memory would be one long
|
||||
// answer away from trouble; a request that stores nothing must pay nothing.
|
||||
CaptureReply int
|
||||
}
|
||||
|
||||
// Usage holds the token counts reported by the engine.
|
||||
@@ -178,6 +186,14 @@ type Outcome struct {
|
||||
Usage *Usage
|
||||
// Bytes is how many bytes were relayed to the client.
|
||||
Bytes int64
|
||||
// Reply is the assistant's text, when CaptureReply asked for it. Truncated
|
||||
// at the cap rather than dropped: a shortened turn replays as a shortened
|
||||
// turn, while nothing at all would replay as the assistant having said
|
||||
// nothing, which is a different and wrong conversation.
|
||||
Reply string
|
||||
// ReplyTruncated says the cap was reached, so nothing downstream records a
|
||||
// half answer as a whole one.
|
||||
ReplyTruncated bool
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -232,11 +248,11 @@ func (p *Proxy) Do(ctx context.Context, w http.ResponseWriter, r Request) (Outco
|
||||
if r.Stream && isEventStream(resp.Header) {
|
||||
return p.relayStream(ctx, cancel, w, resp, r)
|
||||
}
|
||||
return p.relayBuffered(w, resp)
|
||||
return p.relayBuffered(w, resp, r)
|
||||
}
|
||||
|
||||
// relayBuffered copies a non-streaming response through, capturing usage.
|
||||
func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response) (Outcome, error) {
|
||||
func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response, r Request) (Outcome, error) {
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, p.cfg.MaxResponseBytes))
|
||||
if err != nil {
|
||||
return Outcome{Status: resp.StatusCode, Retryable: true}, err
|
||||
@@ -252,6 +268,9 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response) (Outco
|
||||
Bytes: int64(n),
|
||||
Usage: usageFromJSON(body),
|
||||
}
|
||||
if r.CaptureReply > 0 && resp.StatusCode == http.StatusOK {
|
||||
out.Reply, out.ReplyTruncated = capReply(replyFromJSON(body), r.CaptureReply)
|
||||
}
|
||||
if werr != nil {
|
||||
return out, ErrClientGone
|
||||
}
|
||||
@@ -262,7 +281,7 @@ func (p *Proxy) relayBuffered(w http.ResponseWriter, resp *http.Response) (Outco
|
||||
//
|
||||
// abort must cancel the context the upstream request was built with: the idle
|
||||
// watchdog can only stop a silent engine by aborting that read.
|
||||
func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w http.ResponseWriter, resp *http.Response, r Request) (Outcome, error) {
|
||||
func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w http.ResponseWriter, resp *http.Response, r Request) (result Outcome, rerr error) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
return Outcome{Status: resp.StatusCode, Retryable: false},
|
||||
@@ -275,6 +294,15 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
|
||||
out := Outcome{Wrote: true, Status: resp.StatusCode}
|
||||
start := time.Now()
|
||||
// Nothing is accumulated unless a thread asked for it, so a request that
|
||||
// stores nothing pays nothing on the hottest path in the system.
|
||||
reply := replyBuilder{cap: r.CaptureReply}
|
||||
// On every exit rather than at the one that looks like the end: a stream
|
||||
// can also end with the client gone or the engine stalled, and what the
|
||||
// assistant said up to that point is still what it said.
|
||||
defer func() {
|
||||
result.Reply, result.ReplyTruncated = reply.result()
|
||||
}()
|
||||
|
||||
// One watchdog covers both phases: until the first event it enforces
|
||||
// FirstTokenTimeout, afterwards IdleTimeout.
|
||||
@@ -295,6 +323,9 @@ func (p *Proxy) relayStream(ctx context.Context, abort context.CancelFunc, w htt
|
||||
out.TTFT = time.Since(start)
|
||||
wd.setInterval(p.cfg.IdleTimeout)
|
||||
}
|
||||
if r.CaptureReply > 0 {
|
||||
reply.add(replyDelta(ev.data.String()))
|
||||
}
|
||||
if u := ev.usage(); u != nil {
|
||||
out.Usage = u
|
||||
if r.SuppressUsageChunk {
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// replyFromJSON pulls the assistant's text out of a completed response.
|
||||
//
|
||||
// Only the first choice. A request asking for several is not one continuing a
|
||||
// conversation, and storing one of them as "what the assistant said" would
|
||||
// invent a conversation that never happened.
|
||||
func replyFromJSON(body []byte) string {
|
||||
var b struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
// The completions endpoint, which has no message wrapper.
|
||||
Text string `json:"text"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &b); err != nil || len(b.Choices) == 0 {
|
||||
return ""
|
||||
}
|
||||
if c := b.Choices[0].Message.Content; c != "" {
|
||||
return c
|
||||
}
|
||||
return b.Choices[0].Text
|
||||
}
|
||||
|
||||
// replyDelta pulls the incremental text out of one streamed event.
|
||||
func replyDelta(event string) string {
|
||||
if event == "" || event == "[DONE]" {
|
||||
return ""
|
||||
}
|
||||
var b struct {
|
||||
Choices []struct {
|
||||
Delta struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"delta"`
|
||||
Text string `json:"text"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(event), &b); err != nil || len(b.Choices) == 0 {
|
||||
return ""
|
||||
}
|
||||
if c := b.Choices[0].Delta.Content; c != "" {
|
||||
return c
|
||||
}
|
||||
return b.Choices[0].Text
|
||||
}
|
||||
|
||||
// capReply truncates at the cap and says whether it had to.
|
||||
//
|
||||
// Truncated rather than dropped, because a shortened turn replays as a
|
||||
// shortened turn while nothing at all replays as the assistant having said
|
||||
// nothing — a different conversation, and a confusing one to be handed back.
|
||||
func capReply(s string, cap int) (string, bool) {
|
||||
if cap <= 0 || len(s) <= cap {
|
||||
return s, false
|
||||
}
|
||||
// On a rune boundary, so a stored reply is never invalid UTF-8 — it goes
|
||||
// back into a JSON body, and a split multi-byte character would make that
|
||||
// body unserialisable.
|
||||
cut := cap
|
||||
for cut > 0 && !isRuneStart(s[cut]) {
|
||||
cut--
|
||||
}
|
||||
return s[:cut], true
|
||||
}
|
||||
|
||||
func isRuneStart(b byte) bool { return b&0xC0 != 0x80 }
|
||||
|
||||
// replyBuilder accumulates a streamed reply up to a cap.
|
||||
//
|
||||
// It stops appending once full rather than growing and truncating at the end,
|
||||
// so a long answer cannot cost more memory than the cap allows however much
|
||||
// the engine sends.
|
||||
type replyBuilder struct {
|
||||
cap int
|
||||
sb strings.Builder
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (r *replyBuilder) add(s string) {
|
||||
if r.cap <= 0 || s == "" || r.truncated {
|
||||
return
|
||||
}
|
||||
if room := r.cap - r.sb.Len(); len(s) > room {
|
||||
part, _ := capReply(s, room)
|
||||
r.sb.WriteString(part)
|
||||
r.truncated = true
|
||||
return
|
||||
}
|
||||
r.sb.WriteString(s)
|
||||
}
|
||||
|
||||
func (r *replyBuilder) result() (string, bool) { return r.sb.String(), r.truncated }
|
||||
@@ -0,0 +1,95 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNothingIsCapturedUnlessAThreadAsked(t *testing.T) {
|
||||
// This is the hottest path in the system. A gateway that accumulated every
|
||||
// reply in memory would be one long answer away from trouble, so a request
|
||||
// that stores nothing must pay nothing.
|
||||
var r replyBuilder // cap zero, which is every ordinary request
|
||||
r.add("a long assistant reply that nobody asked to keep")
|
||||
if got, truncated := r.result(); got != "" || truncated {
|
||||
t.Errorf("a request with no thread accumulated %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALongReplyCostsNoMoreThanTheCap(t *testing.T) {
|
||||
// Stopping at the cap rather than growing and truncating afterwards is
|
||||
// what makes the cap a memory bound instead of a formatting rule.
|
||||
r := replyBuilder{cap: 10}
|
||||
for range 1000 {
|
||||
r.add(strings.Repeat("x", 100))
|
||||
}
|
||||
got, truncated := r.result()
|
||||
if len(got) > 10 {
|
||||
t.Errorf("accumulated %d bytes against a cap of 10", len(got))
|
||||
}
|
||||
if !truncated {
|
||||
t.Error("truncation was not reported")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATruncatedReplyIsStillValidUTF8(t *testing.T) {
|
||||
// It goes back into a JSON body on the next turn. A split multi-byte
|
||||
// character would make that body unserialisable, and the thread would
|
||||
// break for good rather than for one turn.
|
||||
r := replyBuilder{cap: 4}
|
||||
r.add("üçüncü") // two-byte runes either side of the cut
|
||||
got, _ := r.result()
|
||||
for i, ch := range got {
|
||||
if ch == '�' {
|
||||
t.Fatalf("invalid UTF-8 at byte %d of %q", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAssistantTextIsFoundInBothResponseShapes(t *testing.T) {
|
||||
chat := `{"choices":[{"message":{"role":"assistant","content":"hello"}}]}`
|
||||
if got := replyFromJSON([]byte(chat)); got != "hello" {
|
||||
t.Errorf("chat completion reply = %q", got)
|
||||
}
|
||||
// The completions endpoint, which has no message wrapper.
|
||||
legacy := `{"choices":[{"text":"hello"}]}`
|
||||
if got := replyFromJSON([]byte(legacy)); got != "hello" {
|
||||
t.Errorf("completion reply = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheFirstChoiceIsKept(t *testing.T) {
|
||||
// A request asking for several is not one continuing a conversation, and
|
||||
// storing one of them as "what the assistant said" would invent a
|
||||
// conversation that never happened.
|
||||
body := `{"choices":[{"message":{"content":"one"}},{"message":{"content":"two"}}]}`
|
||||
if got := replyFromJSON([]byte(body)); got != "one" {
|
||||
t.Errorf("reply = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamedDeltasAssembleIntoTheReply(t *testing.T) {
|
||||
r := replyBuilder{cap: 1024}
|
||||
for _, ev := range []string{
|
||||
`{"choices":[{"delta":{"role":"assistant"}}]}`,
|
||||
`{"choices":[{"delta":{"content":"Hel"}}]}`,
|
||||
`{"choices":[{"delta":{"content":"lo"}}]}`,
|
||||
`[DONE]`,
|
||||
} {
|
||||
r.add(replyDelta(ev))
|
||||
}
|
||||
if got, _ := r.result(); got != "Hello" {
|
||||
t.Errorf("assembled %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMalformedEventContributesNothing(t *testing.T) {
|
||||
// An engine sending something unexpected must not corrupt a stored
|
||||
// conversation; the worst case is a turn that is shorter than it was.
|
||||
if got := replyDelta(`not json at all`); got != "" {
|
||||
t.Errorf("garbage produced %q", got)
|
||||
}
|
||||
if got := replyDelta(``); got != "" {
|
||||
t.Errorf("an empty event produced %q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user