Let a conversation continue on a different model than it started on
The request path half. A caller sets x-llmgw-thread and sends only the new turn; llmgw puts the stored history in front of it, sends that to whichever model is serving, and stores the exchange afterwards. A header rather than a body field because some SDKs drop fields they do not know and all of them let a caller set a header. Capturing the reply is opt-in and bounded. This is the hottest path in the system, so a request without a thread accumulates nothing at all, and one with a thread stops accumulating at a cap rather than growing to it — a gateway that kept every reply in memory would be one long answer away from trouble. A reply is truncated on a rune boundary rather than dropped: a shortened turn replays as a shortened turn, while nothing at all replays as the assistant having said nothing, and invalid UTF-8 would break the next turn's body for good rather than for one turn. The streamed path sets the captured reply on every exit, not only the one that looks like the end. A stream can also finish with the client gone or the engine stalled, and what the assistant said up to that point is still what it said. A thread on a project that keeps no conversations is refused. The e2e test asserted that and got a cheerful 200: the request was served, nothing was stored, and the caller would have discovered it only when the model had no idea what they were referring to — which reads as the model being broken. It now asks whether storage is on before doing anything else. History is spliced after the model is finally settled, because a budget fallback may have changed it and trimming is measured against the context of the model that will actually serve. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cebf9906d2
commit
9eb321ae3e
7 files changed
+595
-3
No files matched your search
@@ -1,7 +1,9 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
@@ -178,3 +180,115 @@ func ownMessages(body []byte) []store.Message {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// maxStoredReply bounds what one turn may add to a conversation.
|
||||
//
|
||||
// A cap rather than no cap because this is memory held on the request path for
|
||||
// the length of a reply. Generous enough that ordinary answers are stored
|
||||
// whole, small enough that a runaway generation cannot cost the gateway more
|
||||
// than this per concurrent request.
|
||||
const maxStoredReply = 128 << 10
|
||||
|
||||
// captureBytes says how much of the reply the proxy should hand back.
|
||||
//
|
||||
// Zero for every request not continuing a conversation, which is what keeps
|
||||
// the cost of this feature off the path of requests that do not use it.
|
||||
func (p *parsedRequest) captureBytes() int {
|
||||
if p.ThreadID == "" {
|
||||
return 0
|
||||
}
|
||||
return maxStoredReply
|
||||
}
|
||||
|
||||
// loadThread splices a stored conversation into the request.
|
||||
//
|
||||
// Returns how many turns were dropped to make it fit, or an error to refuse
|
||||
// the request with. A thread on a project that keeps no conversations is
|
||||
// refused rather than ignored: ignoring it would send the model a conversation
|
||||
// with no history, and it would answer confidently about something it was
|
||||
// never told — which reads as the model being wrong rather than as a setting
|
||||
// being off.
|
||||
func (r *Router) loadThread(ctx context.Context, req *parsedRequest) (int, error) {
|
||||
if req.ThreadID == "" || r.store == nil {
|
||||
return 0, nil
|
||||
}
|
||||
if req.ProjectID == "" {
|
||||
// No identity, so no project to scope the thread to. Storing
|
||||
// conversations for an unauthenticated gateway would put every
|
||||
// caller's history in one pool, readable by anyone who guesses an id.
|
||||
return 0, errThreadsNeedAuth
|
||||
}
|
||||
|
||||
// Asked before anything else, because a project that keeps no
|
||||
// conversations must be told so rather than served. Without this the
|
||||
// request succeeds, nothing is stored, and the caller finds out only when
|
||||
// the model has no idea what they are referring to — which reads as the
|
||||
// model being broken. The e2e test that caught this asserted the refusal
|
||||
// and got a cheerful 200.
|
||||
retention, err := r.store.ThreadRetention(ctx, req.ProjectID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if retention == 0 {
|
||||
return 0, store.ErrThreadsNotEnabled
|
||||
}
|
||||
|
||||
// What the caller sent, before the history is spliced in, so the exchange
|
||||
// can be stored afterwards without re-reading a body that has changed.
|
||||
req.ThreadTurns = ownMessages(req.UpstreamBody)
|
||||
|
||||
history, err := r.store.Thread(ctx, req.ProjectID, req.ThreadID)
|
||||
if err != nil && !errors.Is(err, store.ErrNoThread) {
|
||||
return 0, err
|
||||
}
|
||||
// A thread that does not exist yet is the first turn of a new one, which is
|
||||
// the ordinary way to start.
|
||||
if len(history) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
_, entry := r.resolve(req.Model)
|
||||
budget := 0
|
||||
if entry.ContextLength > 0 {
|
||||
// The same four-bytes-per-token estimate the rest of the router uses,
|
||||
// less the output the caller asked for, so trimming and the fit check
|
||||
// cannot disagree about what fits.
|
||||
budget = (entry.ContextLength - req.MaxTokens) * 4
|
||||
if budget < 0 {
|
||||
budget = 0
|
||||
}
|
||||
}
|
||||
|
||||
body, dropped, err := spliceThread(req.UpstreamBody, history, budget)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.UpstreamBody = body
|
||||
req.PromptTokens = len(body) / 4
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
// saveThread stores the exchange once a reply has been produced.
|
||||
//
|
||||
// Best effort, and deliberately after the client has been served: a
|
||||
// conversation that failed to record is a worse outcome than a reply that
|
||||
// never arrived, but only just, and the caller has their answer either way.
|
||||
// The failure is logged rather than swallowed, because a thread silently not
|
||||
// growing is the kind of fault someone discovers days later.
|
||||
func (r *Router) saveThread(ctx context.Context, req *parsedRequest, reply string, model string) {
|
||||
if req.ThreadID == "" || r.store == nil || reply == "" {
|
||||
return
|
||||
}
|
||||
turns := append([]store.Message(nil), req.ThreadTurns...)
|
||||
turns = append(turns, store.Message{Role: "assistant", Content: reply, Model: model})
|
||||
|
||||
if err := r.store.AppendThread(ctx, req.ProjectID, req.ThreadID, turns); err != nil {
|
||||
r.log.Warn("could not store a conversation turn",
|
||||
"thread", req.ThreadID, "project", req.ProjectID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// errThreadsNeedAuth is refused rather than ignored, for the same reason a
|
||||
// disabled project is.
|
||||
var errThreadsNeedAuth = errors.New(
|
||||
"a conversation needs an API key: without one there is no project to keep it in")
|
||||
Reference in new issue
Block a user