Files
llmgw/internal/router/threads.go
T
Ilker ManapandClaude Opus 5 cebf9906d2 Splice a stored conversation into a request, trimming only what must go
The part that makes a thread work across models: the history goes in front of
what the caller just said, because a client using a thread sends the new turn
rather than the conversation again.

Trimming has three rules, and each is there because the alternative fails
quietly. The oldest stored turns go first, so a conversation that outgrew the
model now serving it stays usable — after a budget fallback that model was not
the caller's choice, and a permanently unusable thread would be a strange way
to punish them for running out of budget. The caller's own messages are never
dropped: answering something they did not ask is worse than refusing. And a
system message is kept wherever it sits, because losing one silently changes
what the model is for while it carries on answering.

A model that declares no context length has nothing trimmed. Guessing a limit
for it would drop turns for no reason and the caller would never learn why the
model forgot.

The body is rebuilt through a map so every field the client sent survives.
Forwarding fields llmgw does not understand is how engine-specific options
reach an engine at all, and a thread must not be what starts dropping them.
An empty history returns the body byte for byte, so a request without a thread
takes exactly the path it took before this existed.

Not yet wired to the request path.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-18 11:10:07 +02:00

181 lines
5.2 KiB
Go

package router
import (
"encoding/json"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// threadHeader names the conversation a request continues.
//
// A header rather than a body field so the JSON stays exactly what an OpenAI
// client would send: some SDKs drop fields they do not know, and every one of
// them lets a caller set a header. It sits with the other x-llmgw-* headers
// the router already reads.
const threadHeader = "x-llmgw-thread"
// trimmedHeader reports how many stored turns were left out to make the
// conversation fit the model that served it.
//
// Dropping turns is the right answer — a thread that outgrows a small model
// would otherwise be permanently unusable on it, and after a budget fallback
// that is not a choice the caller made — but it must not be a silent one.
const trimmedHeader = "x-llmgw-thread-trimmed"
// threadMessage is one turn as it goes to an engine.
type threadMessage struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
}
// spliceThread puts a stored conversation in front of the request's own
// messages and returns the body to send upstream.
//
// The history goes first and the request's messages last, because the request
// carries the new turn: a client using a thread sends what it just said, not
// the whole conversation again. That is the point of the feature.
//
// budget is how many bytes of prompt the serving model can take, measured the
// same way promptBytes measures it so the two cannot disagree. Zero means the
// model declares no context length, and then nothing is trimmed — guessing a
// limit for a model that never stated one would drop turns for no reason.
func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) {
if len(history) == 0 {
return body, 0, nil
}
// Decoded into a map so every field the client sent survives untouched.
// The router forwards bodies it does not understand, and a thread must not
// be the thing that starts dropping them.
var raw map[string]json.RawMessage
if err := json.Unmarshal(body, &raw); err != nil {
return nil, 0, err
}
var own []threadMessage
if m, ok := raw["messages"]; ok {
if err := json.Unmarshal(m, &own); err != nil {
return nil, 0, err
}
}
stored := make([]threadMessage, 0, len(history))
for _, m := range history {
c, err := json.Marshal(m.Content)
if err != nil {
return nil, 0, err
}
stored = append(stored, threadMessage{Role: m.Role, Content: c})
}
dropped := 0
if budget > 0 {
stored, dropped = trimToFit(stored, own, budget)
}
merged, err := json.Marshal(append(stored, own...))
if err != nil {
return nil, 0, err
}
raw["messages"] = merged
out, err := json.Marshal(raw)
if err != nil {
return nil, 0, err
}
return out, dropped, nil
}
// trimToFit drops the oldest stored turns until the conversation fits.
//
// The request's own messages are never dropped: they are what the caller just
// said, and a request that cannot fit without them is one to refuse rather
// than to answer with something the caller did not ask.
//
// System messages are kept wherever they are. They are usually instructions
// that shape every answer, and dropping one silently changes what the model is
// for — which is a stranger failure than a forgotten turn.
func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) {
fixed := messagesBytes(own)
for _, m := range stored {
if m.Role == "system" {
fixed += messageBytes(m)
}
}
// Oldest first, which is the order they are in.
keep := make([]bool, len(stored))
for i := range stored {
keep[i] = true
}
used := fixed
for _, m := range stored {
if m.Role != "system" {
used += messageBytes(m)
}
}
dropped := 0
for i := range stored {
if used <= budget {
break
}
if stored[i].Role == "system" {
continue
}
used -= messageBytes(stored[i])
keep[i] = false
dropped++
}
out := make([]threadMessage, 0, len(stored))
for i, m := range stored {
if keep[i] {
out = append(out, m)
}
}
return out, dropped
}
// messageBytes measures one turn the way promptBytes measures it, so a
// conversation trimmed to fit is measured by the same ruler that decides
// whether it fits.
func messageBytes(m threadMessage) int {
n := len(m.Role) + 2
var s string
if err := json.Unmarshal(m.Content, &s); err == nil {
return n + len(s)
}
return n + len(m.Content)
}
func messagesBytes(ms []threadMessage) int {
n := 0
for _, m := range ms {
n += messageBytes(m)
}
return n
}
// ownMessages reads the turns a request carries, for storing after a reply.
func ownMessages(body []byte) []store.Message {
var b struct {
Messages []threadMessage `json:"messages"`
}
if err := json.Unmarshal(body, &b); err != nil {
return nil
}
out := make([]store.Message, 0, len(b.Messages))
for _, m := range b.Messages {
var s string
if err := json.Unmarshal(m.Content, &s); err != nil {
// Structured content — an image, an array of parts. Stored as the
// JSON it arrived as, so replaying it gives the model what the
// client originally sent rather than a flattened version of it.
s = string(m.Content)
}
out = append(out, store.Message{Role: m.Role, Content: s})
}
return out
}