The part that makes a thread work across models: the history goes in front of what the caller just said, because a client using a thread sends the new turn rather than the conversation again. Trimming has three rules, and each is there because the alternative fails quietly. The oldest stored turns go first, so a conversation that outgrew the model now serving it stays usable — after a budget fallback that model was not the caller's choice, and a permanently unusable thread would be a strange way to punish them for running out of budget. The caller's own messages are never dropped: answering something they did not ask is worse than refusing. And a system message is kept wherever it sits, because losing one silently changes what the model is for while it carries on answering. A model that declares no context length has nothing trimmed. Guessing a limit for it would drop turns for no reason and the caller would never learn why the model forgot. The body is rebuilt through a map so every field the client sent survives. Forwarding fields llmgw does not understand is how engine-specific options reach an engine at all, and a thread must not be what starts dropping them. An empty history returns the body byte for byte, so a request without a thread takes exactly the path it took before this existed. Not yet wired to the request path. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
181 lines
5.2 KiB
Go
181 lines
5.2 KiB
Go
package router
|
|
|
|
import (
|
|
"encoding/json"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// threadHeader names the conversation a request continues.
|
|
//
|
|
// A header rather than a body field so the JSON stays exactly what an OpenAI
|
|
// client would send: some SDKs drop fields they do not know, and every one of
|
|
// them lets a caller set a header. It sits with the other x-llmgw-* headers
|
|
// the router already reads.
|
|
const threadHeader = "x-llmgw-thread"
|
|
|
|
// trimmedHeader reports how many stored turns were left out to make the
|
|
// conversation fit the model that served it.
|
|
//
|
|
// Dropping turns is the right answer — a thread that outgrows a small model
|
|
// would otherwise be permanently unusable on it, and after a budget fallback
|
|
// that is not a choice the caller made — but it must not be a silent one.
|
|
const trimmedHeader = "x-llmgw-thread-trimmed"
|
|
|
|
// threadMessage is one turn as it goes to an engine.
|
|
type threadMessage struct {
|
|
Role string `json:"role"`
|
|
Content json.RawMessage `json:"content"`
|
|
}
|
|
|
|
// spliceThread puts a stored conversation in front of the request's own
|
|
// messages and returns the body to send upstream.
|
|
//
|
|
// The history goes first and the request's messages last, because the request
|
|
// carries the new turn: a client using a thread sends what it just said, not
|
|
// the whole conversation again. That is the point of the feature.
|
|
//
|
|
// budget is how many bytes of prompt the serving model can take, measured the
|
|
// same way promptBytes measures it so the two cannot disagree. Zero means the
|
|
// model declares no context length, and then nothing is trimmed — guessing a
|
|
// limit for a model that never stated one would drop turns for no reason.
|
|
func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) {
|
|
if len(history) == 0 {
|
|
return body, 0, nil
|
|
}
|
|
|
|
// Decoded into a map so every field the client sent survives untouched.
|
|
// The router forwards bodies it does not understand, and a thread must not
|
|
// be the thing that starts dropping them.
|
|
var raw map[string]json.RawMessage
|
|
if err := json.Unmarshal(body, &raw); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
var own []threadMessage
|
|
if m, ok := raw["messages"]; ok {
|
|
if err := json.Unmarshal(m, &own); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
}
|
|
|
|
stored := make([]threadMessage, 0, len(history))
|
|
for _, m := range history {
|
|
c, err := json.Marshal(m.Content)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
stored = append(stored, threadMessage{Role: m.Role, Content: c})
|
|
}
|
|
|
|
dropped := 0
|
|
if budget > 0 {
|
|
stored, dropped = trimToFit(stored, own, budget)
|
|
}
|
|
|
|
merged, err := json.Marshal(append(stored, own...))
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
raw["messages"] = merged
|
|
|
|
out, err := json.Marshal(raw)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
return out, dropped, nil
|
|
}
|
|
|
|
// trimToFit drops the oldest stored turns until the conversation fits.
|
|
//
|
|
// The request's own messages are never dropped: they are what the caller just
|
|
// said, and a request that cannot fit without them is one to refuse rather
|
|
// than to answer with something the caller did not ask.
|
|
//
|
|
// System messages are kept wherever they are. They are usually instructions
|
|
// that shape every answer, and dropping one silently changes what the model is
|
|
// for — which is a stranger failure than a forgotten turn.
|
|
func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) {
|
|
fixed := messagesBytes(own)
|
|
for _, m := range stored {
|
|
if m.Role == "system" {
|
|
fixed += messageBytes(m)
|
|
}
|
|
}
|
|
|
|
// Oldest first, which is the order they are in.
|
|
keep := make([]bool, len(stored))
|
|
for i := range stored {
|
|
keep[i] = true
|
|
}
|
|
used := fixed
|
|
for _, m := range stored {
|
|
if m.Role != "system" {
|
|
used += messageBytes(m)
|
|
}
|
|
}
|
|
|
|
dropped := 0
|
|
for i := range stored {
|
|
if used <= budget {
|
|
break
|
|
}
|
|
if stored[i].Role == "system" {
|
|
continue
|
|
}
|
|
used -= messageBytes(stored[i])
|
|
keep[i] = false
|
|
dropped++
|
|
}
|
|
|
|
out := make([]threadMessage, 0, len(stored))
|
|
for i, m := range stored {
|
|
if keep[i] {
|
|
out = append(out, m)
|
|
}
|
|
}
|
|
return out, dropped
|
|
}
|
|
|
|
// messageBytes measures one turn the way promptBytes measures it, so a
|
|
// conversation trimmed to fit is measured by the same ruler that decides
|
|
// whether it fits.
|
|
func messageBytes(m threadMessage) int {
|
|
n := len(m.Role) + 2
|
|
var s string
|
|
if err := json.Unmarshal(m.Content, &s); err == nil {
|
|
return n + len(s)
|
|
}
|
|
return n + len(m.Content)
|
|
}
|
|
|
|
func messagesBytes(ms []threadMessage) int {
|
|
n := 0
|
|
for _, m := range ms {
|
|
n += messageBytes(m)
|
|
}
|
|
return n
|
|
}
|
|
|
|
// ownMessages reads the turns a request carries, for storing after a reply.
|
|
func ownMessages(body []byte) []store.Message {
|
|
var b struct {
|
|
Messages []threadMessage `json:"messages"`
|
|
}
|
|
if err := json.Unmarshal(body, &b); err != nil {
|
|
return nil
|
|
}
|
|
out := make([]store.Message, 0, len(b.Messages))
|
|
for _, m := range b.Messages {
|
|
var s string
|
|
if err := json.Unmarshal(m.Content, &s); err != nil {
|
|
// Structured content — an image, an array of parts. Stored as the
|
|
// JSON it arrived as, so replaying it gives the model what the
|
|
// client originally sent rather than a flattened version of it.
|
|
s = string(m.Content)
|
|
}
|
|
out = append(out, store.Message{Role: m.Role, Content: s})
|
|
}
|
|
return out
|
|
}
|