Splice a stored conversation into a request, trimming only what must go
CI / test (push) Successful in 1m39s

The part that makes a thread work across models: the history goes in front of
what the caller just said, because a client using a thread sends the new turn
rather than the conversation again.

Trimming has three rules, and each is there because the alternative fails
quietly. The oldest stored turns go first, so a conversation that outgrew the
model now serving it stays usable — after a budget fallback that model was not
the caller's choice, and a permanently unusable thread would be a strange way
to punish them for running out of budget. The caller's own messages are never
dropped: answering something they did not ask is worse than refusing. And a
system message is kept wherever it sits, because losing one silently changes
what the model is for while it carries on answering.

A model that declares no context length has nothing trimmed. Guessing a limit
for it would drop turns for no reason and the caller would never learn why the
model forgot.

The body is rebuilt through a map so every field the client sent survives.
Forwarding fields llmgw does not understand is how engine-specific options
reach an engine at all, and a thread must not be what starts dropping them.
An empty history returns the body byte for byte, so a request without a thread
takes exactly the path it took before this existed.

Not yet wired to the request path.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-18 11:10:07 +02:00
1 parent b0592670a4
commit cebf9906d2
2 files changed
+356

No files matched your search

+180
View File
@@ -0,0 +1,180 @@
package router
import (
"encoding/json"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// threadHeader names the conversation a request continues.
//
// A header rather than a body field so the JSON stays exactly what an OpenAI
// client would send: some SDKs drop fields they do not know, and every one of
// them lets a caller set a header. It sits with the other x-llmgw-* headers
// the router already reads.
const threadHeader = "x-llmgw-thread"
// trimmedHeader reports how many stored turns were left out to make the
// conversation fit the model that served it.
//
// Dropping turns is the right answer — a thread that outgrows a small model
// would otherwise be permanently unusable on it, and after a budget fallback
// that is not a choice the caller made — but it must not be a silent one.
const trimmedHeader = "x-llmgw-thread-trimmed"
// threadMessage is one turn as it goes to an engine.
type threadMessage struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
}
// spliceThread puts a stored conversation in front of the request's own
// messages and returns the body to send upstream.
//
// The history goes first and the request's messages last, because the request
// carries the new turn: a client using a thread sends what it just said, not
// the whole conversation again. That is the point of the feature.
//
// budget is how many bytes of prompt the serving model can take, measured the
// same way promptBytes measures it so the two cannot disagree. Zero means the
// model declares no context length, and then nothing is trimmed — guessing a
// limit for a model that never stated one would drop turns for no reason.
func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) {
if len(history) == 0 {
return body, 0, nil
}
// Decoded into a map so every field the client sent survives untouched.
// The router forwards bodies it does not understand, and a thread must not
// be the thing that starts dropping them.
var raw map[string]json.RawMessage
if err := json.Unmarshal(body, &raw); err != nil {
return nil, 0, err
}
var own []threadMessage
if m, ok := raw["messages"]; ok {
if err := json.Unmarshal(m, &own); err != nil {
return nil, 0, err
}
}
stored := make([]threadMessage, 0, len(history))
for _, m := range history {
c, err := json.Marshal(m.Content)
if err != nil {
return nil, 0, err
}
stored = append(stored, threadMessage{Role: m.Role, Content: c})
}
dropped := 0
if budget > 0 {
stored, dropped = trimToFit(stored, own, budget)
}
merged, err := json.Marshal(append(stored, own...))
if err != nil {
return nil, 0, err
}
raw["messages"] = merged
out, err := json.Marshal(raw)
if err != nil {
return nil, 0, err
}
return out, dropped, nil
}
// trimToFit drops the oldest stored turns until the conversation fits.
//
// The request's own messages are never dropped: they are what the caller just
// said, and a request that cannot fit without them is one to refuse rather
// than to answer with something the caller did not ask.
//
// System messages are kept wherever they are. They are usually instructions
// that shape every answer, and dropping one silently changes what the model is
// for — which is a stranger failure than a forgotten turn.
func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) {
fixed := messagesBytes(own)
for _, m := range stored {
if m.Role == "system" {
fixed += messageBytes(m)
}
}
// Oldest first, which is the order they are in.
keep := make([]bool, len(stored))
for i := range stored {
keep[i] = true
}
used := fixed
for _, m := range stored {
if m.Role != "system" {
used += messageBytes(m)
}
}
dropped := 0
for i := range stored {
if used <= budget {
break
}
if stored[i].Role == "system" {
continue
}
used -= messageBytes(stored[i])
keep[i] = false
dropped++
}
out := make([]threadMessage, 0, len(stored))
for i, m := range stored {
if keep[i] {
out = append(out, m)
}
}
return out, dropped
}
// messageBytes measures one turn the way promptBytes measures it, so a
// conversation trimmed to fit is measured by the same ruler that decides
// whether it fits.
func messageBytes(m threadMessage) int {
n := len(m.Role) + 2
var s string
if err := json.Unmarshal(m.Content, &s); err == nil {
return n + len(s)
}
return n + len(m.Content)
}
func messagesBytes(ms []threadMessage) int {
n := 0
for _, m := range ms {
n += messageBytes(m)
}
return n
}
// ownMessages reads the turns a request carries, for storing after a reply.
func ownMessages(body []byte) []store.Message {
var b struct {
Messages []threadMessage `json:"messages"`
}
if err := json.Unmarshal(body, &b); err != nil {
return nil
}
out := make([]store.Message, 0, len(b.Messages))
for _, m := range b.Messages {
var s string
if err := json.Unmarshal(m.Content, &s); err != nil {
// Structured content — an image, an array of parts. Stored as the
// JSON it arrived as, so replaying it gives the model what the
// client originally sent rather than a flattened version of it.
s = string(m.Content)
}
out = append(out, store.Message{Role: m.Role, Content: s})
}
return out
}
+176
View File
@@ -0,0 +1,176 @@
package router
import (
"encoding/json"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// splicedRoles reads back the roles and contents of a spliced body, which is
// what the engine will actually see.
func splicedRoles(t *testing.T, body []byte) []string {
t.Helper()
var b struct {
Messages []struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"messages"`
}
if err := json.Unmarshal(body, &b); err != nil {
t.Fatalf("the spliced body is not valid JSON: %v\n%s", err, body)
}
out := make([]string, 0, len(b.Messages))
for _, m := range b.Messages {
out = append(out, m.Role+":"+m.Content)
}
return out
}
func TestHistoryGoesBeforeWhatTheCallerJustSaid(t *testing.T) {
// The whole point of a thread: the client sends the new turn, not the
// conversation again. Order is what makes the reply make sense.
body := []byte(`{"model":"m","messages":[{"role":"user","content":"third"}]}`)
history := []store.Message{
{Role: "user", Content: "first"},
{Role: "assistant", Content: "second"},
}
out, dropped, err := spliceThread(body, history, 0)
if err != nil {
t.Fatal(err)
}
if dropped != 0 {
t.Errorf("nothing should have been dropped, %d was", dropped)
}
got := splicedRoles(t, out)
want := []string{"user:first", "assistant:second", "user:third"}
if strings.Join(got, "|") != strings.Join(want, "|") {
t.Errorf("messages came out as %v, want %v", got, want)
}
}
func TestEverythingElseInTheBodySurvives(t *testing.T) {
// The router forwards fields it does not understand — that is how engine
// specific options reach an engine at all. Splicing a thread must not be
// the thing that starts dropping them.
body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}],` +
`"temperature":0.2,"stream":true,"some_engine_option":{"nested":[1,2]}}`)
out, _, err := spliceThread(body, []store.Message{{Role: "user", Content: "old"}}, 0)
if err != nil {
t.Fatal(err)
}
var got map[string]json.RawMessage
if err := json.Unmarshal(out, &got); err != nil {
t.Fatal(err)
}
for _, k := range []string{"model", "temperature", "stream", "some_engine_option"} {
if _, ok := got[k]; !ok {
t.Errorf("%q was dropped by splicing", k)
}
}
if string(got["some_engine_option"]) != `{"nested":[1,2]}` {
t.Errorf("a field llmgw does not understand was rewritten: %s", got["some_engine_option"])
}
}
func TestTheOldestTurnsAreDroppedFirst(t *testing.T) {
// A conversation that outgrew the model it is now being served by — after
// a budget fallback, that is not a choice the caller made, so the thread
// has to stay usable.
history := []store.Message{
{Role: "user", Content: strings.Repeat("a", 100)},
{Role: "assistant", Content: strings.Repeat("b", 100)},
{Role: "user", Content: strings.Repeat("c", 100)},
{Role: "assistant", Content: "keep me"},
}
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
out, dropped, err := spliceThread(body, history, 200)
if err != nil {
t.Fatal(err)
}
if dropped == 0 {
t.Fatal("nothing was dropped although the history was over budget")
}
got := strings.Join(splicedRoles(t, out), "|")
if !strings.Contains(got, "keep me") {
t.Errorf("the most recent stored turn was dropped: %s", got)
}
if !strings.Contains(got, "user:now") {
t.Errorf("the caller's own turn was dropped: %s", got)
}
if strings.Contains(got, strings.Repeat("a", 100)) {
t.Errorf("the oldest turn survived while newer ones were dropped: %s", got)
}
}
func TestASystemMessageIsNeverDropped(t *testing.T) {
// It is usually the instruction that shapes every answer. Losing it
// silently changes what the model is for, which is a stranger failure than
// a forgotten turn — and much harder to spot, because the model keeps
// answering.
history := []store.Message{
{Role: "system", Content: "You only answer in Turkish."},
{Role: "user", Content: strings.Repeat("x", 500)},
{Role: "assistant", Content: strings.Repeat("y", 500)},
}
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
out, dropped, err := spliceThread(body, history, 100)
if err != nil {
t.Fatal(err)
}
if dropped != 2 {
t.Errorf("dropped %d turns, expected both non-system ones", dropped)
}
if got := strings.Join(splicedRoles(t, out), "|"); !strings.Contains(got, "only answer in Turkish") {
t.Errorf("the system message was dropped: %s", got)
}
}
func TestAModelWithNoDeclaredContextTrimsNothing(t *testing.T) {
// Guessing a limit for a model that never stated one would drop turns for
// no reason, and the caller would never learn why the model forgot.
history := make([]store.Message, 50)
for i := range history {
history[i] = store.Message{Role: "user", Content: strings.Repeat("z", 1000)}
}
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
_, dropped, err := spliceThread(body, history, 0)
if err != nil {
t.Fatal(err)
}
if dropped != 0 {
t.Errorf("%d turns were dropped for a model that declares no context", dropped)
}
}
func TestNoHistoryLeavesTheBodyByteForByte(t *testing.T) {
// A request without a thread, and the first request of a new one, must
// take exactly the path they took before this existed.
body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)
out, dropped, err := spliceThread(body, nil, 4096)
if err != nil {
t.Fatal(err)
}
if dropped != 0 || string(out) != string(body) {
t.Errorf("an empty history rewrote the body:\n%s", out)
}
}
func TestStructuredContentIsStoredAsItArrived(t *testing.T) {
// An image or an array of parts. Flattening it to a string would replay
// something the client never sent.
body := []byte(`{"model":"m","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`)
got := ownMessages(body)
if len(got) != 1 {
t.Fatalf("read %d messages", len(got))
}
if !strings.HasPrefix(got[0].Content, `[{"type":"text"`) {
t.Errorf("structured content was flattened: %q", got[0].Content)
}
}