From cebf9906d2d457a788b7d186a83286724f18cc8f Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 18 Sep 2026 11:10:07 +0200 Subject: [PATCH] Splice a stored conversation into a request, trimming only what must go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The part that makes a thread work across models: the history goes in front of what the caller just said, because a client using a thread sends the new turn rather than the conversation again. Trimming has three rules, and each is there because the alternative fails quietly. The oldest stored turns go first, so a conversation that outgrew the model now serving it stays usable — after a budget fallback that model was not the caller's choice, and a permanently unusable thread would be a strange way to punish them for running out of budget. The caller's own messages are never dropped: answering something they did not ask is worse than refusing. And a system message is kept wherever it sits, because losing one silently changes what the model is for while it carries on answering. A model that declares no context length has nothing trimmed. Guessing a limit for it would drop turns for no reason and the caller would never learn why the model forgot. The body is rebuilt through a map so every field the client sent survives. Forwarding fields llmgw does not understand is how engine-specific options reach an engine at all, and a thread must not be what starts dropping them. An empty history returns the body byte for byte, so a request without a thread takes exactly the path it took before this existed. Not yet wired to the request path. Co-Authored-By: Claude Opus 5 (1M context) --- internal/router/threads.go | 180 ++++++++++++++++++++++++++++++++ internal/router/threads_test.go | 176 +++++++++++++++++++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 internal/router/threads.go create mode 100644 internal/router/threads_test.go diff --git a/internal/router/threads.go b/internal/router/threads.go new file mode 100644 index 0000000..255ceaa --- /dev/null +++ b/internal/router/threads.go @@ -0,0 +1,180 @@ +package router + +import ( + "encoding/json" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// threadHeader names the conversation a request continues. +// +// A header rather than a body field so the JSON stays exactly what an OpenAI +// client would send: some SDKs drop fields they do not know, and every one of +// them lets a caller set a header. It sits with the other x-llmgw-* headers +// the router already reads. +const threadHeader = "x-llmgw-thread" + +// trimmedHeader reports how many stored turns were left out to make the +// conversation fit the model that served it. +// +// Dropping turns is the right answer — a thread that outgrows a small model +// would otherwise be permanently unusable on it, and after a budget fallback +// that is not a choice the caller made — but it must not be a silent one. +const trimmedHeader = "x-llmgw-thread-trimmed" + +// threadMessage is one turn as it goes to an engine. +type threadMessage struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` +} + +// spliceThread puts a stored conversation in front of the request's own +// messages and returns the body to send upstream. +// +// The history goes first and the request's messages last, because the request +// carries the new turn: a client using a thread sends what it just said, not +// the whole conversation again. That is the point of the feature. +// +// budget is how many bytes of prompt the serving model can take, measured the +// same way promptBytes measures it so the two cannot disagree. Zero means the +// model declares no context length, and then nothing is trimmed — guessing a +// limit for a model that never stated one would drop turns for no reason. +func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) { + if len(history) == 0 { + return body, 0, nil + } + + // Decoded into a map so every field the client sent survives untouched. + // The router forwards bodies it does not understand, and a thread must not + // be the thing that starts dropping them. + var raw map[string]json.RawMessage + if err := json.Unmarshal(body, &raw); err != nil { + return nil, 0, err + } + + var own []threadMessage + if m, ok := raw["messages"]; ok { + if err := json.Unmarshal(m, &own); err != nil { + return nil, 0, err + } + } + + stored := make([]threadMessage, 0, len(history)) + for _, m := range history { + c, err := json.Marshal(m.Content) + if err != nil { + return nil, 0, err + } + stored = append(stored, threadMessage{Role: m.Role, Content: c}) + } + + dropped := 0 + if budget > 0 { + stored, dropped = trimToFit(stored, own, budget) + } + + merged, err := json.Marshal(append(stored, own...)) + if err != nil { + return nil, 0, err + } + raw["messages"] = merged + + out, err := json.Marshal(raw) + if err != nil { + return nil, 0, err + } + return out, dropped, nil +} + +// trimToFit drops the oldest stored turns until the conversation fits. +// +// The request's own messages are never dropped: they are what the caller just +// said, and a request that cannot fit without them is one to refuse rather +// than to answer with something the caller did not ask. +// +// System messages are kept wherever they are. They are usually instructions +// that shape every answer, and dropping one silently changes what the model is +// for — which is a stranger failure than a forgotten turn. +func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) { + fixed := messagesBytes(own) + for _, m := range stored { + if m.Role == "system" { + fixed += messageBytes(m) + } + } + + // Oldest first, which is the order they are in. + keep := make([]bool, len(stored)) + for i := range stored { + keep[i] = true + } + used := fixed + for _, m := range stored { + if m.Role != "system" { + used += messageBytes(m) + } + } + + dropped := 0 + for i := range stored { + if used <= budget { + break + } + if stored[i].Role == "system" { + continue + } + used -= messageBytes(stored[i]) + keep[i] = false + dropped++ + } + + out := make([]threadMessage, 0, len(stored)) + for i, m := range stored { + if keep[i] { + out = append(out, m) + } + } + return out, dropped +} + +// messageBytes measures one turn the way promptBytes measures it, so a +// conversation trimmed to fit is measured by the same ruler that decides +// whether it fits. +func messageBytes(m threadMessage) int { + n := len(m.Role) + 2 + var s string + if err := json.Unmarshal(m.Content, &s); err == nil { + return n + len(s) + } + return n + len(m.Content) +} + +func messagesBytes(ms []threadMessage) int { + n := 0 + for _, m := range ms { + n += messageBytes(m) + } + return n +} + +// ownMessages reads the turns a request carries, for storing after a reply. +func ownMessages(body []byte) []store.Message { + var b struct { + Messages []threadMessage `json:"messages"` + } + if err := json.Unmarshal(body, &b); err != nil { + return nil + } + out := make([]store.Message, 0, len(b.Messages)) + for _, m := range b.Messages { + var s string + if err := json.Unmarshal(m.Content, &s); err != nil { + // Structured content — an image, an array of parts. Stored as the + // JSON it arrived as, so replaying it gives the model what the + // client originally sent rather than a flattened version of it. + s = string(m.Content) + } + out = append(out, store.Message{Role: m.Role, Content: s}) + } + return out +} diff --git a/internal/router/threads_test.go b/internal/router/threads_test.go new file mode 100644 index 0000000..2b7aa67 --- /dev/null +++ b/internal/router/threads_test.go @@ -0,0 +1,176 @@ +package router + +import ( + "encoding/json" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// splicedRoles reads back the roles and contents of a spliced body, which is +// what the engine will actually see. +func splicedRoles(t *testing.T, body []byte) []string { + t.Helper() + var b struct { + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages"` + } + if err := json.Unmarshal(body, &b); err != nil { + t.Fatalf("the spliced body is not valid JSON: %v\n%s", err, body) + } + out := make([]string, 0, len(b.Messages)) + for _, m := range b.Messages { + out = append(out, m.Role+":"+m.Content) + } + return out +} + +func TestHistoryGoesBeforeWhatTheCallerJustSaid(t *testing.T) { + // The whole point of a thread: the client sends the new turn, not the + // conversation again. Order is what makes the reply make sense. + body := []byte(`{"model":"m","messages":[{"role":"user","content":"third"}]}`) + history := []store.Message{ + {Role: "user", Content: "first"}, + {Role: "assistant", Content: "second"}, + } + + out, dropped, err := spliceThread(body, history, 0) + if err != nil { + t.Fatal(err) + } + if dropped != 0 { + t.Errorf("nothing should have been dropped, %d was", dropped) + } + got := splicedRoles(t, out) + want := []string{"user:first", "assistant:second", "user:third"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("messages came out as %v, want %v", got, want) + } +} + +func TestEverythingElseInTheBodySurvives(t *testing.T) { + // The router forwards fields it does not understand — that is how engine + // specific options reach an engine at all. Splicing a thread must not be + // the thing that starts dropping them. + body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}],` + + `"temperature":0.2,"stream":true,"some_engine_option":{"nested":[1,2]}}`) + + out, _, err := spliceThread(body, []store.Message{{Role: "user", Content: "old"}}, 0) + if err != nil { + t.Fatal(err) + } + var got map[string]json.RawMessage + if err := json.Unmarshal(out, &got); err != nil { + t.Fatal(err) + } + for _, k := range []string{"model", "temperature", "stream", "some_engine_option"} { + if _, ok := got[k]; !ok { + t.Errorf("%q was dropped by splicing", k) + } + } + if string(got["some_engine_option"]) != `{"nested":[1,2]}` { + t.Errorf("a field llmgw does not understand was rewritten: %s", got["some_engine_option"]) + } +} + +func TestTheOldestTurnsAreDroppedFirst(t *testing.T) { + // A conversation that outgrew the model it is now being served by — after + // a budget fallback, that is not a choice the caller made, so the thread + // has to stay usable. + history := []store.Message{ + {Role: "user", Content: strings.Repeat("a", 100)}, + {Role: "assistant", Content: strings.Repeat("b", 100)}, + {Role: "user", Content: strings.Repeat("c", 100)}, + {Role: "assistant", Content: "keep me"}, + } + body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`) + + out, dropped, err := spliceThread(body, history, 200) + if err != nil { + t.Fatal(err) + } + if dropped == 0 { + t.Fatal("nothing was dropped although the history was over budget") + } + got := strings.Join(splicedRoles(t, out), "|") + if !strings.Contains(got, "keep me") { + t.Errorf("the most recent stored turn was dropped: %s", got) + } + if !strings.Contains(got, "user:now") { + t.Errorf("the caller's own turn was dropped: %s", got) + } + if strings.Contains(got, strings.Repeat("a", 100)) { + t.Errorf("the oldest turn survived while newer ones were dropped: %s", got) + } +} + +func TestASystemMessageIsNeverDropped(t *testing.T) { + // It is usually the instruction that shapes every answer. Losing it + // silently changes what the model is for, which is a stranger failure than + // a forgotten turn — and much harder to spot, because the model keeps + // answering. + history := []store.Message{ + {Role: "system", Content: "You only answer in Turkish."}, + {Role: "user", Content: strings.Repeat("x", 500)}, + {Role: "assistant", Content: strings.Repeat("y", 500)}, + } + body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`) + + out, dropped, err := spliceThread(body, history, 100) + if err != nil { + t.Fatal(err) + } + if dropped != 2 { + t.Errorf("dropped %d turns, expected both non-system ones", dropped) + } + if got := strings.Join(splicedRoles(t, out), "|"); !strings.Contains(got, "only answer in Turkish") { + t.Errorf("the system message was dropped: %s", got) + } +} + +func TestAModelWithNoDeclaredContextTrimsNothing(t *testing.T) { + // Guessing a limit for a model that never stated one would drop turns for + // no reason, and the caller would never learn why the model forgot. + history := make([]store.Message, 50) + for i := range history { + history[i] = store.Message{Role: "user", Content: strings.Repeat("z", 1000)} + } + body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`) + + _, dropped, err := spliceThread(body, history, 0) + if err != nil { + t.Fatal(err) + } + if dropped != 0 { + t.Errorf("%d turns were dropped for a model that declares no context", dropped) + } +} + +func TestNoHistoryLeavesTheBodyByteForByte(t *testing.T) { + // A request without a thread, and the first request of a new one, must + // take exactly the path they took before this existed. + body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`) + out, dropped, err := spliceThread(body, nil, 4096) + if err != nil { + t.Fatal(err) + } + if dropped != 0 || string(out) != string(body) { + t.Errorf("an empty history rewrote the body:\n%s", out) + } +} + +func TestStructuredContentIsStoredAsItArrived(t *testing.T) { + // An image or an array of parts. Flattening it to a string would replay + // something the client never sent. + body := []byte(`{"model":"m","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`) + got := ownMessages(body) + if len(got) != 1 { + t.Fatalf("read %d messages", len(got)) + } + if !strings.HasPrefix(got[0].Content, `[{"type":"text"`) { + t.Errorf("structured content was flattened: %q", got[0].Content) + } +}