Splice a stored conversation into a request, trimming only what must go
CI / test (push) Successful in 1m39s
CI / test (push) Successful in 1m39s
The part that makes a thread work across models: the history goes in front of what the caller just said, because a client using a thread sends the new turn rather than the conversation again. Trimming has three rules, and each is there because the alternative fails quietly. The oldest stored turns go first, so a conversation that outgrew the model now serving it stays usable — after a budget fallback that model was not the caller's choice, and a permanently unusable thread would be a strange way to punish them for running out of budget. The caller's own messages are never dropped: answering something they did not ask is worse than refusing. And a system message is kept wherever it sits, because losing one silently changes what the model is for while it carries on answering. A model that declares no context length has nothing trimmed. Guessing a limit for it would drop turns for no reason and the caller would never learn why the model forgot. The body is rebuilt through a map so every field the client sent survives. Forwarding fields llmgw does not understand is how engine-specific options reach an engine at all, and a thread must not be what starts dropping them. An empty history returns the body byte for byte, so a request without a thread takes exactly the path it took before this existed. Not yet wired to the request path. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b0592670a4
commit
cebf9906d2
2 files changed
+356
No files matched your search
@@ -0,0 +1,180 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// threadHeader names the conversation a request continues.
|
||||
//
|
||||
// A header rather than a body field so the JSON stays exactly what an OpenAI
|
||||
// client would send: some SDKs drop fields they do not know, and every one of
|
||||
// them lets a caller set a header. It sits with the other x-llmgw-* headers
|
||||
// the router already reads.
|
||||
const threadHeader = "x-llmgw-thread"
|
||||
|
||||
// trimmedHeader reports how many stored turns were left out to make the
|
||||
// conversation fit the model that served it.
|
||||
//
|
||||
// Dropping turns is the right answer — a thread that outgrows a small model
|
||||
// would otherwise be permanently unusable on it, and after a budget fallback
|
||||
// that is not a choice the caller made — but it must not be a silent one.
|
||||
const trimmedHeader = "x-llmgw-thread-trimmed"
|
||||
|
||||
// threadMessage is one turn as it goes to an engine.
|
||||
type threadMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
}
|
||||
|
||||
// spliceThread puts a stored conversation in front of the request's own
|
||||
// messages and returns the body to send upstream.
|
||||
//
|
||||
// The history goes first and the request's messages last, because the request
|
||||
// carries the new turn: a client using a thread sends what it just said, not
|
||||
// the whole conversation again. That is the point of the feature.
|
||||
//
|
||||
// budget is how many bytes of prompt the serving model can take, measured the
|
||||
// same way promptBytes measures it so the two cannot disagree. Zero means the
|
||||
// model declares no context length, and then nothing is trimmed — guessing a
|
||||
// limit for a model that never stated one would drop turns for no reason.
|
||||
func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) {
|
||||
if len(history) == 0 {
|
||||
return body, 0, nil
|
||||
}
|
||||
|
||||
// Decoded into a map so every field the client sent survives untouched.
|
||||
// The router forwards bodies it does not understand, and a thread must not
|
||||
// be the thing that starts dropping them.
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var own []threadMessage
|
||||
if m, ok := raw["messages"]; ok {
|
||||
if err := json.Unmarshal(m, &own); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
stored := make([]threadMessage, 0, len(history))
|
||||
for _, m := range history {
|
||||
c, err := json.Marshal(m.Content)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
stored = append(stored, threadMessage{Role: m.Role, Content: c})
|
||||
}
|
||||
|
||||
dropped := 0
|
||||
if budget > 0 {
|
||||
stored, dropped = trimToFit(stored, own, budget)
|
||||
}
|
||||
|
||||
merged, err := json.Marshal(append(stored, own...))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
raw["messages"] = merged
|
||||
|
||||
out, err := json.Marshal(raw)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return out, dropped, nil
|
||||
}
|
||||
|
||||
// trimToFit drops the oldest stored turns until the conversation fits.
|
||||
//
|
||||
// The request's own messages are never dropped: they are what the caller just
|
||||
// said, and a request that cannot fit without them is one to refuse rather
|
||||
// than to answer with something the caller did not ask.
|
||||
//
|
||||
// System messages are kept wherever they are. They are usually instructions
|
||||
// that shape every answer, and dropping one silently changes what the model is
|
||||
// for — which is a stranger failure than a forgotten turn.
|
||||
func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) {
|
||||
fixed := messagesBytes(own)
|
||||
for _, m := range stored {
|
||||
if m.Role == "system" {
|
||||
fixed += messageBytes(m)
|
||||
}
|
||||
}
|
||||
|
||||
// Oldest first, which is the order they are in.
|
||||
keep := make([]bool, len(stored))
|
||||
for i := range stored {
|
||||
keep[i] = true
|
||||
}
|
||||
used := fixed
|
||||
for _, m := range stored {
|
||||
if m.Role != "system" {
|
||||
used += messageBytes(m)
|
||||
}
|
||||
}
|
||||
|
||||
dropped := 0
|
||||
for i := range stored {
|
||||
if used <= budget {
|
||||
break
|
||||
}
|
||||
if stored[i].Role == "system" {
|
||||
continue
|
||||
}
|
||||
used -= messageBytes(stored[i])
|
||||
keep[i] = false
|
||||
dropped++
|
||||
}
|
||||
|
||||
out := make([]threadMessage, 0, len(stored))
|
||||
for i, m := range stored {
|
||||
if keep[i] {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, dropped
|
||||
}
|
||||
|
||||
// messageBytes measures one turn the way promptBytes measures it, so a
|
||||
// conversation trimmed to fit is measured by the same ruler that decides
|
||||
// whether it fits.
|
||||
func messageBytes(m threadMessage) int {
|
||||
n := len(m.Role) + 2
|
||||
var s string
|
||||
if err := json.Unmarshal(m.Content, &s); err == nil {
|
||||
return n + len(s)
|
||||
}
|
||||
return n + len(m.Content)
|
||||
}
|
||||
|
||||
func messagesBytes(ms []threadMessage) int {
|
||||
n := 0
|
||||
for _, m := range ms {
|
||||
n += messageBytes(m)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ownMessages reads the turns a request carries, for storing after a reply.
|
||||
func ownMessages(body []byte) []store.Message {
|
||||
var b struct {
|
||||
Messages []threadMessage `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]store.Message, 0, len(b.Messages))
|
||||
for _, m := range b.Messages {
|
||||
var s string
|
||||
if err := json.Unmarshal(m.Content, &s); err != nil {
|
||||
// Structured content — an image, an array of parts. Stored as the
|
||||
// JSON it arrived as, so replaying it gives the model what the
|
||||
// client originally sent rather than a flattened version of it.
|
||||
s = string(m.Content)
|
||||
}
|
||||
out = append(out, store.Message{Role: m.Role, Content: s})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// splicedRoles reads back the roles and contents of a spliced body, which is
|
||||
// what the engine will actually see.
|
||||
func splicedRoles(t *testing.T, body []byte) []string {
|
||||
t.Helper()
|
||||
var b struct {
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &b); err != nil {
|
||||
t.Fatalf("the spliced body is not valid JSON: %v\n%s", err, body)
|
||||
}
|
||||
out := make([]string, 0, len(b.Messages))
|
||||
for _, m := range b.Messages {
|
||||
out = append(out, m.Role+":"+m.Content)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestHistoryGoesBeforeWhatTheCallerJustSaid(t *testing.T) {
|
||||
// The whole point of a thread: the client sends the new turn, not the
|
||||
// conversation again. Order is what makes the reply make sense.
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"third"}]}`)
|
||||
history := []store.Message{
|
||||
{Role: "user", Content: "first"},
|
||||
{Role: "assistant", Content: "second"},
|
||||
}
|
||||
|
||||
out, dropped, err := spliceThread(body, history, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dropped != 0 {
|
||||
t.Errorf("nothing should have been dropped, %d was", dropped)
|
||||
}
|
||||
got := splicedRoles(t, out)
|
||||
want := []string{"user:first", "assistant:second", "user:third"}
|
||||
if strings.Join(got, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("messages came out as %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEverythingElseInTheBodySurvives(t *testing.T) {
|
||||
// The router forwards fields it does not understand — that is how engine
|
||||
// specific options reach an engine at all. Splicing a thread must not be
|
||||
// the thing that starts dropping them.
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}],` +
|
||||
`"temperature":0.2,"stream":true,"some_engine_option":{"nested":[1,2]}}`)
|
||||
|
||||
out, _, err := spliceThread(body, []store.Message{{Role: "user", Content: "old"}}, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]json.RawMessage
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"model", "temperature", "stream", "some_engine_option"} {
|
||||
if _, ok := got[k]; !ok {
|
||||
t.Errorf("%q was dropped by splicing", k)
|
||||
}
|
||||
}
|
||||
if string(got["some_engine_option"]) != `{"nested":[1,2]}` {
|
||||
t.Errorf("a field llmgw does not understand was rewritten: %s", got["some_engine_option"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOldestTurnsAreDroppedFirst(t *testing.T) {
|
||||
// A conversation that outgrew the model it is now being served by — after
|
||||
// a budget fallback, that is not a choice the caller made, so the thread
|
||||
// has to stay usable.
|
||||
history := []store.Message{
|
||||
{Role: "user", Content: strings.Repeat("a", 100)},
|
||||
{Role: "assistant", Content: strings.Repeat("b", 100)},
|
||||
{Role: "user", Content: strings.Repeat("c", 100)},
|
||||
{Role: "assistant", Content: "keep me"},
|
||||
}
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
|
||||
|
||||
out, dropped, err := spliceThread(body, history, 200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dropped == 0 {
|
||||
t.Fatal("nothing was dropped although the history was over budget")
|
||||
}
|
||||
got := strings.Join(splicedRoles(t, out), "|")
|
||||
if !strings.Contains(got, "keep me") {
|
||||
t.Errorf("the most recent stored turn was dropped: %s", got)
|
||||
}
|
||||
if !strings.Contains(got, "user:now") {
|
||||
t.Errorf("the caller's own turn was dropped: %s", got)
|
||||
}
|
||||
if strings.Contains(got, strings.Repeat("a", 100)) {
|
||||
t.Errorf("the oldest turn survived while newer ones were dropped: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASystemMessageIsNeverDropped(t *testing.T) {
|
||||
// It is usually the instruction that shapes every answer. Losing it
|
||||
// silently changes what the model is for, which is a stranger failure than
|
||||
// a forgotten turn — and much harder to spot, because the model keeps
|
||||
// answering.
|
||||
history := []store.Message{
|
||||
{Role: "system", Content: "You only answer in Turkish."},
|
||||
{Role: "user", Content: strings.Repeat("x", 500)},
|
||||
{Role: "assistant", Content: strings.Repeat("y", 500)},
|
||||
}
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
|
||||
|
||||
out, dropped, err := spliceThread(body, history, 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dropped != 2 {
|
||||
t.Errorf("dropped %d turns, expected both non-system ones", dropped)
|
||||
}
|
||||
if got := strings.Join(splicedRoles(t, out), "|"); !strings.Contains(got, "only answer in Turkish") {
|
||||
t.Errorf("the system message was dropped: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWithNoDeclaredContextTrimsNothing(t *testing.T) {
|
||||
// Guessing a limit for a model that never stated one would drop turns for
|
||||
// no reason, and the caller would never learn why the model forgot.
|
||||
history := make([]store.Message, 50)
|
||||
for i := range history {
|
||||
history[i] = store.Message{Role: "user", Content: strings.Repeat("z", 1000)}
|
||||
}
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"now"}]}`)
|
||||
|
||||
_, dropped, err := spliceThread(body, history, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dropped != 0 {
|
||||
t.Errorf("%d turns were dropped for a model that declares no context", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoHistoryLeavesTheBodyByteForByte(t *testing.T) {
|
||||
// A request without a thread, and the first request of a new one, must
|
||||
// take exactly the path they took before this existed.
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":"hi"}]}`)
|
||||
out, dropped, err := spliceThread(body, nil, 4096)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dropped != 0 || string(out) != string(body) {
|
||||
t.Errorf("an empty history rewrote the body:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredContentIsStoredAsItArrived(t *testing.T) {
|
||||
// An image or an array of parts. Flattening it to a string would replay
|
||||
// something the client never sent.
|
||||
body := []byte(`{"model":"m","messages":[{"role":"user","content":[{"type":"text","text":"hi"}]}]}`)
|
||||
got := ownMessages(body)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("read %d messages", len(got))
|
||||
}
|
||||
if !strings.HasPrefix(got[0].Content, `[{"type":"text"`) {
|
||||
t.Errorf("structured content was flattened: %q", got[0].Content)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user