package router import ( "encoding/json" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // threadHeader names the conversation a request continues. // // A header rather than a body field so the JSON stays exactly what an OpenAI // client would send: some SDKs drop fields they do not know, and every one of // them lets a caller set a header. It sits with the other x-llmgw-* headers // the router already reads. const threadHeader = "x-llmgw-thread" // trimmedHeader reports how many stored turns were left out to make the // conversation fit the model that served it. // // Dropping turns is the right answer — a thread that outgrows a small model // would otherwise be permanently unusable on it, and after a budget fallback // that is not a choice the caller made — but it must not be a silent one. const trimmedHeader = "x-llmgw-thread-trimmed" // threadMessage is one turn as it goes to an engine. type threadMessage struct { Role string `json:"role"` Content json.RawMessage `json:"content"` } // spliceThread puts a stored conversation in front of the request's own // messages and returns the body to send upstream. // // The history goes first and the request's messages last, because the request // carries the new turn: a client using a thread sends what it just said, not // the whole conversation again. That is the point of the feature. // // budget is how many bytes of prompt the serving model can take, measured the // same way promptBytes measures it so the two cannot disagree. Zero means the // model declares no context length, and then nothing is trimmed — guessing a // limit for a model that never stated one would drop turns for no reason. func spliceThread(body []byte, history []store.Message, budget int) ([]byte, int, error) { if len(history) == 0 { return body, 0, nil } // Decoded into a map so every field the client sent survives untouched. // The router forwards bodies it does not understand, and a thread must not // be the thing that starts dropping them. var raw map[string]json.RawMessage if err := json.Unmarshal(body, &raw); err != nil { return nil, 0, err } var own []threadMessage if m, ok := raw["messages"]; ok { if err := json.Unmarshal(m, &own); err != nil { return nil, 0, err } } stored := make([]threadMessage, 0, len(history)) for _, m := range history { c, err := json.Marshal(m.Content) if err != nil { return nil, 0, err } stored = append(stored, threadMessage{Role: m.Role, Content: c}) } dropped := 0 if budget > 0 { stored, dropped = trimToFit(stored, own, budget) } merged, err := json.Marshal(append(stored, own...)) if err != nil { return nil, 0, err } raw["messages"] = merged out, err := json.Marshal(raw) if err != nil { return nil, 0, err } return out, dropped, nil } // trimToFit drops the oldest stored turns until the conversation fits. // // The request's own messages are never dropped: they are what the caller just // said, and a request that cannot fit without them is one to refuse rather // than to answer with something the caller did not ask. // // System messages are kept wherever they are. They are usually instructions // that shape every answer, and dropping one silently changes what the model is // for — which is a stranger failure than a forgotten turn. func trimToFit(stored, own []threadMessage, budget int) ([]threadMessage, int) { fixed := messagesBytes(own) for _, m := range stored { if m.Role == "system" { fixed += messageBytes(m) } } // Oldest first, which is the order they are in. keep := make([]bool, len(stored)) for i := range stored { keep[i] = true } used := fixed for _, m := range stored { if m.Role != "system" { used += messageBytes(m) } } dropped := 0 for i := range stored { if used <= budget { break } if stored[i].Role == "system" { continue } used -= messageBytes(stored[i]) keep[i] = false dropped++ } out := make([]threadMessage, 0, len(stored)) for i, m := range stored { if keep[i] { out = append(out, m) } } return out, dropped } // messageBytes measures one turn the way promptBytes measures it, so a // conversation trimmed to fit is measured by the same ruler that decides // whether it fits. func messageBytes(m threadMessage) int { n := len(m.Role) + 2 var s string if err := json.Unmarshal(m.Content, &s); err == nil { return n + len(s) } return n + len(m.Content) } func messagesBytes(ms []threadMessage) int { n := 0 for _, m := range ms { n += messageBytes(m) } return n } // ownMessages reads the turns a request carries, for storing after a reply. func ownMessages(body []byte) []store.Message { var b struct { Messages []threadMessage `json:"messages"` } if err := json.Unmarshal(body, &b); err != nil { return nil } out := make([]store.Message, 0, len(b.Messages)) for _, m := range b.Messages { var s string if err := json.Unmarshal(m.Content, &s); err != nil { // Structured content — an image, an array of parts. Stored as the // JSON it arrived as, so replaying it gives the model what the // client originally sent rather than a flattened version of it. s = string(m.Content) } out = append(out, store.Message{Role: m.Role, Content: s}) } return out }