// Package prefix estimates which replica already holds the prefix of a request // (plan §7.5 tier 1). // // It is an estimate and says so. The router does not know an engine's real // cache; it knows what it sent where, which is a good enough proxy because // engines cache what they are sent. Nothing here is ever authoritative — a // wrong guess costs a prefill, not a wrong answer. // // Tokens are deliberately absent. Matching on bytes needs no tokenizer, and // needing the engine's exact tokenizer is what makes tier 2 expensive enough to // be a separate project. package prefix import ( "sync" "time" ) // Config tunes the tree. type Config struct { // MaxNodes bounds the tree. Past it the least recently used branches go, or // a long-running router accumulates every prompt it has ever seen. MaxNodes int // MinMatch is the shortest match worth acting on, in bytes. Below it the // overlap is coincidence — two English prompts share "The" — and treating // coincidence as affinity is how prefix routing drowns one replica. MinMatch int // TTL is how long an entry is believed. An engine evicts on its own // schedule, so an old record describes a cache that may be long gone. TTL time.Duration } // DefaultConfig returns values that work without measurement. func DefaultConfig() Config { return Config{MaxNodes: 100_000, MinMatch: 64, TTL: 10 * time.Minute} } func (c *Config) normalize() { d := DefaultConfig() if c.MaxNodes <= 0 { c.MaxNodes = d.MaxNodes } if c.MinMatch <= 0 { c.MinMatch = d.MinMatch } if c.TTL <= 0 { c.TTL = d.TTL } } // seen is how often and how recently a replica was sent this prefix. // // The count is what makes the estimate stable. Picking the most *recent* // replica sounds right and is not: with a randomizing picker every replica ends // up in the node eventually, the freshest one flips with every request, and the // tree spends its time chasing whatever the picker last did. Measured: prefix // routing made cache hits worse than no prefix routing at all, 28.7% against // 32.8%. type seen struct { count int last time.Time } // Tree is a radix tree of request prefixes, per replica. type Tree struct { cfg Config now func() time.Time mu sync.Mutex root *node nodes int } type node struct { // label is this node's slice of the prefix, shared by everything below it. label string children map[byte]*node // replicas is which endpoints were sent a request passing through this // node — not only ending at it. Recorded along the whole path, because a // prompt that diverges partway through a label still shares everything // before the divergence, and shared leading bytes are exactly what a prefix // cache holds. // // Per replica rather than a set, so one replica's entry can expire without // taking the others with it. replicas map[string]*seen // touched drives eviction. touched time.Time } // New builds an empty tree. func New(cfg Config) *Tree { cfg.normalize() return &Tree{cfg: cfg, now: time.Now, root: &node{children: map[byte]*node{}}, nodes: 1} } // Match reports the replica with the longest usable prefix in common, and how // many bytes matched. // // Empty when nothing matched far enough to be worth acting on — which is a // normal answer, not a failure, and the caller must be able to proceed on load // alone when it comes back. func (t *Tree) Match(prompt string) (replica string, matched int) { if prompt == "" { return "", 0 } now := t.now() t.mu.Lock() defer t.mu.Unlock() cur := t.root consumed := 0 best, bestLen := "", 0 for consumed < len(prompt) { child, ok := cur.children[prompt[consumed]] if !ok { break } shared := commonPrefix(child.label, prompt[consumed:]) if shared == 0 { break } consumed += shared cur = child child.touched = now // A partial label counts. Everything recorded at or below this node // began with the bytes just matched, so a replica here holds that much // of this prompt however the rest diverges — and a cache hit is about // shared leading bytes, not about where two prompts stop agreeing. // // Requiring the whole label to match made this tree inert: the first // request for a prompt becomes one long label, the second diverges in // its last few bytes, and every lookup after that returned nothing. // Measured as 144 misses and zero hits before this was fixed. if r, ok := cur.owner(now, t.cfg.TTL); ok && consumed > bestLen { best, bestLen = r, consumed } if shared < len(child.label) { break } } if bestLen < t.cfg.MinMatch { return "", 0 } return best, bestLen } // Observe records that a replica was sent this prompt. func (t *Tree) Observe(prompt, replica string) { if prompt == "" || replica == "" { return } now := t.now() t.mu.Lock() defer t.mu.Unlock() cur := t.root rest := prompt for len(rest) > 0 { child, ok := cur.children[rest[0]] if !ok { leaf := &node{label: rest, children: map[byte]*node{}, touched: now} cur.children[rest[0]] = leaf t.nodes++ cur = leaf break } shared := commonPrefix(child.label, rest) if shared < len(child.label) { // The existing label diverges partway, so it splits into a shared // head and what used to hang below it. tail := &node{ label: child.label[shared:], children: child.children, replicas: child.replicas, touched: child.touched, } child.label = child.label[:shared] child.children = map[byte]*node{tail.label[0]: tail} child.replicas = nil t.nodes++ } child.touched = now cur = child // Marked on the way through, not only at the end: a later prompt that // diverges here still shares everything above, and that is what makes // a partial match meaningful. cur.record(replica, now) rest = rest[shared:] } cur.record(replica, now) if t.nodes > t.cfg.MaxNodes { t.evictLocked(now) } } // Forget drops a replica from the tree, for one that has gone away. // // Leaving it would have the router steering traffic towards a cache that no // longer exists, and a replica id is reused by nothing, so the entries would // never be corrected — only expire. func (t *Tree) Forget(replica string) { t.mu.Lock() defer t.mu.Unlock() forget(t.root, replica) } func forget(n *node, replica string) { delete(n.replicas, replica) for _, c := range n.children { forget(c, replica) } } // Size reports how many nodes the tree holds, for metrics and tests. func (t *Tree) Size() int { t.mu.Lock() defer t.mu.Unlock() return t.nodes } // record marks that a replica was sent something passing through this node. func (n *node) record(replica string, now time.Time) { if n.replicas == nil { n.replicas = map[string]*seen{} } if e, ok := n.replicas[replica]; ok { e.count++ e.last = now } else { n.replicas[replica] = &seen{count: 1, last: now} } n.touched = now } // owner returns the replica most likely to hold this prefix. // // Most often sent, not most recently: a count is stable under a randomizing // picker, and it still moves when traffic genuinely does. Ties go to the more // recent one, which is the only thing recency is good for here. func (n *node) owner(now time.Time, ttl time.Duration) (string, bool) { var best string var bestSeen *seen for r, s := range n.replicas { if now.Sub(s.last) > ttl { continue } switch { case bestSeen == nil, s.count > bestSeen.count, s.count == bestSeen.count && s.last.After(bestSeen.last): best, bestSeen = r, s } } return best, best != "" } // evictLocked drops the coldest half of the tree's branches. // // Half rather than just enough: trimming to exactly the limit means evicting // again on the very next insert, and the walk is the expensive part. func (t *Tree) evictLocked(now time.Time) { cutoff := t.coldestHalf(now) t.nodes = 1 + prune(t.root, cutoff) } // coldestHalf finds the touch time that splits the tree in two. func (t *Tree) coldestHalf(now time.Time) time.Time { var times []time.Time collect(t.root, ×) if len(times) == 0 { return now } // A partial sort would do, but the tree is evicted rarely and this is // simpler to be sure of. for i := 1; i < len(times); i++ { for j := i; j > 0 && times[j].Before(times[j-1]); j-- { times[j], times[j-1] = times[j-1], times[j] } } return times[len(times)/2] } func collect(n *node, out *[]time.Time) { for _, c := range n.children { *out = append(*out, c.touched) collect(c, out) } } // prune removes branches untouched since the cutoff, returning what remains. func prune(n *node, cutoff time.Time) int { kept := 0 for b, c := range n.children { if c.touched.Before(cutoff) { delete(n.children, b) continue } kept += 1 + prune(c, cutoff) } return kept } func commonPrefix(a, b string) int { n := min(len(a), len(b)) i := 0 for i < n && a[i] == b[i] { i++ } return i }