Chat goes to the replica that has been answering soonest
CI / test (push) Failing after 7m18s

The router keeps each endpoint's time to first token, smoothed over its last
streamed answers, and the chat profile scores by it beside session, queue,
KV and inflight: the fastest endpoint 1, one twice as slow 0.5, one not yet
measured 1 so a new replica is tried. Load still sees a slow drain from the
engine's own counters; this sees what answers actually took. BACKLOG L4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-25 17:01:30 +01:00
1 parent 0b37796067
commit 73e1bb6733
9 files changed
+106 -3

No files matched your search

+25
View File
@@ -117,6 +117,10 @@ type Snapshot struct {
// that a replica is slow.
DrainRate float64
// TTFT is the time to first token of the streamed answers this endpoint
// gave lately, exponentially smoothed; zero until it has given one.
TTFT time.Duration
// Inflight counts requests the router has dispatched here that have not
// finished. It compensates for scrape lag (§7.4); without it, herding is
// unavoidable within a 2s scrape interval.
@@ -165,6 +169,7 @@ type live struct {
lastSucc float64
drainRate float64
maxRunning int
ttft float64 // seconds, smoothed
inflight int
brk *breaker.Breaker
@@ -247,6 +252,25 @@ func (r *Registry) Infos() []Info {
return out
}
// ObserveTTFT records how long an answer streamed here took to its first token.
func (r *Registry) ObserveTTFT(id string, d time.Duration) {
if d <= 0 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
l, ok := r.m[id]
if !ok {
return
}
const alpha = 0.2 // a few answers move it; one slow one does not
if l.ttft == 0 {
l.ttft = d.Seconds()
} else {
l.ttft = alpha*d.Seconds() + (1-alpha)*l.ttft
}
}
// Observe records a successful scrape and updates the drain rate.
func (r *Registry) Observe(id string, s Stats) {
r.mu.Lock()
@@ -296,6 +320,7 @@ func (r *Registry) Snapshots(model string) []Snapshot {
Info: l.info,
Stats: l.stats,
DrainRate: l.drainRate,
TTFT: time.Duration(l.ttft * float64(time.Second)),
Inflight: l.inflight,
RunningCapacity: l.maxRunning,
Stale: stale,
+12
View File
@@ -307,3 +307,15 @@ func TestLastUsedSurvivesAnEndpointListRefresh(t *testing.T) {
t.Errorf("last used = %v (present=%v) after a republish, want %v", got, ok, before)
}
}
func TestTTFTIsSmoothedPerEndpoint(t *testing.T) {
r := NewRegistry(Config{})
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
r.ObserveTTFT("a", time.Second)
r.ObserveTTFT("a", 2*time.Second)
r.ObserveTTFT("nobody", time.Second)
s := r.Snapshots("m")
if len(s) != 1 || s[0].TTFT != 1200*time.Millisecond {
t.Errorf("%v", s)
}
}
+3
View File
@@ -1261,6 +1261,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
r.keyRefused(model, dec.Endpoint.ID)
}
releaseAdapter()
if perr == nil && out.Status == http.StatusOK {
r.reg.ObserveTTFT(dec.Endpoint.ID, out.TTFT)
}
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
uspan.SetAttributes(genAIUsage(req.Path, model, out.Usage)...)
endSpan(uspan, perr)
+20
View File
@@ -0,0 +1,20 @@
package scheduler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
func TestLatencyPrefersTheEndpointThatAnswersSooner(t *testing.T) {
eps := []endpoint.Snapshot{
{TTFT: 200 * time.Millisecond},
{TTFT: 400 * time.Millisecond},
{}, // not yet measured
}
got := Latency().Score(nil, eps)
if got[0] != 1 || got[1] != 0.5 || got[2] != 1 {
t.Errorf("scores %v, want [1 0.5 1]", got)
}
}
+1
View File
@@ -56,6 +56,7 @@ func Chat(c ProfileConfig) *Profile {
{Queue(c.TargetWaiting), 1.0},
{KV(c.KVCeiling), 1.0},
{Inflight(c.TargetInflight), 1.0},
{Latency(), 1.0},
{Weight(), 0.5},
}, 2.0),
Picker: P2C(),
+33
View File
@@ -2,6 +2,7 @@ package scheduler
import (
"math"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
@@ -178,3 +179,35 @@ func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
}
return out
}
// Latency scores by time to first token as this router has seen it: the
// fastest endpoint 1, one twice as slow 0.5. Kong and others balance by it;
// for chat it is what a person waits for. An endpoint not yet measured
// scores 1, so a new replica is tried rather than avoided.
//
// Beside Load, not instead: Load sees a replica draining slowly from its
// own counters, before any answer of this router's has come back from it;
// this sees what answers actually took, a slow card or a busy host included.
func Latency() Scorer { return latencyScorer{} }
type latencyScorer struct{}
func (latencyScorer) Name() string { return "latency" }
func (latencyScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
best := time.Duration(0)
for _, e := range eps {
if e.TTFT > 0 && (best == 0 || e.TTFT < best) {
best = e.TTFT
}
}
out := make([]float64, len(eps))
for i, e := range eps {
if e.TTFT <= 0 || best == 0 {
out[i] = 1
continue
}
out[i] = float64(best) / float64(e.TTFT)
}
return out
}