Chat goes to the replica that has been answering soonest
The router keeps each endpoint's time to first token, smoothed over its last streamed answers, and the chat profile scores by it beside session, queue, KV and inflight: the fastest endpoint 1, one twice as slow 0.5, one not yet measured 1 so a new replica is tried. Load still sees a slow drain from the engine's own counters; this sees what answers actually took. BACKLOG L4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
0b37796067
commit
73e1bb6733
9 files changed
+106
-3
No files matched your search
@@ -117,6 +117,10 @@ type Snapshot struct {
|
||||
// that a replica is slow.
|
||||
DrainRate float64
|
||||
|
||||
// TTFT is the time to first token of the streamed answers this endpoint
|
||||
// gave lately, exponentially smoothed; zero until it has given one.
|
||||
TTFT time.Duration
|
||||
|
||||
// Inflight counts requests the router has dispatched here that have not
|
||||
// finished. It compensates for scrape lag (§7.4); without it, herding is
|
||||
// unavoidable within a 2s scrape interval.
|
||||
@@ -165,6 +169,7 @@ type live struct {
|
||||
lastSucc float64
|
||||
drainRate float64
|
||||
maxRunning int
|
||||
ttft float64 // seconds, smoothed
|
||||
|
||||
inflight int
|
||||
brk *breaker.Breaker
|
||||
@@ -247,6 +252,25 @@ func (r *Registry) Infos() []Info {
|
||||
return out
|
||||
}
|
||||
|
||||
// ObserveTTFT records how long an answer streamed here took to its first token.
|
||||
func (r *Registry) ObserveTTFT(id string, d time.Duration) {
|
||||
if d <= 0 {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
l, ok := r.m[id]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
const alpha = 0.2 // a few answers move it; one slow one does not
|
||||
if l.ttft == 0 {
|
||||
l.ttft = d.Seconds()
|
||||
} else {
|
||||
l.ttft = alpha*d.Seconds() + (1-alpha)*l.ttft
|
||||
}
|
||||
}
|
||||
|
||||
// Observe records a successful scrape and updates the drain rate.
|
||||
func (r *Registry) Observe(id string, s Stats) {
|
||||
r.mu.Lock()
|
||||
@@ -296,6 +320,7 @@ func (r *Registry) Snapshots(model string) []Snapshot {
|
||||
Info: l.info,
|
||||
Stats: l.stats,
|
||||
DrainRate: l.drainRate,
|
||||
TTFT: time.Duration(l.ttft * float64(time.Second)),
|
||||
Inflight: l.inflight,
|
||||
RunningCapacity: l.maxRunning,
|
||||
Stale: stale,
|
||||
|
||||
@@ -307,3 +307,15 @@ func TestLastUsedSurvivesAnEndpointListRefresh(t *testing.T) {
|
||||
t.Errorf("last used = %v (present=%v) after a republish, want %v", got, ok, before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTTFTIsSmoothedPerEndpoint(t *testing.T) {
|
||||
r := NewRegistry(Config{})
|
||||
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
||||
r.ObserveTTFT("a", time.Second)
|
||||
r.ObserveTTFT("a", 2*time.Second)
|
||||
r.ObserveTTFT("nobody", time.Second)
|
||||
s := r.Snapshots("m")
|
||||
if len(s) != 1 || s[0].TTFT != 1200*time.Millisecond {
|
||||
t.Errorf("%v", s)
|
||||
}
|
||||
}
|
||||
@@ -1261,6 +1261,9 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
r.keyRefused(model, dec.Endpoint.ID)
|
||||
}
|
||||
releaseAdapter()
|
||||
if perr == nil && out.Status == http.StatusOK {
|
||||
r.reg.ObserveTTFT(dec.Endpoint.ID, out.TTFT)
|
||||
}
|
||||
uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status))
|
||||
uspan.SetAttributes(genAIUsage(req.Path, model, out.Usage)...)
|
||||
endSpan(uspan, perr)
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
func TestLatencyPrefersTheEndpointThatAnswersSooner(t *testing.T) {
|
||||
eps := []endpoint.Snapshot{
|
||||
{TTFT: 200 * time.Millisecond},
|
||||
{TTFT: 400 * time.Millisecond},
|
||||
{}, // not yet measured
|
||||
}
|
||||
got := Latency().Score(nil, eps)
|
||||
if got[0] != 1 || got[1] != 0.5 || got[2] != 1 {
|
||||
t.Errorf("scores %v, want [1 0.5 1]", got)
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,7 @@ func Chat(c ProfileConfig) *Profile {
|
||||
{Queue(c.TargetWaiting), 1.0},
|
||||
{KV(c.KVCeiling), 1.0},
|
||||
{Inflight(c.TargetInflight), 1.0},
|
||||
{Latency(), 1.0},
|
||||
{Weight(), 0.5},
|
||||
}, 2.0),
|
||||
Picker: P2C(),
|
||||
|
||||
@@ -2,6 +2,7 @@ package scheduler
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
@@ -178,3 +179,35 @@ func (s sessionScorer) Score(c *Ctx, eps []endpoint.Snapshot) []float64 {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Latency scores by time to first token as this router has seen it: the
|
||||
// fastest endpoint 1, one twice as slow 0.5. Kong and others balance by it;
|
||||
// for chat it is what a person waits for. An endpoint not yet measured
|
||||
// scores 1, so a new replica is tried rather than avoided.
|
||||
//
|
||||
// Beside Load, not instead: Load sees a replica draining slowly from its
|
||||
// own counters, before any answer of this router's has come back from it;
|
||||
// this sees what answers actually took, a slow card or a busy host included.
|
||||
func Latency() Scorer { return latencyScorer{} }
|
||||
|
||||
type latencyScorer struct{}
|
||||
|
||||
func (latencyScorer) Name() string { return "latency" }
|
||||
|
||||
func (latencyScorer) Score(_ *Ctx, eps []endpoint.Snapshot) []float64 {
|
||||
best := time.Duration(0)
|
||||
for _, e := range eps {
|
||||
if e.TTFT > 0 && (best == 0 || e.TTFT < best) {
|
||||
best = e.TTFT
|
||||
}
|
||||
}
|
||||
out := make([]float64, len(eps))
|
||||
for i, e := range eps {
|
||||
if e.TTFT <= 0 || best == 0 {
|
||||
out[i] = 1
|
||||
continue
|
||||
}
|
||||
out[i] = float64(best) / float64(e.TTFT)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in new issue
Block a user