The router keeps each endpoint's time to first token, smoothed over its last streamed answers, and the chat profile scores by it beside session, queue, KV and inflight: the fastest endpoint 1, one twice as slow 0.5, one not yet measured 1 so a new replica is tried. Load still sees a slow drain from the engine's own counters; this sees what answers actually took. BACKLOG L4. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
21 lines
437 B
Go
21 lines
437 B
Go
package scheduler
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
func TestLatencyPrefersTheEndpointThatAnswersSooner(t *testing.T) {
|
|
eps := []endpoint.Snapshot{
|
|
{TTFT: 200 * time.Millisecond},
|
|
{TTFT: 400 * time.Millisecond},
|
|
{}, // not yet measured
|
|
}
|
|
got := Latency().Score(nil, eps)
|
|
if got[0] != 1 || got[1] != 0.5 || got[2] != 1 {
|
|
t.Errorf("scores %v, want [1 0.5 1]", got)
|
|
}
|
|
}
|