- The similar-question cache embeds a question as its project would: a project kept in the cluster has external fallbacks refused, and an embedding model outside the cluster is not asked at all. The vectors are compared outside the cache's lock. The question's bound is in characters. - The restorer puts values back in streamed tool calls' arguments (escaped for the JSON they are), completions' text and reasoning, each held across events on its own, and releases a held tail with the finish event. A whole answer is walked as JSON, so a value with a quote no longer breaks tool-call arguments. - An outside guardrail service reads the user's turn as redacted, and a streamed answer once rather than at the finish chunk and again at the end. guardrails.md says it can stop a stream only under buffer_all. - A replica's smoothed time to first token is forgotten after five minutes without an answer: one found slow and then given nothing was never tried again. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
343 lines
9.0 KiB
Go
343 lines
9.0 KiB
Go
package endpoint
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
)
|
|
|
|
type clock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
}
|
|
|
|
func newClock() *clock {
|
|
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
|
|
}
|
|
func (c *clock) now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
return c.t
|
|
}
|
|
func (c *clock) advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
c.t = c.t.Add(d)
|
|
c.mu.Unlock()
|
|
}
|
|
|
|
func newReg(t *testing.T) (*Registry, *clock) {
|
|
t.Helper()
|
|
r := NewRegistry(Config{StaleAfter: 10 * time.Second, Breaker: breaker.DefaultConfig()})
|
|
c := newClock()
|
|
r.SetClock(c.now)
|
|
return r, c
|
|
}
|
|
|
|
func TestReplaceKeepsLiveState(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", URL: "http://a", Models: []string{"m"}}})
|
|
r.Observe("a", Stats{Running: 5})
|
|
|
|
// Re-supplying the same endpoint must not lose its stats.
|
|
r.Replace([]Info{
|
|
{ID: "a", URL: "http://a", Models: []string{"m"}, Weight: 2},
|
|
{ID: "b", URL: "http://b", Models: []string{"m"}},
|
|
})
|
|
|
|
snaps := r.Snapshots("m")
|
|
if len(snaps) != 2 {
|
|
t.Fatalf("expected 2 endpoints, got %d", len(snaps))
|
|
}
|
|
if snaps[0].Stats.Running != 5 {
|
|
t.Error("Replace reset the live state")
|
|
}
|
|
if snaps[0].Weight != 2 {
|
|
t.Error("Replace did not update the static info")
|
|
}
|
|
}
|
|
|
|
func TestReplaceRemovesMissingEndpoints(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}, {ID: "b", Models: []string{"m"}}})
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
if got := r.IDs(); len(got) != 1 || got[0] != "a" {
|
|
t.Errorf("endpoint dropped from the list was not removed: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestFilterByModel(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{
|
|
{ID: "a", Models: []string{"llama"}},
|
|
{ID: "b", Models: []string{"qwen"}},
|
|
{ID: "c", Models: []string{"llama", "qwen"}},
|
|
})
|
|
|
|
if got := len(r.Snapshots("llama")); got != 2 {
|
|
t.Errorf("got %d endpoints for llama", got)
|
|
}
|
|
if got := len(r.Snapshots("")); got != 3 {
|
|
t.Errorf("empty model should return all, got %d", got)
|
|
}
|
|
}
|
|
|
|
func TestStalenessDetection(t *testing.T) {
|
|
r, c := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
// With no scrape at all the endpoint counts as stale.
|
|
if !r.Snapshots("m")[0].Stale {
|
|
t.Error("not marked stale despite having no measurement")
|
|
}
|
|
|
|
r.Observe("a", Stats{Running: 1})
|
|
if r.Snapshots("m")[0].Stale {
|
|
t.Error("a fresh measurement was marked stale")
|
|
}
|
|
|
|
c.advance(11 * time.Second)
|
|
s := r.Snapshots("m")[0]
|
|
if !s.Stale {
|
|
t.Error("not marked stale after StaleAfter elapsed")
|
|
}
|
|
if s.Age < 11*time.Second {
|
|
t.Errorf("Age is wrong: %v", s.Age)
|
|
}
|
|
}
|
|
|
|
func TestDrainRateCalculation(t *testing.T) {
|
|
r, c := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
r.Observe("a", Stats{SuccessTotal: 100})
|
|
c.advance(time.Second)
|
|
r.Observe("a", Stats{SuccessTotal: 110}) // 10 requests/sec
|
|
|
|
if got := r.Snapshots("m")[0].DrainRate; got <= 0 {
|
|
t.Fatalf("drain rate was not computed: %v", got)
|
|
}
|
|
|
|
// Exponential smoothing: one sample approaches the raw rate but stays below it.
|
|
if got := r.Snapshots("m")[0].DrainRate; got > 10 {
|
|
t.Errorf("drain rate exceeded the raw value: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestDrainRateResetsWhenCounterGoesBackwards(t *testing.T) {
|
|
// Counters reset when an engine restarts; that must not yield a negative rate.
|
|
r, c := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
r.Observe("a", Stats{SuccessTotal: 1000})
|
|
c.advance(time.Second)
|
|
r.Observe("a", Stats{SuccessTotal: 1100})
|
|
c.advance(time.Second)
|
|
r.Observe("a", Stats{SuccessTotal: 5}) // restarted
|
|
|
|
if got := r.Snapshots("m")[0].DrainRate; got != 0 {
|
|
t.Errorf("drain rate was not reset after the counter went backwards: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAcquireCountsInflight(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
rel1, ok := r.Acquire("a")
|
|
if !ok {
|
|
t.Fatal("Acquire was refused")
|
|
}
|
|
rel2, ok := r.Acquire("a")
|
|
if !ok {
|
|
t.Fatal("second Acquire was refused")
|
|
}
|
|
if got := r.Inflight("a"); got != 2 {
|
|
t.Errorf("inflight=%d, expected 2", got)
|
|
}
|
|
|
|
rel1(true)
|
|
rel2(true)
|
|
if got := r.Inflight("a"); got != 0 {
|
|
t.Errorf("inflight=%d after release", got)
|
|
}
|
|
}
|
|
|
|
func TestReleaseIsIdempotent(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
rel, _ := r.Acquire("a")
|
|
rel(true)
|
|
rel(true) // must be idempotent; the counter must not go negative
|
|
|
|
if got := r.Inflight("a"); got != 0 {
|
|
t.Errorf("double release corrupted the counter: %d", got)
|
|
}
|
|
}
|
|
|
|
func TestFailuresOpenTheCircuit(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
for range 3 { // DefaultConfig threshold is 3
|
|
rel, ok := r.Acquire("a")
|
|
if !ok {
|
|
t.Fatal("circuit opened too early")
|
|
}
|
|
rel(false)
|
|
}
|
|
|
|
if _, ok := r.Acquire("a"); ok {
|
|
t.Error("circuit did not open after consecutive failures")
|
|
}
|
|
if r.Snapshots("m")[0].Breaker != breaker.Open {
|
|
t.Error("snapshot does not reflect the breaker state")
|
|
}
|
|
}
|
|
|
|
func TestScrapeFailureDoesNotTripBreaker(t *testing.T) {
|
|
// A scrape failure is not a request failure: the endpoint may be serving
|
|
// fine while only its metrics endpoint is unreachable.
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
|
|
for range 10 {
|
|
r.ScrapeFailed("a")
|
|
}
|
|
if _, ok := r.Acquire("a"); !ok {
|
|
t.Error("scrape failures opened the circuit")
|
|
}
|
|
}
|
|
|
|
func TestUnknownEndpoint(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
if _, ok := r.Acquire("missing"); ok {
|
|
t.Error("an unregistered endpoint was allowed")
|
|
}
|
|
r.Observe("missing", Stats{}) // must not panic
|
|
}
|
|
|
|
func TestConcurrentAccess(t *testing.T) {
|
|
r, _ := newReg(t)
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}, {ID: "b", Models: []string{"m"}}})
|
|
|
|
var wg sync.WaitGroup
|
|
for i := range 100 {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
id := "a"
|
|
if i%2 == 0 {
|
|
id = "b"
|
|
}
|
|
if rel, ok := r.Acquire(id); ok {
|
|
rel(i%3 != 0)
|
|
}
|
|
r.Observe(id, Stats{Running: i})
|
|
_ = r.Snapshots("m")
|
|
_ = r.Inflight(id)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func TestLastUsedIsRecordedOnDispatch(t *testing.T) {
|
|
// The fact nothing recorded. Whether a replica is earning the cards it
|
|
// holds is the first question on a fleet with more models than GPUs.
|
|
r := NewRegistry(Config{})
|
|
clock := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
|
|
r.SetClock(func() time.Time { return clock })
|
|
r.Replace([]Info{{ID: "a", URL: "http://x", Models: []string{"m"}}})
|
|
|
|
if len(r.LastUsed()) != 0 {
|
|
t.Fatal("an endpoint nothing has used reported a last-use time")
|
|
}
|
|
|
|
release, ok := r.Acquire("a")
|
|
if !ok {
|
|
t.Fatal("acquire refused")
|
|
}
|
|
release(true)
|
|
|
|
used := r.LastUsed()
|
|
if got, ok := used["a"]; !ok || !got.Equal(clock) {
|
|
t.Errorf("last used = %v (present=%v), want %v", got, ok, clock)
|
|
}
|
|
}
|
|
|
|
func TestAnUnusedEndpointIsAbsentRatherThanZero(t *testing.T) {
|
|
// A zero time would be the oldest idle replica in any comparison, so a
|
|
// router restart would nominate the whole fleet for eviction at once.
|
|
r := NewRegistry(Config{})
|
|
r.Replace([]Info{
|
|
{ID: "used", URL: "http://x", Models: []string{"m"}},
|
|
{ID: "never", URL: "http://y", Models: []string{"m"}},
|
|
})
|
|
release, _ := r.Acquire("used")
|
|
release(true)
|
|
|
|
used := r.LastUsed()
|
|
if _, ok := used["never"]; ok {
|
|
t.Error("an endpoint nothing has used appeared in the map")
|
|
}
|
|
if _, ok := used["used"]; !ok {
|
|
t.Error("the used endpoint is missing")
|
|
}
|
|
}
|
|
|
|
func TestLastUsedSurvivesAnEndpointListRefresh(t *testing.T) {
|
|
// The list is republished whenever anything changes, which on a large
|
|
// fleet is often. Losing usage on every republish would make idleness
|
|
// unmeasurable exactly where it matters most.
|
|
r := NewRegistry(Config{})
|
|
r.Replace([]Info{{ID: "a", URL: "http://x", Models: []string{"m"}}})
|
|
release, _ := r.Acquire("a")
|
|
release(true)
|
|
before := r.LastUsed()["a"]
|
|
|
|
// Same endpoint, republished alongside a new one.
|
|
r.Replace([]Info{
|
|
{ID: "a", URL: "http://x", Models: []string{"m"}},
|
|
{ID: "b", URL: "http://y", Models: []string{"m"}},
|
|
})
|
|
if got, ok := r.LastUsed()["a"]; !ok || !got.Equal(before) {
|
|
t.Errorf("last used = %v (present=%v) after a republish, want %v", got, ok, before)
|
|
}
|
|
}
|
|
|
|
func TestTTFTIsSmoothedPerEndpoint(t *testing.T) {
|
|
r := NewRegistry(Config{})
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
r.ObserveTTFT("a", time.Second)
|
|
r.ObserveTTFT("a", 2*time.Second)
|
|
r.ObserveTTFT("nobody", time.Second)
|
|
s := r.Snapshots("m")
|
|
if len(s) != 1 || s[0].TTFT != 1200*time.Millisecond {
|
|
t.Errorf("%v", s)
|
|
}
|
|
}
|
|
|
|
func TestAReplicaFoundSlowIsTriedAgainLater(t *testing.T) {
|
|
// Slow once and then given nothing, its figure would never mend.
|
|
r := NewRegistry(Config{})
|
|
now := time.Now()
|
|
r.now = func() time.Time { return now }
|
|
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
|
|
r.ObserveTTFT("a", 10*time.Second)
|
|
if s := r.Snapshots("m"); s[0].TTFT != 10*time.Second {
|
|
t.Fatalf("%v", s[0].TTFT)
|
|
}
|
|
now = now.Add(6 * time.Minute)
|
|
if s := r.Snapshots("m"); s[0].TTFT != 0 {
|
|
t.Errorf("six minutes on, still %v", s[0].TTFT)
|
|
}
|
|
// The next answer starts the figure afresh rather than smoothing it in.
|
|
r.ObserveTTFT("a", time.Second)
|
|
if s := r.Snapshots("m"); s[0].TTFT != time.Second {
|
|
t.Errorf("after a new answer %v", s[0].TTFT)
|
|
}
|
|
}
|