Files
llmgw/internal/router/endpoint/endpoint_test.go
T
Ilker ManapandClaude Opus 5.5 54c6b66cb9
CI / test (push) Successful in 9m48s
Fixes from review: the similar cache, restore, outside services, latency
- The similar-question cache embeds a question as its project would: a
  project kept in the cluster has external fallbacks refused, and an
  embedding model outside the cluster is not asked at all. The vectors are
  compared outside the cache's lock. The question's bound is in characters.
- The restorer puts values back in streamed tool calls' arguments (escaped
  for the JSON they are), completions' text and reasoning, each held across
  events on its own, and releases a held tail with the finish event. A
  whole answer is walked as JSON, so a value with a quote no longer breaks
  tool-call arguments.
- An outside guardrail service reads the user's turn as redacted, and a
  streamed answer once rather than at the finish chunk and again at the
  end. guardrails.md says it can stop a stream only under buffer_all.
- A replica's smoothed time to first token is forgotten after five minutes
  without an answer: one found slow and then given nothing was never tried
  again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:15:43 +01:00

343 lines
9.0 KiB
Go

package endpoint
import (
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
)
type clock struct {
mu sync.Mutex
t time.Time
}
func newClock() *clock {
return &clock{t: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) advance(d time.Duration) {
c.mu.Lock()
c.t = c.t.Add(d)
c.mu.Unlock()
}
func newReg(t *testing.T) (*Registry, *clock) {
t.Helper()
r := NewRegistry(Config{StaleAfter: 10 * time.Second, Breaker: breaker.DefaultConfig()})
c := newClock()
r.SetClock(c.now)
return r, c
}
func TestReplaceKeepsLiveState(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", URL: "http://a", Models: []string{"m"}}})
r.Observe("a", Stats{Running: 5})
// Re-supplying the same endpoint must not lose its stats.
r.Replace([]Info{
{ID: "a", URL: "http://a", Models: []string{"m"}, Weight: 2},
{ID: "b", URL: "http://b", Models: []string{"m"}},
})
snaps := r.Snapshots("m")
if len(snaps) != 2 {
t.Fatalf("expected 2 endpoints, got %d", len(snaps))
}
if snaps[0].Stats.Running != 5 {
t.Error("Replace reset the live state")
}
if snaps[0].Weight != 2 {
t.Error("Replace did not update the static info")
}
}
func TestReplaceRemovesMissingEndpoints(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}, {ID: "b", Models: []string{"m"}}})
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
if got := r.IDs(); len(got) != 1 || got[0] != "a" {
t.Errorf("endpoint dropped from the list was not removed: %v", got)
}
}
func TestFilterByModel(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{
{ID: "a", Models: []string{"llama"}},
{ID: "b", Models: []string{"qwen"}},
{ID: "c", Models: []string{"llama", "qwen"}},
})
if got := len(r.Snapshots("llama")); got != 2 {
t.Errorf("got %d endpoints for llama", got)
}
if got := len(r.Snapshots("")); got != 3 {
t.Errorf("empty model should return all, got %d", got)
}
}
func TestStalenessDetection(t *testing.T) {
r, c := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
// With no scrape at all the endpoint counts as stale.
if !r.Snapshots("m")[0].Stale {
t.Error("not marked stale despite having no measurement")
}
r.Observe("a", Stats{Running: 1})
if r.Snapshots("m")[0].Stale {
t.Error("a fresh measurement was marked stale")
}
c.advance(11 * time.Second)
s := r.Snapshots("m")[0]
if !s.Stale {
t.Error("not marked stale after StaleAfter elapsed")
}
if s.Age < 11*time.Second {
t.Errorf("Age is wrong: %v", s.Age)
}
}
func TestDrainRateCalculation(t *testing.T) {
r, c := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
r.Observe("a", Stats{SuccessTotal: 100})
c.advance(time.Second)
r.Observe("a", Stats{SuccessTotal: 110}) // 10 requests/sec
if got := r.Snapshots("m")[0].DrainRate; got <= 0 {
t.Fatalf("drain rate was not computed: %v", got)
}
// Exponential smoothing: one sample approaches the raw rate but stays below it.
if got := r.Snapshots("m")[0].DrainRate; got > 10 {
t.Errorf("drain rate exceeded the raw value: %v", got)
}
}
func TestDrainRateResetsWhenCounterGoesBackwards(t *testing.T) {
// Counters reset when an engine restarts; that must not yield a negative rate.
r, c := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
r.Observe("a", Stats{SuccessTotal: 1000})
c.advance(time.Second)
r.Observe("a", Stats{SuccessTotal: 1100})
c.advance(time.Second)
r.Observe("a", Stats{SuccessTotal: 5}) // restarted
if got := r.Snapshots("m")[0].DrainRate; got != 0 {
t.Errorf("drain rate was not reset after the counter went backwards: %v", got)
}
}
func TestAcquireCountsInflight(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
rel1, ok := r.Acquire("a")
if !ok {
t.Fatal("Acquire was refused")
}
rel2, ok := r.Acquire("a")
if !ok {
t.Fatal("second Acquire was refused")
}
if got := r.Inflight("a"); got != 2 {
t.Errorf("inflight=%d, expected 2", got)
}
rel1(true)
rel2(true)
if got := r.Inflight("a"); got != 0 {
t.Errorf("inflight=%d after release", got)
}
}
func TestReleaseIsIdempotent(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
rel, _ := r.Acquire("a")
rel(true)
rel(true) // must be idempotent; the counter must not go negative
if got := r.Inflight("a"); got != 0 {
t.Errorf("double release corrupted the counter: %d", got)
}
}
func TestFailuresOpenTheCircuit(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
for range 3 { // DefaultConfig threshold is 3
rel, ok := r.Acquire("a")
if !ok {
t.Fatal("circuit opened too early")
}
rel(false)
}
if _, ok := r.Acquire("a"); ok {
t.Error("circuit did not open after consecutive failures")
}
if r.Snapshots("m")[0].Breaker != breaker.Open {
t.Error("snapshot does not reflect the breaker state")
}
}
func TestScrapeFailureDoesNotTripBreaker(t *testing.T) {
// A scrape failure is not a request failure: the endpoint may be serving
// fine while only its metrics endpoint is unreachable.
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
for range 10 {
r.ScrapeFailed("a")
}
if _, ok := r.Acquire("a"); !ok {
t.Error("scrape failures opened the circuit")
}
}
func TestUnknownEndpoint(t *testing.T) {
r, _ := newReg(t)
if _, ok := r.Acquire("missing"); ok {
t.Error("an unregistered endpoint was allowed")
}
r.Observe("missing", Stats{}) // must not panic
}
func TestConcurrentAccess(t *testing.T) {
r, _ := newReg(t)
r.Replace([]Info{{ID: "a", Models: []string{"m"}}, {ID: "b", Models: []string{"m"}}})
var wg sync.WaitGroup
for i := range 100 {
wg.Add(1)
go func() {
defer wg.Done()
id := "a"
if i%2 == 0 {
id = "b"
}
if rel, ok := r.Acquire(id); ok {
rel(i%3 != 0)
}
r.Observe(id, Stats{Running: i})
_ = r.Snapshots("m")
_ = r.Inflight(id)
}()
}
wg.Wait()
}
func TestLastUsedIsRecordedOnDispatch(t *testing.T) {
// The fact nothing recorded. Whether a replica is earning the cards it
// holds is the first question on a fleet with more models than GPUs.
r := NewRegistry(Config{})
clock := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC)
r.SetClock(func() time.Time { return clock })
r.Replace([]Info{{ID: "a", URL: "http://x", Models: []string{"m"}}})
if len(r.LastUsed()) != 0 {
t.Fatal("an endpoint nothing has used reported a last-use time")
}
release, ok := r.Acquire("a")
if !ok {
t.Fatal("acquire refused")
}
release(true)
used := r.LastUsed()
if got, ok := used["a"]; !ok || !got.Equal(clock) {
t.Errorf("last used = %v (present=%v), want %v", got, ok, clock)
}
}
func TestAnUnusedEndpointIsAbsentRatherThanZero(t *testing.T) {
// A zero time would be the oldest idle replica in any comparison, so a
// router restart would nominate the whole fleet for eviction at once.
r := NewRegistry(Config{})
r.Replace([]Info{
{ID: "used", URL: "http://x", Models: []string{"m"}},
{ID: "never", URL: "http://y", Models: []string{"m"}},
})
release, _ := r.Acquire("used")
release(true)
used := r.LastUsed()
if _, ok := used["never"]; ok {
t.Error("an endpoint nothing has used appeared in the map")
}
if _, ok := used["used"]; !ok {
t.Error("the used endpoint is missing")
}
}
func TestLastUsedSurvivesAnEndpointListRefresh(t *testing.T) {
// The list is republished whenever anything changes, which on a large
// fleet is often. Losing usage on every republish would make idleness
// unmeasurable exactly where it matters most.
r := NewRegistry(Config{})
r.Replace([]Info{{ID: "a", URL: "http://x", Models: []string{"m"}}})
release, _ := r.Acquire("a")
release(true)
before := r.LastUsed()["a"]
// Same endpoint, republished alongside a new one.
r.Replace([]Info{
{ID: "a", URL: "http://x", Models: []string{"m"}},
{ID: "b", URL: "http://y", Models: []string{"m"}},
})
if got, ok := r.LastUsed()["a"]; !ok || !got.Equal(before) {
t.Errorf("last used = %v (present=%v) after a republish, want %v", got, ok, before)
}
}
func TestTTFTIsSmoothedPerEndpoint(t *testing.T) {
r := NewRegistry(Config{})
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
r.ObserveTTFT("a", time.Second)
r.ObserveTTFT("a", 2*time.Second)
r.ObserveTTFT("nobody", time.Second)
s := r.Snapshots("m")
if len(s) != 1 || s[0].TTFT != 1200*time.Millisecond {
t.Errorf("%v", s)
}
}
func TestAReplicaFoundSlowIsTriedAgainLater(t *testing.T) {
// Slow once and then given nothing, its figure would never mend.
r := NewRegistry(Config{})
now := time.Now()
r.now = func() time.Time { return now }
r.Replace([]Info{{ID: "a", Models: []string{"m"}}})
r.ObserveTTFT("a", 10*time.Second)
if s := r.Snapshots("m"); s[0].TTFT != 10*time.Second {
t.Fatalf("%v", s[0].TTFT)
}
now = now.Add(6 * time.Minute)
if s := r.Snapshots("m"); s[0].TTFT != 0 {
t.Errorf("six minutes on, still %v", s[0].TTFT)
}
// The next answer starts the figure afresh rather than smoothing it in.
r.ObserveTTFT("a", time.Second)
if s := r.Snapshots("m"); s[0].TTFT != time.Second {
t.Errorf("after a new answer %v", s[0].TTFT)
}
}