Files
llmgw/internal/router/endpoint/endpoint.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

298 lines
7.5 KiB
Go

// Package endpoint tracks the upstream replicas the router can send to, along
// with their live state.
//
// In phase 1 the endpoint list comes from configuration. In phase 3 the control
// plane's EndpointSource (plan §6.4) will feed the same Registry; the scheduler
// and proxy must not notice the difference, which is why the API is shaped
// around wholesale list replacement from the start.
package endpoint
import (
"maps"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
)
// Info is a replica's static description.
type Info struct {
// ID uniquely identifies the replica and labels its metrics.
ID string
// URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash.
URL string
// Models lists the model names this replica serves.
Models []string
// Pool names the (fabric, gpu_class) pool. Used for observability and pool
// preference only; phase 1 may well have a single pool.
Pool string
// Weight is the capacity weight measured by ctl bench (§6.2.1). 0 means 1.
Weight float64
// Priority expresses pool preference; higher pools are tried first.
Priority int
}
// Serves reports whether the replica serves the given model.
func (i Info) Serves(model string) bool {
return slices.Contains(i.Models, model)
}
// Stats is the router-side view of the engine metric contract (plan §7.6).
type Stats struct {
// The three mandatory fields.
Running int
Waiting int
KVCacheUsage float64
// Optional.
BlockSize int
NumGPUBlocks int
// Raw counters, kept so derived rates can be computed.
PromptTokens float64
GenerationTokens float64
SuccessTotal float64
}
// Snapshot is a complete view of one endpoint at scheduling time.
// It is a value type so the scheduler can work without holding a lock.
type Snapshot struct {
Info
Stats Stats
// DrainRate is the observed completion rate in requests/sec, exponentially
// smoothed. This is AIBrix's load-balance signal: Running alone cannot tell
// that a replica is slow.
DrainRate float64
// Inflight counts requests the router has dispatched here that have not
// finished. It compensates for scrape lag (§7.4); without it, herding is
// unavoidable within a 2s scrape interval.
Inflight int
// Stale means no successful scrape arrived within StaleAfter. Stale
// endpoints are not dropped, only scored down.
Stale bool
// Age is the time since the last successful scrape.
Age time.Duration
// Breaker is the circuit breaker's current state, read without side effects.
Breaker breaker.State
}
// Config holds Registry settings.
type Config struct {
// StaleAfter is how long an endpoint may go without a successful scrape
// before being considered stale.
StaleAfter time.Duration
// Breaker holds per-endpoint circuit breaker settings.
Breaker breaker.Config
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
StaleAfter: 10 * time.Second,
Breaker: breaker.DefaultConfig(),
}
}
// live is one endpoint's mutable state.
type live struct {
info Info
stats Stats
scrapedAt time.Time
lastSucc float64
drainRate float64
inflight int
brk *breaker.Breaker
}
// Registry holds the live state of all endpoints. It is safe for concurrent use.
type Registry struct {
cfg Config
now func() time.Time
mu sync.RWMutex
m map[string]*live
}
// NewRegistry creates an empty registry.
func NewRegistry(cfg Config) *Registry {
if cfg.StaleAfter <= 0 {
cfg.StaleAfter = 10 * time.Second
}
return &Registry{cfg: cfg, now: time.Now, m: make(map[string]*live)}
}
// SetClock replaces the time source, for tests.
func (r *Registry) SetClock(now func() time.Time) {
r.mu.Lock()
r.now = now
r.mu.Unlock()
}
// Replace reconciles the endpoint list with the given set. Endpoints that
// remain keep their live state, new ones get a breaker, removed ones are dropped.
func (r *Registry) Replace(infos []Info) {
r.mu.Lock()
defer r.mu.Unlock()
seen := make(map[string]struct{}, len(infos))
for _, in := range infos {
seen[in.ID] = struct{}{}
if l, ok := r.m[in.ID]; ok {
l.info = in
continue
}
r.m[in.ID] = &live{info: in, brk: breaker.New(r.cfg.Breaker)}
}
for id := range r.m {
if _, ok := seen[id]; !ok {
delete(r.m, id)
}
}
}
// IDs returns the registered endpoint IDs in sorted order.
func (r *Registry) IDs() []string {
r.mu.RLock()
defer r.mu.RUnlock()
return slices.Sorted(maps.Keys(r.m))
}
// Infos returns the registered endpoint descriptions, sorted by ID.
func (r *Registry) Infos() []Info {
r.mu.RLock()
defer r.mu.RUnlock()
out := make([]Info, 0, len(r.m))
for _, l := range r.m {
out = append(out, l.info)
}
slices.SortFunc(out, func(a, b Info) int { return strcmp(a.ID, b.ID) })
return out
}
// Observe records a successful scrape and updates the drain rate.
func (r *Registry) Observe(id string, s Stats) {
r.mu.Lock()
defer r.mu.Unlock()
l, ok := r.m[id]
if !ok {
return
}
now := r.now()
if !l.scrapedAt.IsZero() {
dt := now.Sub(l.scrapedAt).Seconds()
// A counter going backwards means the engine restarted; skip the derivative.
if dt > 0 && s.SuccessTotal >= l.lastSucc {
rate := (s.SuccessTotal - l.lastSucc) / dt
const alpha = 0.3 // exponential smoothing factor
l.drainRate = alpha*rate + (1-alpha)*l.drainRate
} else if s.SuccessTotal < l.lastSucc {
l.drainRate = 0
}
}
l.stats = s
l.lastSucc = s.SuccessTotal
l.scrapedAt = now
}
// Snapshots returns a view of every endpoint serving the given model, or of all
// endpoints when model is empty. Breaker state is read, never consumed.
func (r *Registry) Snapshots(model string) []Snapshot {
r.mu.RLock()
defer r.mu.RUnlock()
now := r.now()
out := make([]Snapshot, 0, len(r.m))
for _, l := range r.m {
if model != "" && !l.info.Serves(model) {
continue
}
var age time.Duration
stale := true
if !l.scrapedAt.IsZero() {
age = now.Sub(l.scrapedAt)
stale = age > r.cfg.StaleAfter
}
out = append(out, Snapshot{
Info: l.info,
Stats: l.stats,
DrainRate: l.drainRate,
Inflight: l.inflight,
Stale: stale,
Age: age,
Breaker: l.brk.State(),
})
}
slices.SortFunc(out, func(a, b Snapshot) int { return strcmp(a.ID, b.ID) })
return out
}
// Acquire asks permission to send a request to an endpoint.
//
// It returns (nil, false) if the circuit breaker refuses. On success the
// inflight counter is incremented and the returned function must be called to
// report the outcome: release(err == nil).
func (r *Registry) Acquire(id string) (release func(ok bool), granted bool) {
r.mu.Lock()
l, exists := r.m[id]
r.mu.Unlock()
if !exists {
return nil, false
}
if !l.brk.Allow() {
return nil, false
}
r.mu.Lock()
l.inflight++
r.mu.Unlock()
var once sync.Once
return func(ok bool) {
once.Do(func() {
r.mu.Lock()
l.inflight--
r.mu.Unlock()
if ok {
l.brk.Success()
} else {
l.brk.Failure()
}
})
}, true
}
// ScrapeFailed reports a failed metrics scrape.
//
// A scrape failure is not a request failure, so it does not touch the breaker:
// the endpoint may be serving fine while only its metrics endpoint is down.
// It will go stale on its own and score lower (§7.6).
func (r *Registry) ScrapeFailed(string) {}
// Inflight returns an endpoint's current inflight count, for tests and metrics.
func (r *Registry) Inflight(id string) int {
r.mu.RLock()
defer r.mu.RUnlock()
if l, ok := r.m[id]; ok {
return l.inflight
}
return 0
}
func strcmp(a, b string) int {
switch {
case a < b:
return -1
case a > b:
return 1
default:
return 0
}
}