Files
llmgw/internal/router/endpoint/endpoint.go
T
Ilker ManapandClaude Opus 5.5 54c6b66cb9 Fixes from review: the similar cache, restore, outside services, latency
- The similar-question cache embeds a question as its project would: a
  project kept in the cluster has external fallbacks refused, and an
  embedding model outside the cluster is not asked at all. The vectors are
  compared outside the cache's lock. The question's bound is in characters.
- The restorer puts values back in streamed tool calls' arguments (escaped
  for the JSON they are), completions' text and reasoning, each held across
  events on its own, and releases a held tail with the finish event. A
  whole answer is walked as JSON, so a value with a quote no longer breaks
  tool-call arguments.
- An outside guardrail service reads the user's turn as redacted, and a
  streamed answer once rather than at the finish chunk and again at the
  end. guardrails.md says it can stop a stream only under buffer_all.
- A replica's smoothed time to first token is forgotten after five minutes
  without an answer: one found slow and then given nothing was never tried
  again.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 08:15:43 +01:00

434 lines
13 KiB
Go

// Package endpoint tracks the upstream replicas the router can send to, along
// with their live state.
//
// In phase 1 the endpoint list comes from configuration. In phase 3 the control
// plane's EndpointSource (plan §6.4) will feed the same Registry; the scheduler
// and proxy must not notice the difference, which is why the API is shaped
// around wholesale list replacement from the start.
package endpoint
import (
"maps"
"slices"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
)
// Info is a replica's static description.
type Info struct {
// ID uniquely identifies the replica and labels its metrics.
ID string
// URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash.
URL string
// APIKey is what the engine demands of every request, sent as a bearer
// token. Empty for an engine that asks for none.
APIKey string
// Models lists the model names this replica serves.
Models []string
// Engine names the inference engine serving here, which decides what its
// metrics are called. Empty means vLLM's names.
Engine string
// Pool names the (fabric, gpu_class) pool. Used for observability and pool
// preference only; phase 1 may well have a single pool.
Pool string
// Weight is the capacity weight measured by ctl bench (§6.2.1). 0 means 1.
Weight float64
// Priority expresses pool preference; higher pools are tried first.
Priority int
// UpstreamModel is what the endpoint calls the model, when that differs
// from the catalog's name: an external provider's. Requests are rewritten
// to it. Empty sends the name as the client did.
UpstreamModel string
// Peer names the federated llmgw cluster behind an external endpoint,
// empty for every other. PeerCluster is that cluster's own id, which a
// request that has already passed through it carries in its hop header,
// and PeerTags whether requests sent there name who sent them.
Peer string
PeerCluster string
PeerTags bool
// Workflow is the ComfyUI workflow sent for an endpoint whose engine is
// EngineComfyUI: a built-in's name or the workflow's JSON.
Workflow string
// Comfy is, for a shared ComfyUI replica that answers for several
// models, each one's workflow and checkpoint, by every name it goes by.
// Nil for a replica of one model, which Workflow and UpstreamModel say.
Comfy map[string]ComfyRoute
}
// ComfyRoute is what a request for one model of a shared ComfyUI replica is
// sent.
type ComfyRoute struct {
Workflow string
Checkpoint string
}
// ComfyFor is the workflow and checkpoint a request for model is sent here.
func (i Info) ComfyFor(model string) (workflow, checkpoint string) {
if r, ok := i.Comfy[model]; ok {
return r.Workflow, r.Checkpoint
}
return i.Workflow, i.UpstreamModel
}
// EngineExternal marks an endpoint that is a provider outside the cluster.
// It publishes no metrics, so it is never scraped and never reads as stale.
const EngineExternal = "external"
// EngineComfyUI marks a ComfyUI server. The router speaks ComfyUI's API to
// it, not OpenAI's, and it publishes no metrics either.
const EngineComfyUI = "comfyui"
// Scraped reports whether an engine publishes metrics the router reads. One
// that does not is never read as stale for having none.
func Scraped(engine string) bool { return engine != EngineExternal && engine != EngineComfyUI }
// Serves reports whether the replica serves the given model.
func (i Info) Serves(model string) bool {
return slices.Contains(i.Models, model)
}
// Stats is the router-side view of the engine metric contract (plan §7.6).
type Stats struct {
// The three mandatory fields.
Running int
Waiting int
KVCacheUsage float64
// Optional.
BlockSize int
NumGPUBlocks int
// Raw counters, kept so derived rates can be computed.
PromptTokens float64
GenerationTokens float64
SuccessTotal float64
}
// Snapshot is a complete view of one endpoint at scheduling time.
// It is a value type so the scheduler can work without holding a lock.
type Snapshot struct {
Info
Stats Stats
// DrainRate is the observed completion rate in requests/sec, exponentially
// smoothed. This is AIBrix's load-balance signal: Running alone cannot tell
// that a replica is slow.
DrainRate float64
// TTFT is the time to first token of the streamed answers this endpoint
// gave lately, exponentially smoothed; zero until it has given one.
TTFT time.Duration
// Inflight counts requests the router has dispatched here that have not
// finished. It compensates for scrape lag (§7.4); without it, herding is
// unavoidable within a 2s scrape interval.
Inflight int
// RunningCapacity is the highest concurrent request count ever observed on
// this endpoint — in practice the engine's batch capacity, learned rather
// than configured. Without it, requests the router has dispatched but the
// engine has not yet reported are indistinguishable from queue depth, and
// during a scrape interval that makes a perfectly healthy engine look
// overloaded.
RunningCapacity int
// Stale means no successful scrape arrived within StaleAfter. Stale
// endpoints are not dropped, only scored down.
Stale bool
// Age is the time since the last successful scrape.
Age time.Duration
// Breaker is the circuit breaker's current state, read without side effects.
Breaker breaker.State
}
// Config holds Registry settings.
type Config struct {
// StaleAfter is how long an endpoint may go without a successful scrape
// before being considered stale.
StaleAfter time.Duration
// Breaker holds per-endpoint circuit breaker settings.
Breaker breaker.Config
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
StaleAfter: 10 * time.Second,
Breaker: breaker.DefaultConfig(),
}
}
// live is one endpoint's mutable state.
type live struct {
info Info
stats Stats
scrapedAt time.Time
lastSucc float64
drainRate float64
maxRunning int
ttft float64 // seconds, smoothed
ttftAt time.Time
inflight int
brk *breaker.Breaker
// lastUsed is when a request was last dispatched here.
//
// The fact nothing recorded. Whether a replica is earning the cards it
// holds is the first question on a fleet with more models than GPUs, and
// the control plane had no way to ask it: it knew what was placed and
// whether it was healthy, never whether anyone wanted it.
//
// Recorded here because Acquire is already the one place a request meets an
// endpoint, and already holds the lock. One timestamp per dispatch, and
// collection is one pass over the replicas rather than anything per
// request — which is what makes it affordable on a fleet of hundreds.
lastUsed time.Time
}
// Registry holds the live state of all endpoints. It is safe for concurrent use.
type Registry struct {
cfg Config
now func() time.Time
mu sync.RWMutex
m map[string]*live
}
// NewRegistry creates an empty registry.
func NewRegistry(cfg Config) *Registry {
if cfg.StaleAfter <= 0 {
cfg.StaleAfter = 10 * time.Second
}
return &Registry{cfg: cfg, now: time.Now, m: make(map[string]*live)}
}
// SetClock replaces the time source, for tests.
func (r *Registry) SetClock(now func() time.Time) {
r.mu.Lock()
r.now = now
r.mu.Unlock()
}
// Replace reconciles the endpoint list with the given set. Endpoints that
// remain keep their live state, new ones get a breaker, removed ones are dropped.
func (r *Registry) Replace(infos []Info) {
r.mu.Lock()
defer r.mu.Unlock()
seen := make(map[string]struct{}, len(infos))
for _, in := range infos {
seen[in.ID] = struct{}{}
if l, ok := r.m[in.ID]; ok {
l.info = in
continue
}
r.m[in.ID] = &live{info: in, brk: breaker.New(r.cfg.Breaker)}
}
for id := range r.m {
if _, ok := seen[id]; !ok {
delete(r.m, id)
}
}
}
// IDs returns the registered endpoint IDs in sorted order.
func (r *Registry) IDs() []string {
r.mu.RLock()
defer r.mu.RUnlock()
return slices.Sorted(maps.Keys(r.m))
}
// Infos returns the registered endpoint descriptions, sorted by ID.
func (r *Registry) Infos() []Info {
r.mu.RLock()
defer r.mu.RUnlock()
out := make([]Info, 0, len(r.m))
for _, l := range r.m {
out = append(out, l.info)
}
slices.SortFunc(out, func(a, b Info) int { return strcmp(a.ID, b.ID) })
return out
}
// ttftForgotten is how long a replica's time to first token is believed
// without a new answer: one found slow and then given nothing would
// otherwise never be tried again, and its figure never mended.
const ttftForgotten = 5 * time.Minute
// ttftNow is the smoothed time to first token, none once forgotten.
func (l *live) ttftNow(now time.Time) time.Duration {
if l.ttft == 0 || now.Sub(l.ttftAt) > ttftForgotten {
return 0
}
return time.Duration(l.ttft * float64(time.Second))
}
// ObserveTTFT records how long an answer streamed here took to its first token.
func (r *Registry) ObserveTTFT(id string, d time.Duration) {
if d <= 0 {
return
}
r.mu.Lock()
defer r.mu.Unlock()
l, ok := r.m[id]
if !ok {
return
}
const alpha = 0.2 // a few answers move it; one slow one does not
now := r.now()
defer func() { l.ttftAt = now }()
if l.ttft == 0 || now.Sub(l.ttftAt) > ttftForgotten {
l.ttft = d.Seconds()
} else {
l.ttft = alpha*d.Seconds() + (1-alpha)*l.ttft
}
}
// Observe records a successful scrape and updates the drain rate.
func (r *Registry) Observe(id string, s Stats) {
r.mu.Lock()
defer r.mu.Unlock()
l, ok := r.m[id]
if !ok {
return
}
now := r.now()
if !l.scrapedAt.IsZero() {
dt := now.Sub(l.scrapedAt).Seconds()
// A counter going backwards means the engine restarted; skip the derivative.
if dt > 0 && s.SuccessTotal >= l.lastSucc {
rate := (s.SuccessTotal - l.lastSucc) / dt
const alpha = 0.3 // exponential smoothing factor
l.drainRate = alpha*rate + (1-alpha)*l.drainRate
} else if s.SuccessTotal < l.lastSucc {
l.drainRate = 0
}
}
l.stats = s
l.lastSucc = s.SuccessTotal
l.scrapedAt = now
l.maxRunning = max(l.maxRunning, s.Running)
}
// Snapshots returns a view of every endpoint serving the given model, or of all
// endpoints when model is empty. Breaker state is read, never consumed.
func (r *Registry) Snapshots(model string) []Snapshot {
r.mu.RLock()
defer r.mu.RUnlock()
now := r.now()
out := make([]Snapshot, 0, len(r.m))
for _, l := range r.m {
if model != "" && !l.info.Serves(model) {
continue
}
var age time.Duration
stale := Scraped(l.info.Engine)
if !l.scrapedAt.IsZero() {
age = now.Sub(l.scrapedAt)
stale = age > r.cfg.StaleAfter
}
out = append(out, Snapshot{
Info: l.info,
Stats: l.stats,
DrainRate: l.drainRate,
TTFT: l.ttftNow(now),
Inflight: l.inflight,
RunningCapacity: l.maxRunning,
Stale: stale,
Age: age,
Breaker: l.brk.State(),
})
}
slices.SortFunc(out, func(a, b Snapshot) int { return strcmp(a.ID, b.ID) })
return out
}
// Acquire asks permission to send a request to an endpoint.
//
// It returns (nil, false) if the circuit breaker refuses. On success the
// inflight counter is incremented and the returned function must be called to
// report the outcome: release(err == nil).
func (r *Registry) Acquire(id string) (release func(ok bool), granted bool) {
r.mu.Lock()
l, exists := r.m[id]
r.mu.Unlock()
if !exists {
return nil, false
}
if !l.brk.Allow() {
return nil, false
}
r.mu.Lock()
l.inflight++
l.lastUsed = r.now()
r.mu.Unlock()
var once sync.Once
return func(ok bool) {
once.Do(func() {
r.mu.Lock()
l.inflight--
r.mu.Unlock()
if ok {
l.brk.Success()
} else {
l.brk.Failure()
}
})
}, true
}
// LastUsed reports when each endpoint last had a request dispatched to it.
//
// Endpoints that have never been used are absent rather than zero-valued: "not
// since this router started" and "not for an hour" are different facts, and a
// zero time would quietly become the oldest idle replica in any comparison —
// so a router restart would nominate everything for eviction at once.
func (r *Registry) LastUsed() map[string]time.Time {
r.mu.RLock()
defer r.mu.RUnlock()
out := make(map[string]time.Time, len(r.m))
for id, l := range r.m {
if l.lastUsed.IsZero() {
continue
}
out[id] = l.lastUsed
}
return out
}
// ScrapeFailed reports a failed metrics scrape.
//
// A scrape failure is not a request failure, so it does not touch the breaker:
// the endpoint may be serving fine while only its metrics endpoint is down.
// It will go stale on its own and score lower (§7.6).
func (r *Registry) ScrapeFailed(string) {}
// Inflight returns an endpoint's current inflight count, for tests and metrics.
func (r *Registry) Inflight(id string) int {
r.mu.RLock()
defer r.mu.RUnlock()
if l, ok := r.m[id]; ok {
return l.inflight
}
return 0
}
func strcmp(a, b string) int {
switch {
case a < b:
return -1
case a > b:
return 1
default:
return 0
}
}