llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
312 lines
8.2 KiB
Go
312 lines
8.2 KiB
Go
// Package endpoint tracks the upstream replicas the router can send to, along
|
|
// with their live state.
|
|
//
|
|
// In phase 1 the endpoint list comes from configuration. In phase 3 the control
|
|
// plane's EndpointSource (plan §6.4) will feed the same Registry; the scheduler
|
|
// and proxy must not notice the difference, which is why the API is shaped
|
|
// around wholesale list replacement from the start.
|
|
package endpoint
|
|
|
|
import (
|
|
"maps"
|
|
"slices"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
)
|
|
|
|
// Info is a replica's static description.
|
|
type Info struct {
|
|
// ID uniquely identifies the replica and labels its metrics.
|
|
ID string
|
|
// URL is the engine's root address (e.g. http://gpu-a-01:8000), no trailing slash.
|
|
URL string
|
|
// Models lists the model names this replica serves.
|
|
Models []string
|
|
// Engine names the inference engine serving here, which decides what its
|
|
// metrics are called. Empty means vLLM's names.
|
|
Engine string
|
|
// Pool names the (fabric, gpu_class) pool. Used for observability and pool
|
|
// preference only; phase 1 may well have a single pool.
|
|
Pool string
|
|
// Weight is the capacity weight measured by ctl bench (§6.2.1). 0 means 1.
|
|
Weight float64
|
|
// Priority expresses pool preference; higher pools are tried first.
|
|
Priority int
|
|
}
|
|
|
|
// Serves reports whether the replica serves the given model.
|
|
func (i Info) Serves(model string) bool {
|
|
return slices.Contains(i.Models, model)
|
|
}
|
|
|
|
// Stats is the router-side view of the engine metric contract (plan §7.6).
|
|
type Stats struct {
|
|
// The three mandatory fields.
|
|
Running int
|
|
Waiting int
|
|
KVCacheUsage float64
|
|
|
|
// Optional.
|
|
BlockSize int
|
|
NumGPUBlocks int
|
|
|
|
// Raw counters, kept so derived rates can be computed.
|
|
PromptTokens float64
|
|
GenerationTokens float64
|
|
SuccessTotal float64
|
|
}
|
|
|
|
// Snapshot is a complete view of one endpoint at scheduling time.
|
|
// It is a value type so the scheduler can work without holding a lock.
|
|
type Snapshot struct {
|
|
Info
|
|
Stats Stats
|
|
|
|
// DrainRate is the observed completion rate in requests/sec, exponentially
|
|
// smoothed. This is AIBrix's load-balance signal: Running alone cannot tell
|
|
// that a replica is slow.
|
|
DrainRate float64
|
|
|
|
// Inflight counts requests the router has dispatched here that have not
|
|
// finished. It compensates for scrape lag (§7.4); without it, herding is
|
|
// unavoidable within a 2s scrape interval.
|
|
Inflight int
|
|
|
|
// RunningCapacity is the highest concurrent request count ever observed on
|
|
// this endpoint — in practice the engine's batch capacity, learned rather
|
|
// than configured. Without it, requests the router has dispatched but the
|
|
// engine has not yet reported are indistinguishable from queue depth, and
|
|
// during a scrape interval that makes a perfectly healthy engine look
|
|
// overloaded.
|
|
RunningCapacity int
|
|
|
|
// Stale means no successful scrape arrived within StaleAfter. Stale
|
|
// endpoints are not dropped, only scored down.
|
|
Stale bool
|
|
// Age is the time since the last successful scrape.
|
|
Age time.Duration
|
|
// Breaker is the circuit breaker's current state, read without side effects.
|
|
Breaker breaker.State
|
|
}
|
|
|
|
// Config holds Registry settings.
|
|
type Config struct {
|
|
// StaleAfter is how long an endpoint may go without a successful scrape
|
|
// before being considered stale.
|
|
StaleAfter time.Duration
|
|
// Breaker holds per-endpoint circuit breaker settings.
|
|
Breaker breaker.Config
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
StaleAfter: 10 * time.Second,
|
|
Breaker: breaker.DefaultConfig(),
|
|
}
|
|
}
|
|
|
|
// live is one endpoint's mutable state.
|
|
type live struct {
|
|
info Info
|
|
|
|
stats Stats
|
|
scrapedAt time.Time
|
|
lastSucc float64
|
|
drainRate float64
|
|
maxRunning int
|
|
|
|
inflight int
|
|
brk *breaker.Breaker
|
|
}
|
|
|
|
// Registry holds the live state of all endpoints. It is safe for concurrent use.
|
|
type Registry struct {
|
|
cfg Config
|
|
now func() time.Time
|
|
|
|
mu sync.RWMutex
|
|
m map[string]*live
|
|
}
|
|
|
|
// NewRegistry creates an empty registry.
|
|
func NewRegistry(cfg Config) *Registry {
|
|
if cfg.StaleAfter <= 0 {
|
|
cfg.StaleAfter = 10 * time.Second
|
|
}
|
|
return &Registry{cfg: cfg, now: time.Now, m: make(map[string]*live)}
|
|
}
|
|
|
|
// SetClock replaces the time source, for tests.
|
|
func (r *Registry) SetClock(now func() time.Time) {
|
|
r.mu.Lock()
|
|
r.now = now
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
// Replace reconciles the endpoint list with the given set. Endpoints that
|
|
// remain keep their live state, new ones get a breaker, removed ones are dropped.
|
|
func (r *Registry) Replace(infos []Info) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
seen := make(map[string]struct{}, len(infos))
|
|
for _, in := range infos {
|
|
seen[in.ID] = struct{}{}
|
|
if l, ok := r.m[in.ID]; ok {
|
|
l.info = in
|
|
continue
|
|
}
|
|
r.m[in.ID] = &live{info: in, brk: breaker.New(r.cfg.Breaker)}
|
|
}
|
|
for id := range r.m {
|
|
if _, ok := seen[id]; !ok {
|
|
delete(r.m, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
// IDs returns the registered endpoint IDs in sorted order.
|
|
func (r *Registry) IDs() []string {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
return slices.Sorted(maps.Keys(r.m))
|
|
}
|
|
|
|
// Infos returns the registered endpoint descriptions, sorted by ID.
|
|
func (r *Registry) Infos() []Info {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
out := make([]Info, 0, len(r.m))
|
|
for _, l := range r.m {
|
|
out = append(out, l.info)
|
|
}
|
|
slices.SortFunc(out, func(a, b Info) int { return strcmp(a.ID, b.ID) })
|
|
return out
|
|
}
|
|
|
|
// Observe records a successful scrape and updates the drain rate.
|
|
func (r *Registry) Observe(id string, s Stats) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
l, ok := r.m[id]
|
|
if !ok {
|
|
return
|
|
}
|
|
now := r.now()
|
|
if !l.scrapedAt.IsZero() {
|
|
dt := now.Sub(l.scrapedAt).Seconds()
|
|
// A counter going backwards means the engine restarted; skip the derivative.
|
|
if dt > 0 && s.SuccessTotal >= l.lastSucc {
|
|
rate := (s.SuccessTotal - l.lastSucc) / dt
|
|
const alpha = 0.3 // exponential smoothing factor
|
|
l.drainRate = alpha*rate + (1-alpha)*l.drainRate
|
|
} else if s.SuccessTotal < l.lastSucc {
|
|
l.drainRate = 0
|
|
}
|
|
}
|
|
l.stats = s
|
|
l.lastSucc = s.SuccessTotal
|
|
l.scrapedAt = now
|
|
l.maxRunning = max(l.maxRunning, s.Running)
|
|
}
|
|
|
|
// Snapshots returns a view of every endpoint serving the given model, or of all
|
|
// endpoints when model is empty. Breaker state is read, never consumed.
|
|
func (r *Registry) Snapshots(model string) []Snapshot {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
|
|
now := r.now()
|
|
out := make([]Snapshot, 0, len(r.m))
|
|
for _, l := range r.m {
|
|
if model != "" && !l.info.Serves(model) {
|
|
continue
|
|
}
|
|
var age time.Duration
|
|
stale := true
|
|
if !l.scrapedAt.IsZero() {
|
|
age = now.Sub(l.scrapedAt)
|
|
stale = age > r.cfg.StaleAfter
|
|
}
|
|
out = append(out, Snapshot{
|
|
Info: l.info,
|
|
Stats: l.stats,
|
|
DrainRate: l.drainRate,
|
|
Inflight: l.inflight,
|
|
RunningCapacity: l.maxRunning,
|
|
Stale: stale,
|
|
Age: age,
|
|
Breaker: l.brk.State(),
|
|
})
|
|
}
|
|
slices.SortFunc(out, func(a, b Snapshot) int { return strcmp(a.ID, b.ID) })
|
|
return out
|
|
}
|
|
|
|
// Acquire asks permission to send a request to an endpoint.
|
|
//
|
|
// It returns (nil, false) if the circuit breaker refuses. On success the
|
|
// inflight counter is incremented and the returned function must be called to
|
|
// report the outcome: release(err == nil).
|
|
func (r *Registry) Acquire(id string) (release func(ok bool), granted bool) {
|
|
r.mu.Lock()
|
|
l, exists := r.m[id]
|
|
r.mu.Unlock()
|
|
if !exists {
|
|
return nil, false
|
|
}
|
|
if !l.brk.Allow() {
|
|
return nil, false
|
|
}
|
|
|
|
r.mu.Lock()
|
|
l.inflight++
|
|
r.mu.Unlock()
|
|
|
|
var once sync.Once
|
|
return func(ok bool) {
|
|
once.Do(func() {
|
|
r.mu.Lock()
|
|
l.inflight--
|
|
r.mu.Unlock()
|
|
if ok {
|
|
l.brk.Success()
|
|
} else {
|
|
l.brk.Failure()
|
|
}
|
|
})
|
|
}, true
|
|
}
|
|
|
|
// ScrapeFailed reports a failed metrics scrape.
|
|
//
|
|
// A scrape failure is not a request failure, so it does not touch the breaker:
|
|
// the endpoint may be serving fine while only its metrics endpoint is down.
|
|
// It will go stale on its own and score lower (§7.6).
|
|
func (r *Registry) ScrapeFailed(string) {}
|
|
|
|
// Inflight returns an endpoint's current inflight count, for tests and metrics.
|
|
func (r *Registry) Inflight(id string) int {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
if l, ok := r.m[id]; ok {
|
|
return l.inflight
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func strcmp(a, b string) int {
|
|
switch {
|
|
case a < b:
|
|
return -1
|
|
case a > b:
|
|
return 1
|
|
default:
|
|
return 0
|
|
}
|
|
}
|