Files
llmgw/internal/router/router.go
T
Ilker ManapandClaude Opus 5 1c57a4e378
CI / test (push) Successful in 1m29s
Let the catalog change while the cluster is running
Adding a model was an operator procedure with downtime: edit a file on the
control plane, restart it, wait out the settling window, watch replicas
churn while adoption catches up. I did that eight times in a day, and one
of the restarts cost two minutes of outage while stalled replicas timed
out. None of it was necessary. The reconciler already compares desired
against actual on every pass, so a new catalog is simply a different
desired.

A Catalog stays immutable — New is the only way to build one and it refuses
an invalid one — and what changes is which immutable catalog is current.
Readers take the pointer and finish their operation against it, so a swap
mid-request is invisible rather than a race. The router is told rather than
asked, the same shape as the endpoint list and for the same reason: a push
happens once per change, a pull would happen once per request.

The router still seeds from configuration as well as being pushed to. A
router with a static endpoint list and no control plane is a supported mode
and has nothing to push it a catalog; without seeding it would resolve no
model at all.

Which turned up a bug that was latent while catalogs were fixed.
reconcileModel only ever visits models that are in the catalog, so a
replica whose model had been removed was visited by nothing: it kept its
GPUs, kept counting, and kept running an engine for a name no client could
ask for. The endpoint publisher skipped such a replica with the comment
"the reconciler will tear it down", which was not true. It is now, and the
comment says what actually happens instead of what was assumed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-17 01:08:21 +02:00

557 lines
18 KiB
Go

// Package router wires the pieces into a working OpenAI-compatible gateway:
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
//
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
// breaker); this package holds the orchestration and the HTTP surface.
package router
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"slices"
"strings"
"sync/atomic"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// Router serves the OpenAI API in front of a set of engine replicas.
type Router struct {
cfg config.Router
log *slog.Logger
reg *endpoint.Registry
poller *scrape.Poller
px *proxy.Proxy
m *metrics
// adm is nil when admission control is disabled.
adm *admission.Controller
profiles map[string]*scheduler.Profile
// models maps every name a client may use — canonical names and aliases —
// onto its catalog entry.
//
// Held atomically because the catalog is no longer fixed at startup: adding
// a model used to mean restarting the control plane, and this is the
// router's half of not having to. A request takes the map once and works
// from it, so a swap mid-request is invisible rather than a race.
models atomic.Pointer[map[string]catalog.Model]
// landing configures the public page, which the router serves but does not
// otherwise care about.
landing config.Landing
// auth, store, acct, recorder and accounts are nil when authentication is
// disabled.
auth *auth.Authenticator
accounts *account.Service
store *storepkg.SQLite
acct *accounting.Accountant
recorder *accounting.Recorder
// prefix is nil unless approximate prefix routing is turned on.
prefix *scheduler.PrefixScorer
// clusterView is nil until the control plane hands it over, and stays nil
// in a router-only deployment.
clusterView web.Source
clusterUsage web.UsageSource
}
// SetClusterView gives the router the data the /admin page renders.
//
// Set by the server command once the control plane exists, because the router
// is built first. Nil leaves /admin unmounted, which is what a router-only
// deployment should have: no cluster to show.
func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
r.clusterView, r.clusterUsage = src, usage
}
// Auth exposes the authenticator so ctl and the batch writer can reach it.
// Nil when authentication is off.
func (r *Router) Auth() *auth.Authenticator { return r.auth }
// Store exposes the key store, or nil when authentication is off.
func (r *Router) Store() *storepkg.SQLite { return r.store }
// New builds a router from configuration. Start must be called to begin scraping.
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
if log == nil {
log = slog.Default()
}
reg := endpoint.NewRegistry(endpoint.Config{
StaleAfter: cfg.Router.StaleAfter,
Breaker: breaker.Config{
FailureThreshold: cfg.Router.FailureThreshold,
BaseCooldown: cfg.Router.Cooldown,
},
})
reg.Replace(toInfos(cfg.Endpoints))
sc := scheduler.ProfileConfig{
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
TargetInflight: cfg.Router.Scoring.TargetInflight,
KVCeiling: cfg.Router.Scoring.KVCeiling,
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
}
// The metrics come first because the prefix scorer reports into them, and
// the decision metric is the whole difference between tuning this and
// guessing at it.
metrics := newMetrics()
var prefixScorer *scheduler.PrefixScorer
if cfg.Router.Prefix.Enabled {
prefixScorer = scheduler.NewPrefixScorer(
prefix.New(prefix.Config{
MaxNodes: cfg.Router.Prefix.MaxNodes,
MinMatch: cfg.Router.Prefix.MinMatch,
TTL: cfg.Router.Prefix.TTL,
}),
prefix.Thresholds{
MatchRatio: cfg.Router.Prefix.MatchRatio,
BalanceAbs: cfg.Router.Prefix.BalanceAbs,
BalanceRel: cfg.Router.Prefix.BalanceRel,
},
metrics.observePrefix)
sc.Prefix = prefixScorer
}
profiles := map[string]*scheduler.Profile{}
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
profiles[name] = scheduler.Builtin(name, sc)
}
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
}
r := &Router{
cfg: cfg.Router,
landing: cfg.Landing,
log: log,
reg: reg,
profiles: profiles,
prefix: prefixScorer,
m: metrics,
px: proxy.New(proxy.Config{
DialTimeout: cfg.Router.Timeouts.Dial,
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
GenerationTimeout: cfg.Router.Timeouts.Generation,
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
IdleTimeout: cfg.Router.Timeouts.Idle,
TotalTimeout: cfg.Router.Timeouts.Total,
MaxResponseBytes: cfg.Router.MaxBodyBytes,
}),
}
r.poller = scrape.NewPoller(reg, scrape.Config{
Interval: cfg.Router.ScrapeInterval,
Timeout: cfg.Router.ScrapeTimeout,
}, log)
if cfg.Auth.Enabled {
st, err := storepkg.OpenSQLite(context.Background(), cfg.Auth.Store)
if err != nil {
return nil, fmt.Errorf("auth store: %w", err)
}
r.store = st
// Applied where no scope set one, so a project created without limits
// is still metered. Zero leaves it unlimited, which was the old
// behaviour and is now a choice rather than an omission.
var defRPM, defTPM *int
if cfg.Auth.DefaultRPM > 0 {
defRPM = &cfg.Auth.DefaultRPM
}
if cfg.Auth.DefaultTPM > 0 {
defTPM = &cfg.Auth.DefaultTPM
}
st.SetDefaultLimits(defRPM, defTPM)
r.auth = auth.New(st, auth.Config{
TTL: cfg.Auth.CacheTTL,
NegativeTTL: cfg.Auth.NegativeTTL,
})
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
for name, p := range cfg.Auth.Pricing {
prices[name] = accounting.Price{
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k,
}
}
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
FlushInterval: cfg.Auth.FlushInterval,
BatchSize: cfg.Auth.BatchSize,
Capacity: cfg.Auth.BufferSize,
}, log)
r.accounts = account.New(st, account.Config{
SignupOpen: cfg.Auth.SignupOpen,
}, log)
r.acct = accounting.New(
accounting.NewPricing(prices, accounting.Price{
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
}),
r.recorder, log)
}
if cfg.Router.Admission.Enabled {
adm, err := admission.New(admissionConfig(cfg.Router), r.saturation)
if err != nil {
return nil, fmt.Errorf("admission: %w", err)
}
r.adm = adm
}
// Seeded from configuration, not only pushed. A router running without a
// control plane — a static endpoint list, which is a supported mode — has
// nothing to push it a catalog, and would otherwise resolve no model at
// all. A control plane pushes over the top of this at startup and on every
// change after.
if len(cfg.Models) > 0 {
seed := map[string]catalog.Model{}
for _, m := range cfg.Models {
seed[m.Name] = m
for _, a := range m.Aliases {
seed[a] = m
}
}
r.models.Store(&seed)
}
return r, nil
}
func admissionConfig(rc config.Router) admission.Config {
bands := make([]admission.Band, 0, len(rc.Admission.Bands))
for _, b := range rc.Admission.Bands {
bands = append(bands, admission.Band{
Name: b.Name,
Priority: b.Priority,
SaturationCeiling: b.SaturationCeiling,
})
}
return admission.Config{
Bands: bands,
DefaultBand: rc.Admission.DefaultBand,
Thresholds: admission.Thresholds{
// Admission and the queue scorer must agree on what a healthy
// buffer is, or the two layers fight each other.
TargetWaitingPerReplica: rc.Scoring.TargetWaiting,
KVCeiling: rc.Scoring.KVCeiling,
},
QueueTTL: rc.Admission.QueueTTL,
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
Interval: rc.Admission.Interval,
}
}
// saturation reports the pool pressure for a model, for the admission layer.
//
// outstanding is what admission has let through. It is compared against the
// registry's inflight counters rather than the scraped engine stats, because
// inflight updates the instant a request is dispatched while scraped stats lag
// by a whole interval. Comparing against scraped data would make every
// dispatched-but-unscraped request look like queue depth and throttle the
// steady state down to a few requests per scrape.
//
// What is left over — admitted but not yet dispatched — is the thundering herd:
// a burst that passed the check together before any of it became visible.
func (r *Router) saturation(model string, outstanding int) (float64, bool) {
eps := r.reg.Snapshots(model)
if len(eps) == 0 {
return 0, false
}
dispatched := 0
for _, e := range eps {
dispatched += e.Inflight
}
return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{
TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting,
KVCeiling: r.cfg.Scoring.KVCeiling,
}), true
}
func toInfos(eps []config.Endpoint) []endpoint.Info {
out := make([]endpoint.Info, 0, len(eps))
for _, e := range eps {
out = append(out, endpoint.Info{
ID: e.ID,
URL: strings.TrimRight(e.URL, "/"),
Models: e.Models,
Engine: e.Engine,
Pool: e.Pool,
Weight: e.Weight,
Priority: e.Priority,
})
}
return out
}
// Start begins scraping endpoint metrics and dispatching admissions.
// It returns immediately.
func (r *Router) Start(ctx context.Context) {
r.poller.Sync(ctx)
if r.adm != nil {
r.adm.Start()
}
if r.recorder != nil {
r.recorder.Start(ctx)
}
if r.acct != nil && r.store != nil {
// Budgets are counted in memory; the store is what makes them survive a
// restart. A failure here is not fatal — refusing to start because last
// month's spend could not be read would be worse than starting with
// counters that are low.
if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil {
r.log.Warn("could not restore budgets; they start from zero this period", "err", err)
}
}
}
// Stop halts scraping and admission.
func (r *Router) Stop() {
r.poller.Stop()
if r.adm != nil {
r.adm.Close()
}
// Drain before closing the database, or the last batch has nowhere to go.
if r.recorder != nil {
r.recorder.Stop()
}
if r.store != nil {
_ = r.store.Close()
}
}
// Registry exposes the endpoint registry, for tests and admin endpoints.
func (r *Router) Registry() *endpoint.Registry { return r.reg }
// SetEndpoints replaces the endpoint list and resynchronizes metric scraping.
//
// This is how the control plane feeds the router from phase 3 onwards, taking
// over from the static configuration list (plan §6.4). Endpoints that survive
// the update keep their live state — inflight counts, breaker state, drain rate
// — so a replica that was already serving does not start from cold just
// because another one moved.
// modelMap returns the name→model map in force.
//
// Never nil: a router built before any catalog arrived resolves nothing, which
// is the same answer it gave for an unknown name before.
func (r *Router) modelMap() map[string]catalog.Model {
if m := r.models.Load(); m != nil {
return *m
}
return nil
}
// SetCatalog replaces the models this router knows about.
//
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
// the control plane owns what should exist, the router needs to know, and
// neither has to be in the other's process for that to work.
func (r *Router) SetCatalog(c *catalog.Catalog) {
m := map[string]catalog.Model{}
for _, entry := range c.Models() {
m[entry.Name] = entry
for _, a := range entry.Aliases {
m[a] = entry
}
}
r.models.Store(&m)
}
// LastUsed reports when each endpoint was last sent a request, satisfying
// control.UsageSource so the control plane can tell a busy replica from one
// holding its cards out of habit.
func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() }
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
if r.prefix != nil {
// A replica that has gone takes its cache with it. Leaving its entries
// in the tree would steer traffic towards something that is not there,
// and since nothing reuses a replica id the entries would never be
// corrected — only expire, minutes later.
before := map[string]struct{}{}
for _, s := range r.reg.Snapshots("") {
before[s.ID] = struct{}{}
}
for _, i := range infos {
delete(before, i.ID)
}
for id := range before {
r.prefix.Forget(id)
}
}
r.reg.Replace(infos)
r.poller.Sync(ctx)
}
// --- model resolution ---
// resolve maps a client-supplied model name onto the catalog. Unknown names are
// passed through unchanged: an endpoint may serve a model the catalog does not
// list, and refusing it would be unhelpful.
func (r *Router) resolve(name string) (canonical string, m catalog.Model) {
if e, ok := r.modelMap()[name]; ok {
return e.Name, e
}
return name, catalog.Model{Name: name}
}
func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile {
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
if p, ok := r.profiles[name]; ok && name != "" {
return p
}
}
return r.profiles["chat"]
}
// --- dispatch ---
// attempt describes one upstream try, for logging and metrics.
type dispatchResult struct {
endpointID string
pool string
servedModel
outcome proxy.Outcome
err error
}
type servedModel struct {
model string
// fallback is true when the served model is not the one that was requested.
fallback bool
}
// dispatch resolves the model, then tries endpoints until one works, the client
// has been written to, or the attempt budget runs out.
//
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
// client. Once bytes are out, a retry would splice two different responses
// together, which is worse than the error the client would otherwise see.
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
canonical, entry := r.resolve(req.Model)
// The requested model first, then its fallbacks. A fallback is a different
// model, so the client is told about it via x-llmgw-served-model.
chain := append([]string{canonical}, entry.Fallbacks...)
var last dispatchResult
for i, model := range chain {
res := r.dispatchOne(ctx, w, req, model, entry)
res.model = model
res.fallback = i > 0
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
return res
}
last = res
// Only a total absence of capacity justifies moving to another model;
// a bad request would fail identically there.
if !errors.Is(res.err, errNoCapacity) {
return res
}
}
return last
}
var errNoCapacity = errors.New("no endpoint available")
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
candidates := r.reg.Snapshots(model)
if len(candidates) == 0 {
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
}
profile := r.profileFor(entry, req.ProfileOverride)
sctx := &scheduler.Ctx{
Now: time.Now(),
Req: scheduler.Request{
Model: model,
SessionID: req.SessionID,
PromptPrefix: req.PromptPrefix,
PromptTokens: req.PromptTokens,
MaxTokens: req.MaxTokens,
},
}
attempts := max(r.cfg.Retries+1, 1)
var last dispatchResult
for range attempts {
if len(candidates) == 0 {
break
}
dec, err := profile.Schedule(sctx, candidates)
if err != nil {
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
break
}
release, granted := r.reg.Acquire(dec.Endpoint.ID)
if !granted {
// The breaker opened between scoring and dispatch; drop it and
// score again rather than failing the request.
candidates = without(candidates, dec.Endpoint.ID)
continue
}
out, perr := r.px.Do(ctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
Body: req.UpstreamBody,
Header: req.Header,
Stream: req.Stream,
SuppressUsageChunk: req.SuppressUsageChunk,
})
// A retryable failure is the endpoint's fault; a client disconnect is not.
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
r.m.observeScores(dec)
// Recorded after the decision, not during it: the scorer proposes and
// the picker disposes, so recording the proposal would fill the tree
// with replicas that were never sent anything — the one thing that
// would make the estimate worse than no estimate at all.
if r.prefix != nil {
r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID)
}
if perr == nil || out.Wrote || !out.Retryable {
return res
}
last = res
r.log.Warn("upstream attempt failed, trying another endpoint",
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
candidates = without(candidates, dec.Endpoint.ID)
}
if last.err == nil {
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
} else if !errors.Is(last.err, errNoCapacity) {
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
}
return last
}
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
}