response_header sounds like a check that the endpoint is alive, and for a streaming request it is: headers arrive before generation starts. For a non-streaming request the engine sends nothing until the completion is finished, so the same setting is the total generation budget wearing a liveness timeout's name. At 60s that was invisible on GPUs and a hard cap of roughly a hundred tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120 tokens failed at exactly one minute, and neither the setting's name nor the error said why. Two transports now: the short header timeout for streaming, where it still catches a sick endpoint, and a generation budget for non-streaming, defaulting to ten minutes. One number could not mean both things. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
500 lines
16 KiB
Go
500 lines
16 KiB
Go
// Package router wires the pieces into a working OpenAI-compatible gateway:
|
|
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
|
|
//
|
|
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
|
|
// breaker); this package holds the orchestration and the HTTP surface.
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// Router serves the OpenAI API in front of a set of engine replicas.
|
|
type Router struct {
|
|
cfg config.Router
|
|
log *slog.Logger
|
|
|
|
reg *endpoint.Registry
|
|
poller *scrape.Poller
|
|
px *proxy.Proxy
|
|
m *metrics
|
|
// adm is nil when admission control is disabled.
|
|
adm *admission.Controller
|
|
|
|
profiles map[string]*scheduler.Profile
|
|
// models maps every name a client may use — canonical names and aliases —
|
|
// onto its catalog entry.
|
|
models map[string]catalog.Model
|
|
|
|
// landing configures the public page, which the router serves but does not
|
|
// otherwise care about.
|
|
landing config.Landing
|
|
|
|
// auth, store, acct, recorder and accounts are nil when authentication is
|
|
// disabled.
|
|
auth *auth.Authenticator
|
|
accounts *account.Service
|
|
store *storepkg.SQLite
|
|
acct *accounting.Accountant
|
|
recorder *accounting.Recorder
|
|
|
|
// prefix is nil unless approximate prefix routing is turned on.
|
|
prefix *scheduler.PrefixScorer
|
|
|
|
// clusterView is nil until the control plane hands it over, and stays nil
|
|
// in a router-only deployment.
|
|
clusterView web.Source
|
|
clusterUsage web.UsageSource
|
|
}
|
|
|
|
// SetClusterView gives the router the data the /admin page renders.
|
|
//
|
|
// Set by the server command once the control plane exists, because the router
|
|
// is built first. Nil leaves /admin unmounted, which is what a router-only
|
|
// deployment should have: no cluster to show.
|
|
func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
|
|
r.clusterView, r.clusterUsage = src, usage
|
|
}
|
|
|
|
// Auth exposes the authenticator so ctl and the batch writer can reach it.
|
|
// Nil when authentication is off.
|
|
func (r *Router) Auth() *auth.Authenticator { return r.auth }
|
|
|
|
// Store exposes the key store, or nil when authentication is off.
|
|
func (r *Router) Store() *storepkg.SQLite { return r.store }
|
|
|
|
// New builds a router from configuration. Start must be called to begin scraping.
|
|
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
|
|
reg := endpoint.NewRegistry(endpoint.Config{
|
|
StaleAfter: cfg.Router.StaleAfter,
|
|
Breaker: breaker.Config{
|
|
FailureThreshold: cfg.Router.FailureThreshold,
|
|
BaseCooldown: cfg.Router.Cooldown,
|
|
},
|
|
})
|
|
reg.Replace(toInfos(cfg.Endpoints))
|
|
|
|
sc := scheduler.ProfileConfig{
|
|
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
|
|
TargetInflight: cfg.Router.Scoring.TargetInflight,
|
|
KVCeiling: cfg.Router.Scoring.KVCeiling,
|
|
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
|
|
}
|
|
// The metrics come first because the prefix scorer reports into them, and
|
|
// the decision metric is the whole difference between tuning this and
|
|
// guessing at it.
|
|
metrics := newMetrics()
|
|
var prefixScorer *scheduler.PrefixScorer
|
|
if cfg.Router.Prefix.Enabled {
|
|
prefixScorer = scheduler.NewPrefixScorer(
|
|
prefix.New(prefix.Config{
|
|
MaxNodes: cfg.Router.Prefix.MaxNodes,
|
|
MinMatch: cfg.Router.Prefix.MinMatch,
|
|
TTL: cfg.Router.Prefix.TTL,
|
|
}),
|
|
prefix.Thresholds{
|
|
MatchRatio: cfg.Router.Prefix.MatchRatio,
|
|
BalanceAbs: cfg.Router.Prefix.BalanceAbs,
|
|
BalanceRel: cfg.Router.Prefix.BalanceRel,
|
|
},
|
|
metrics.observePrefix)
|
|
sc.Prefix = prefixScorer
|
|
}
|
|
|
|
profiles := map[string]*scheduler.Profile{}
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
profiles[name] = scheduler.Builtin(name, sc)
|
|
}
|
|
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
|
|
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
|
|
}
|
|
|
|
models := map[string]catalog.Model{}
|
|
for _, m := range cfg.Models {
|
|
models[m.Name] = m
|
|
for _, a := range m.Aliases {
|
|
models[a] = m
|
|
}
|
|
}
|
|
|
|
r := &Router{
|
|
cfg: cfg.Router,
|
|
landing: cfg.Landing,
|
|
log: log,
|
|
reg: reg,
|
|
profiles: profiles,
|
|
models: models,
|
|
prefix: prefixScorer,
|
|
m: metrics,
|
|
px: proxy.New(proxy.Config{
|
|
DialTimeout: cfg.Router.Timeouts.Dial,
|
|
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
|
|
GenerationTimeout: cfg.Router.Timeouts.Generation,
|
|
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
|
|
IdleTimeout: cfg.Router.Timeouts.Idle,
|
|
TotalTimeout: cfg.Router.Timeouts.Total,
|
|
MaxResponseBytes: cfg.Router.MaxBodyBytes,
|
|
}),
|
|
}
|
|
r.poller = scrape.NewPoller(reg, scrape.Config{
|
|
Interval: cfg.Router.ScrapeInterval,
|
|
Timeout: cfg.Router.ScrapeTimeout,
|
|
}, log)
|
|
|
|
if cfg.Auth.Enabled {
|
|
st, err := storepkg.OpenSQLite(context.Background(), cfg.Auth.Store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("auth store: %w", err)
|
|
}
|
|
r.store = st
|
|
r.auth = auth.New(st, auth.Config{
|
|
TTL: cfg.Auth.CacheTTL,
|
|
NegativeTTL: cfg.Auth.NegativeTTL,
|
|
})
|
|
|
|
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
|
|
for name, p := range cfg.Auth.Pricing {
|
|
prices[name] = accounting.Price{
|
|
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k,
|
|
}
|
|
}
|
|
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
|
|
FlushInterval: cfg.Auth.FlushInterval,
|
|
BatchSize: cfg.Auth.BatchSize,
|
|
Capacity: cfg.Auth.BufferSize,
|
|
}, log)
|
|
r.accounts = account.New(st, account.Config{
|
|
SignupOpen: cfg.Auth.SignupOpen,
|
|
}, log)
|
|
|
|
r.acct = accounting.New(
|
|
accounting.NewPricing(prices, accounting.Price{
|
|
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
|
|
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
|
|
}),
|
|
r.recorder, log)
|
|
}
|
|
|
|
if cfg.Router.Admission.Enabled {
|
|
adm, err := admission.New(admissionConfig(cfg.Router), r.saturation)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("admission: %w", err)
|
|
}
|
|
r.adm = adm
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func admissionConfig(rc config.Router) admission.Config {
|
|
bands := make([]admission.Band, 0, len(rc.Admission.Bands))
|
|
for _, b := range rc.Admission.Bands {
|
|
bands = append(bands, admission.Band{
|
|
Name: b.Name,
|
|
Priority: b.Priority,
|
|
SaturationCeiling: b.SaturationCeiling,
|
|
})
|
|
}
|
|
return admission.Config{
|
|
Bands: bands,
|
|
DefaultBand: rc.Admission.DefaultBand,
|
|
Thresholds: admission.Thresholds{
|
|
// Admission and the queue scorer must agree on what a healthy
|
|
// buffer is, or the two layers fight each other.
|
|
TargetWaitingPerReplica: rc.Scoring.TargetWaiting,
|
|
KVCeiling: rc.Scoring.KVCeiling,
|
|
},
|
|
QueueTTL: rc.Admission.QueueTTL,
|
|
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
|
|
Interval: rc.Admission.Interval,
|
|
}
|
|
}
|
|
|
|
// saturation reports the pool pressure for a model, for the admission layer.
|
|
//
|
|
// outstanding is what admission has let through. It is compared against the
|
|
// registry's inflight counters rather than the scraped engine stats, because
|
|
// inflight updates the instant a request is dispatched while scraped stats lag
|
|
// by a whole interval. Comparing against scraped data would make every
|
|
// dispatched-but-unscraped request look like queue depth and throttle the
|
|
// steady state down to a few requests per scrape.
|
|
//
|
|
// What is left over — admitted but not yet dispatched — is the thundering herd:
|
|
// a burst that passed the check together before any of it became visible.
|
|
func (r *Router) saturation(model string, outstanding int) (float64, bool) {
|
|
eps := r.reg.Snapshots(model)
|
|
if len(eps) == 0 {
|
|
return 0, false
|
|
}
|
|
dispatched := 0
|
|
for _, e := range eps {
|
|
dispatched += e.Inflight
|
|
}
|
|
return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{
|
|
TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting,
|
|
KVCeiling: r.cfg.Scoring.KVCeiling,
|
|
}), true
|
|
}
|
|
|
|
func toInfos(eps []config.Endpoint) []endpoint.Info {
|
|
out := make([]endpoint.Info, 0, len(eps))
|
|
for _, e := range eps {
|
|
out = append(out, endpoint.Info{
|
|
ID: e.ID,
|
|
URL: strings.TrimRight(e.URL, "/"),
|
|
Models: e.Models,
|
|
Engine: e.Engine,
|
|
Pool: e.Pool,
|
|
Weight: e.Weight,
|
|
Priority: e.Priority,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Start begins scraping endpoint metrics and dispatching admissions.
|
|
// It returns immediately.
|
|
func (r *Router) Start(ctx context.Context) {
|
|
r.poller.Sync(ctx)
|
|
if r.adm != nil {
|
|
r.adm.Start()
|
|
}
|
|
if r.recorder != nil {
|
|
r.recorder.Start(ctx)
|
|
}
|
|
if r.acct != nil && r.store != nil {
|
|
// Budgets are counted in memory; the store is what makes them survive a
|
|
// restart. A failure here is not fatal — refusing to start because last
|
|
// month's spend could not be read would be worse than starting with
|
|
// counters that are low.
|
|
if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil {
|
|
r.log.Warn("could not restore budgets; they start from zero this period", "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop halts scraping and admission.
|
|
func (r *Router) Stop() {
|
|
r.poller.Stop()
|
|
if r.adm != nil {
|
|
r.adm.Close()
|
|
}
|
|
// Drain before closing the database, or the last batch has nowhere to go.
|
|
if r.recorder != nil {
|
|
r.recorder.Stop()
|
|
}
|
|
if r.store != nil {
|
|
_ = r.store.Close()
|
|
}
|
|
}
|
|
|
|
// Registry exposes the endpoint registry, for tests and admin endpoints.
|
|
func (r *Router) Registry() *endpoint.Registry { return r.reg }
|
|
|
|
// SetEndpoints replaces the endpoint list and resynchronizes metric scraping.
|
|
//
|
|
// This is how the control plane feeds the router from phase 3 onwards, taking
|
|
// over from the static configuration list (plan §6.4). Endpoints that survive
|
|
// the update keep their live state — inflight counts, breaker state, drain rate
|
|
// — so a replica that was already serving does not start from cold just
|
|
// because another one moved.
|
|
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
|
|
if r.prefix != nil {
|
|
// A replica that has gone takes its cache with it. Leaving its entries
|
|
// in the tree would steer traffic towards something that is not there,
|
|
// and since nothing reuses a replica id the entries would never be
|
|
// corrected — only expire, minutes later.
|
|
before := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
before[s.ID] = struct{}{}
|
|
}
|
|
for _, i := range infos {
|
|
delete(before, i.ID)
|
|
}
|
|
for id := range before {
|
|
r.prefix.Forget(id)
|
|
}
|
|
}
|
|
|
|
r.reg.Replace(infos)
|
|
r.poller.Sync(ctx)
|
|
}
|
|
|
|
// --- model resolution ---
|
|
|
|
// resolve maps a client-supplied model name onto the catalog. Unknown names are
|
|
// passed through unchanged: an endpoint may serve a model the catalog does not
|
|
// list, and refusing it would be unhelpful.
|
|
func (r *Router) resolve(name string) (canonical string, m catalog.Model) {
|
|
if e, ok := r.models[name]; ok {
|
|
return e.Name, e
|
|
}
|
|
return name, catalog.Model{Name: name}
|
|
}
|
|
|
|
func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile {
|
|
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
|
|
if p, ok := r.profiles[name]; ok && name != "" {
|
|
return p
|
|
}
|
|
}
|
|
return r.profiles["chat"]
|
|
}
|
|
|
|
// --- dispatch ---
|
|
|
|
// attempt describes one upstream try, for logging and metrics.
|
|
type dispatchResult struct {
|
|
endpointID string
|
|
pool string
|
|
servedModel
|
|
outcome proxy.Outcome
|
|
err error
|
|
}
|
|
|
|
type servedModel struct {
|
|
model string
|
|
// fallback is true when the served model is not the one that was requested.
|
|
fallback bool
|
|
}
|
|
|
|
// dispatch resolves the model, then tries endpoints until one works, the client
|
|
// has been written to, or the attempt budget runs out.
|
|
//
|
|
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
|
|
// client. Once bytes are out, a retry would splice two different responses
|
|
// together, which is worse than the error the client would otherwise see.
|
|
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
|
|
canonical, entry := r.resolve(req.Model)
|
|
|
|
// The requested model first, then its fallbacks. A fallback is a different
|
|
// model, so the client is told about it via x-llmgw-served-model.
|
|
chain := append([]string{canonical}, entry.Fallbacks...)
|
|
|
|
var last dispatchResult
|
|
for i, model := range chain {
|
|
res := r.dispatchOne(ctx, w, req, model, entry)
|
|
res.model = model
|
|
res.fallback = i > 0
|
|
|
|
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
|
|
return res
|
|
}
|
|
last = res
|
|
// Only a total absence of capacity justifies moving to another model;
|
|
// a bad request would fail identically there.
|
|
if !errors.Is(res.err, errNoCapacity) {
|
|
return res
|
|
}
|
|
}
|
|
return last
|
|
}
|
|
|
|
var errNoCapacity = errors.New("no endpoint available")
|
|
|
|
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
|
|
candidates := r.reg.Snapshots(model)
|
|
if len(candidates) == 0 {
|
|
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
|
|
}
|
|
|
|
profile := r.profileFor(entry, req.ProfileOverride)
|
|
sctx := &scheduler.Ctx{
|
|
Now: time.Now(),
|
|
Req: scheduler.Request{
|
|
Model: model,
|
|
SessionID: req.SessionID,
|
|
PromptPrefix: req.PromptPrefix,
|
|
PromptTokens: req.PromptTokens,
|
|
MaxTokens: req.MaxTokens,
|
|
},
|
|
}
|
|
|
|
attempts := max(r.cfg.Retries+1, 1)
|
|
var last dispatchResult
|
|
|
|
for range attempts {
|
|
if len(candidates) == 0 {
|
|
break
|
|
}
|
|
dec, err := profile.Schedule(sctx, candidates)
|
|
if err != nil {
|
|
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
|
|
break
|
|
}
|
|
|
|
release, granted := r.reg.Acquire(dec.Endpoint.ID)
|
|
if !granted {
|
|
// The breaker opened between scoring and dispatch; drop it and
|
|
// score again rather than failing the request.
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
continue
|
|
}
|
|
|
|
out, perr := r.px.Do(ctx, w, proxy.Request{
|
|
URL: dec.Endpoint.URL + req.Path,
|
|
Body: req.UpstreamBody,
|
|
Header: req.Header,
|
|
Stream: req.Stream,
|
|
SuppressUsageChunk: req.SuppressUsageChunk,
|
|
})
|
|
// A retryable failure is the endpoint's fault; a client disconnect is not.
|
|
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
|
|
|
|
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
|
|
r.m.observeScores(dec)
|
|
|
|
// Recorded after the decision, not during it: the scorer proposes and
|
|
// the picker disposes, so recording the proposal would fill the tree
|
|
// with replicas that were never sent anything — the one thing that
|
|
// would make the estimate worse than no estimate at all.
|
|
if r.prefix != nil {
|
|
r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID)
|
|
}
|
|
|
|
if perr == nil || out.Wrote || !out.Retryable {
|
|
return res
|
|
}
|
|
last = res
|
|
r.log.Warn("upstream attempt failed, trying another endpoint",
|
|
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
}
|
|
|
|
if last.err == nil {
|
|
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
|
|
} else if !errors.Is(last.err, errNoCapacity) {
|
|
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
|
|
}
|
|
return last
|
|
}
|
|
|
|
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
|
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
|
}
|