// Package router wires the pieces into a working OpenAI-compatible gateway: // resolve the model, pick an endpoint, proxy the stream, retry when it is safe. // // The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy, // breaker); this package holds the orchestration and the HTTP surface. package router import ( "context" "errors" "fmt" "log/slog" "math" "net/http" "slices" "strings" "sync/atomic" "time" "github.com/prometheus/client_golang/prometheus" "gitea.mynodes.xyz/ilker/llmgw/internal/account" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" "gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // Router serves the OpenAI API in front of a set of engine replicas. type Router struct { cfg config.Router log *slog.Logger reg *endpoint.Registry poller *scrape.Poller px *proxy.Proxy m *metrics // adm is nil when admission control is disabled. adm *admission.Controller profiles map[string]*scheduler.Profile // models maps every name a client may use — canonical names and aliases — // onto its catalog entry. // // Held atomically because the catalog is no longer fixed at startup: adding // a model used to mean restarting the control plane, and this is the // router's half of not having to. A request takes the map once and works // from it, so a swap mid-request is invisible rather than a race. models atomic.Pointer[map[string]catalog.Model] // extraMetrics are collectors other components publish on /metrics. extraMetrics []prometheus.Collector // activator loads a model that is configured but not currently running. activator Activator // metricsHandler is built with the admin registry and reused by any // listener that should serve metrics without serving pprof. metricsHandler http.Handler // landing configures the public page, which the router serves but does not // otherwise care about. landing config.Landing // auth, store, acct, recorder and accounts are nil when authentication is // disabled. auth *auth.Authenticator accounts *account.Service store *storepkg.DB acct *accounting.Accountant // threads holds conversations and memory notes. It is store when auth is on, // its own file when only the threads block is configured, and nil when // nothing is kept. Separate from store because a non-nil store also switches // on the accounts, web pages and usage recording, none of which belong to a // gateway that only wants to remember things. threads *storepkg.DB // anonymous means threads has no project to scope to: everything shares the // one pool named by anonymousProject. anonymous bool // personalOff refuses personal use: the chat page then only offers the // projects a person is a member of. personalOff bool // personalDefaults is what personal use is held to when nobody wrote a // limit for the person, for the quota page to say so. personalDefaults *storepkg.Limits // llamaVersion is the llama.cpp build the nodes install, so the mirror page // can refuse weights that build cannot load before fetching them. llamaVersion string // ownsThreads is true when threads is not store and so must be closed here. ownsThreads bool recorder *accounting.Recorder // prefix is nil unless approximate prefix routing is turned on. prefix *scheduler.PrefixScorer // clusterView is nil until the control plane hands it over, and stays nil // in a router-only deployment. clusterView web.Source clusterUsage web.UsageSource // mirror is nil unless a model mirror is configured, which leaves // /admin/mirror unmounted — the right page for a deployment with no mirror // is no page, not an empty one pointing at nothing. mirror web.MirrorAdmin // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry // catalogAdmin puts a mirrored model into service. Nil without a store, // where a model added at runtime would not survive a restart. catalogAdmin web.CatalogAdmin // history is the metrics store the dashboard charts read from. Nil where // none is configured, which leaves the charts off the page. history web.History histWindow time.Duration } // SetClusterView gives the router the data the /admin page renders. // // Set by the server command once the control plane exists, because the router // is built first. Nil leaves /admin unmounted, which is what a router-only // deployment should have: no cluster to show. func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) { r.clusterView, r.clusterUsage = src, usage } // SetMirror gives the router a model mirror to administer. // // Set by the server command when one is configured, for the same reason as the // cluster view: the router is built before the things its admin pages show. func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m } // SetRegistry gives the mirror page somewhere to look models up. func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } // SetCatalogAdmin lets the mirror page put a model into service. func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c } // SetHistory gives the dashboard somewhere to read the cluster's past. func (r *Router) SetHistory(h web.History, window time.Duration) { r.history, r.histWindow = h, window } // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } // Store exposes the key store, or nil when authentication is off. func (r *Router) Store() *storepkg.DB { return r.store } // New builds a router from configuration. Start must be called to begin scraping. func New(cfg config.Server, log *slog.Logger) (*Router, error) { if log == nil { log = slog.Default() } reg := endpoint.NewRegistry(endpoint.Config{ StaleAfter: cfg.Router.StaleAfter, Breaker: breaker.Config{ FailureThreshold: cfg.Router.FailureThreshold, BaseCooldown: cfg.Router.Cooldown, }, }) reg.Replace(toInfos(cfg.Endpoints)) sc := scheduler.ProfileConfig{ TargetWaiting: cfg.Router.Scoring.TargetWaiting, TargetInflight: cfg.Router.Scoring.TargetInflight, KVCeiling: cfg.Router.Scoring.KVCeiling, SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor, } // The metrics come first because the prefix scorer reports into them, and // the decision metric is the whole difference between tuning this and // guessing at it. metrics := newMetrics() var prefixScorer *scheduler.PrefixScorer if cfg.Router.Prefix.Enabled { prefixScorer = scheduler.NewPrefixScorer( prefix.New(prefix.Config{ MaxNodes: cfg.Router.Prefix.MaxNodes, MinMatch: cfg.Router.Prefix.MinMatch, TTL: cfg.Router.Prefix.TTL, }), prefix.Thresholds{ MatchRatio: cfg.Router.Prefix.MatchRatio, BalanceAbs: cfg.Router.Prefix.BalanceAbs, BalanceRel: cfg.Router.Prefix.BalanceRel, }, metrics.observePrefix) sc.Prefix = prefixScorer } profiles := map[string]*scheduler.Profile{} for _, name := range []string{"chat", "batch", "long-context", "debug"} { profiles[name] = scheduler.Builtin(name, sc) } if _, ok := profiles[cfg.Router.DefaultProfile]; !ok { return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile) } r := &Router{ cfg: cfg.Router, landing: cfg.Landing, log: log, reg: reg, profiles: profiles, prefix: prefixScorer, m: metrics, px: proxy.New(proxy.Config{ DialTimeout: cfg.Router.Timeouts.Dial, ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader, GenerationTimeout: cfg.Router.Timeouts.Generation, FirstTokenTimeout: cfg.Router.Timeouts.FirstToken, IdleTimeout: cfg.Router.Timeouts.Idle, TotalTimeout: cfg.Router.Timeouts.Total, MaxResponseBytes: cfg.Router.MaxBodyBytes, }), } r.poller = scrape.NewPoller(reg, scrape.Config{ Interval: cfg.Router.ScrapeInterval, Timeout: cfg.Router.ScrapeTimeout, }, log) if cfg.Auth.Enabled { st, err := storepkg.Open(context.Background(), cfg.Auth.Store) if err != nil { return nil, fmt.Errorf("auth store: %w", err) } r.store = st // Applied where no scope set one, so a project created without limits // is still metered. Zero leaves it unlimited, which was the old // behaviour and is now a choice rather than an omission. var defRPM, defTPM *int if cfg.Auth.DefaultRPM > 0 { defRPM = &cfg.Auth.DefaultRPM } if cfg.Auth.DefaultTPM > 0 { defTPM = &cfg.Auth.DefaultTPM } st.SetDefaultLimits(defRPM, defTPM) st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal)) r.personalOff = cfg.Auth.Personal.Disabled r.personalDefaults = personalLimits(cfg.Auth.Personal) r.llamaVersion = cfg.WantedEngines()["llamacpp"] r.auth = auth.New(st, auth.Config{ TTL: cfg.Auth.CacheTTL, NegativeTTL: cfg.Auth.NegativeTTL, }) prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing)) for name, p := range cfg.Auth.Pricing { prices[name] = accounting.Price{ PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, } } r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{ FlushInterval: cfg.Auth.FlushInterval, BatchSize: cfg.Auth.BatchSize, Capacity: cfg.Auth.BufferSize, }, log) r.accounts = account.New(st, account.Config{ SignupOpen: cfg.Auth.SignupOpen, }, log) r.acct = accounting.New( accounting.NewPricing(prices, accounting.Price{ PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k, CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k, }), r.recorder, log) r.threads = r.store } else if cfg.Threads.Store != "" { st, err := storepkg.Open(context.Background(), cfg.Threads.Store) if err != nil { return nil, fmt.Errorf("threads store: %w", err) } if err := st.EnsureAnonymousProject(context.Background(), anonymousProject, cfg.Threads.RetainHours); err != nil { _ = st.Close() return nil, fmt.Errorf("threads store: %w", err) } r.threads, r.anonymous, r.ownsThreads = st, true, true } if cfg.Router.Admission.Enabled { adm, err := admission.New(admissionConfig(cfg.Router), r.saturation) if err != nil { return nil, fmt.Errorf("admission: %w", err) } r.adm = adm } // Seeded from configuration, not only pushed. A router running without a // control plane — a static endpoint list, which is a supported mode — has // nothing to push it a catalog, and would otherwise resolve no model at // all. A control plane pushes over the top of this at startup and on every // change after. if len(cfg.Models) > 0 { seed := map[string]catalog.Model{} for _, m := range cfg.Models { seed[m.Name] = m for _, a := range m.Aliases { seed[a] = m } } r.models.Store(&seed) } return r, nil } func admissionConfig(rc config.Router) admission.Config { bands := make([]admission.Band, 0, len(rc.Admission.Bands)) for _, b := range rc.Admission.Bands { bands = append(bands, admission.Band{ Name: b.Name, Priority: b.Priority, SaturationCeiling: b.SaturationCeiling, }) } return admission.Config{ Bands: bands, DefaultBand: rc.Admission.DefaultBand, Thresholds: admission.Thresholds{ // Admission and the queue scorer must agree on what a healthy // buffer is, or the two layers fight each other. TargetWaitingPerReplica: rc.Scoring.TargetWaiting, KVCeiling: rc.Scoring.KVCeiling, }, QueueTTL: rc.Admission.QueueTTL, QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand, Interval: rc.Admission.Interval, } } // saturation reports the pool pressure for a model, for the admission layer. // // outstanding is what admission has let through. It is compared against the // registry's inflight counters rather than the scraped engine stats, because // inflight updates the instant a request is dispatched while scraped stats lag // by a whole interval. Comparing against scraped data would make every // dispatched-but-unscraped request look like queue depth and throttle the // steady state down to a few requests per scrape. // // What is left over — admitted but not yet dispatched — is the thundering herd: // a burst that passed the check together before any of it became visible. func (r *Router) saturation(model string, outstanding int) (float64, bool) { eps := r.reg.Snapshots(model) if len(eps) == 0 { return 0, false } dispatched := 0 for _, e := range eps { dispatched += e.Inflight } return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{ TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting, KVCeiling: r.cfg.Scoring.KVCeiling, }), true } func toInfos(eps []config.Endpoint) []endpoint.Info { out := make([]endpoint.Info, 0, len(eps)) for _, e := range eps { out = append(out, endpoint.Info{ ID: e.ID, URL: strings.TrimRight(e.URL, "/"), Models: e.Models, Engine: e.Engine, Pool: e.Pool, Weight: e.Weight, Priority: e.Priority, }) } return out } // Start begins scraping endpoint metrics and dispatching admissions. // It returns immediately. func (r *Router) Start(ctx context.Context) { r.poller.Sync(ctx) if r.adm != nil { r.adm.Start() } if r.recorder != nil { r.recorder.Start(ctx) } if r.acct != nil && r.store != nil { // Budgets are counted in memory; the store is what makes them survive a // restart. A failure here is not fatal — refusing to start because last // month's spend could not be read would be worse than starting with // counters that are low. if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil { r.log.Warn("could not restore budgets; they start from zero this period", "err", err) } } if r.threads != nil { go r.pruneThreads(ctx) } } // pruneThreads enforces each project's retention window. // // Without something running this, "we keep conversations for 24 hours" is a // comment rather than a promise — the rows would simply accumulate, and the // number a project chose would describe nothing. // // Hourly, and once at startup. A gateway that has been down for a week should // not wait another hour before honouring a window that expired during it, and // an hour is fine granularity for a limit expressed in hours. func (r *Router) pruneThreads(ctx context.Context) { prune := func() { n, err := r.threads.PruneThreads(ctx, time.Now()) if err != nil { r.log.Warn("could not prune expired conversations", "err", err) return } if n > 0 { r.log.Info("removed conversations past their retention", "threads", n) } m, err := r.threads.PruneMemory(ctx, time.Now()) if err != nil { r.log.Warn("could not prune expired memory notes", "err", err) return } if m > 0 { r.log.Info("removed memory notes past their retention", "notes", m) } } prune() t := time.NewTicker(time.Hour) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: prune() } } } // Stop halts scraping and admission. func (r *Router) Stop() { r.poller.Stop() if r.adm != nil { r.adm.Close() } // Drain before closing the database, or the last batch has nowhere to go. if r.recorder != nil { r.recorder.Stop() } if r.store != nil { _ = r.store.Close() } if r.ownsThreads { _ = r.threads.Close() } } // Registry exposes the endpoint registry, for tests and admin endpoints. func (r *Router) Registry() *endpoint.Registry { return r.reg } // SetEndpoints replaces the endpoint list and resynchronizes metric scraping. // // This is how the control plane feeds the router from phase 3 onwards, taking // over from the static configuration list (plan §6.4). Endpoints that survive // the update keep their live state — inflight counts, breaker state, drain rate // — so a replica that was already serving does not start from cold just // because another one moved. // modelMap returns the name→model map in force. // // Never nil: a router built before any catalog arrived resolves nothing, which // is the same answer it gave for an unknown name before. func (r *Router) modelMap() map[string]catalog.Model { if m := r.models.Load(); m != nil { return *m } return nil } // SetCatalog replaces the models this router knows about. // // The catalog's counterpart to SetEndpoints, and pushed for the same reason: // the control plane owns what should exist, the router needs to know, and // neither has to be in the other's process for that to work. func (r *Router) SetCatalog(c *catalog.Catalog) { m := map[string]catalog.Model{} for _, entry := range c.Models() { m[entry.Name] = entry for _, a := range entry.Aliases { m[a] = entry } } r.models.Store(&m) } // LastUsed reports when each endpoint was last sent a request, satisfying // control.UsageSource so the control plane can tell a busy replica from one // holding its cards out of habit. func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() } func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) { if r.prefix != nil { // A replica that has gone takes its cache with it. Leaving its entries // in the tree would steer traffic towards something that is not there, // and since nothing reuses a replica id the entries would never be // corrected — only expire, minutes later. before := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { before[s.ID] = struct{}{} } for _, i := range infos { delete(before, i.ID) } for id := range before { r.prefix.Forget(id) } } r.reg.Replace(infos) r.poller.Sync(ctx) } // --- model resolution --- // resolve maps a client-supplied model name onto the catalog. Unknown names are // passed through unchanged: an endpoint may serve a model the catalog does not // list, and refusing it would be unhelpful. func (r *Router) resolve(name string) (canonical string, m catalog.Model) { if e, ok := r.modelMap()[name]; ok { return e.Name, e } return name, catalog.Model{Name: name} } func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile { for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} { if p, ok := r.profiles[name]; ok && name != "" { return p } } return r.profiles["chat"] } // --- dispatch --- // attempt describes one upstream try, for logging and metrics. type dispatchResult struct { endpointID string pool string servedModel outcome proxy.Outcome err error } type servedModel struct { model string // fallback is true when the served model is not the one that was requested. fallback bool } // dispatch resolves the model, then tries endpoints until one works, the client // has been written to, or the attempt budget runs out. // // The retry rule (plan §7.7) is strict: retry only while nothing has reached the // client. Once bytes are out, a retry would splice two different responses // together, which is worse than the error the client would otherwise see. func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult { canonical, entry := r.resolve(req.Model) // The requested model first, then its fallbacks. A fallback is a different // model, so the client is told about it via x-llmgw-served-model. chain := append([]string{canonical}, entry.Fallbacks...) var last dispatchResult for i, model := range chain { if i > 0 { // Before dispatching, because dispatchOne writes the response and // a header set after that is a header nobody receives. It was set // in finish until now, which is after the proxy has already called // WriteHeader — so this has announced a fallback to nobody since it // was written. w.Header().Set("x-llmgw-served-model", model) } res := r.dispatchOne(ctx, w, req, model, entry) res.model = model res.fallback = i > 0 if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) { return res } last = res // Only a total absence of capacity justifies moving to another model; // a bad request would fail identically there. if !errors.Is(res.err, errNoCapacity) { return res } } return last } var errNoCapacity = errors.New("no endpoint available") // errLoading means the model is being started and the wait ran out. // // Distinct from having no capacity because the right client behaviour differs: // this one succeeds on a retry, and the other does not. var errLoading = errors.New("model loading") func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult { candidates := r.reg.Snapshots(model) if len(candidates) == 0 { // Nothing is serving it. That may be because nothing ever will, or // because it loads when asked and nobody has asked yet — and the second // is not an error, it is the feature working. if r.activate(ctx, model) { candidates = r.reg.Snapshots(model) } } if len(candidates) == 0 { if r.activator != nil && r.activator.Want(model) { // Asked for, still loading. Saying "no endpoint serves this" would // be a lie, and a client told to retry will succeed where one told // the model does not exist gives up. return dispatchResult{err: fmt.Errorf("%w: %s", errLoading, "the model is loading; retry shortly")} } return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))} } profile := r.profileFor(entry, req.ProfileOverride) sctx := &scheduler.Ctx{ Now: time.Now(), Req: scheduler.Request{ Model: model, SessionID: req.SessionID, PromptPrefix: req.PromptPrefix, PromptTokens: req.PromptTokens, MaxTokens: req.MaxTokens, }, } attempts := max(r.cfg.Retries+1, 1) var last dispatchResult for range attempts { if len(candidates) == 0 { break } dec, err := profile.Schedule(sctx, candidates) if err != nil { last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)} break } release, granted := r.reg.Acquire(dec.Endpoint.ID) if !granted { // The breaker opened between scoring and dispatch; drop it and // score again rather than failing the request. candidates = without(candidates, dec.Endpoint.ID) continue } out, perr := r.px.Do(ctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, Body: req.UpstreamBody, Header: req.Header, Stream: req.Stream, SuppressUsageChunk: req.SuppressUsageChunk, CaptureReply: req.captureBytes(), }) // A retryable failure is the endpoint's fault; a client disconnect is not. release(perr == nil || errors.Is(perr, proxy.ErrClientGone)) res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr} r.m.observeScores(dec) // Recorded after the decision, not during it: the scorer proposes and // the picker disposes, so recording the proposal would fill the tree // with replicas that were never sent anything — the one thing that // would make the estimate worse than no estimate at all. if r.prefix != nil { r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID) } if perr == nil || out.Wrote || !out.Retryable { return res } last = res r.log.Warn("upstream attempt failed, trying another endpoint", "endpoint", dec.Endpoint.ID, "model", model, "err", perr) candidates = without(candidates, dec.Endpoint.ID) } if last.err == nil { last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model) } else if !errors.Is(last.err, errNoCapacity) { last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err) } return last } func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot { return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id }) } // personalLimits turns the personal block into limits, nil when it sets none. func personalLimits(p config.Personal) *storepkg.Limits { var l storepkg.Limits if p.RPM > 0 { v := p.RPM l.RPM = &v } if p.TPM > 0 { v := p.TPM l.TPM = &v } if p.Budget > 0 { v := int64(math.Round(p.Budget * 1e6)) l.Budget = &v l.BudgetPeriod = storepkg.PeriodMonth if p.Period == "day" { l.BudgetPeriod = storepkg.PeriodDay } } if l.RPM == nil && l.TPM == nil && l.Budget == nil { return nil } return &l }