// Package router wires the pieces into a working OpenAI-compatible gateway: // resolve the model, pick an endpoint, proxy the stream, retry when it is safe. // // The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy, // breaker); this package holds the orchestration and the HTTP surface. package router import ( "context" "errors" "fmt" "log/slog" "math" "net" "net/http" "net/netip" "slices" "strings" "sync/atomic" "time" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "gitea.mynodes.xyz/ilker/llmgw/internal/account" "gitea.mynodes.xyz/ilker/llmgw/internal/accounting" "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/admission" "gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker" "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" "gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix" "gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler" "gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/web" ) // Router serves the OpenAI API in front of a set of engine replicas. type Router struct { cfg config.Router log *slog.Logger reg *endpoint.Registry poller *scrape.Poller px *proxy.Proxy m *metrics // adm is nil when admission control is disabled. adm *admission.Controller profiles map[string]*scheduler.Profile // models maps every name a client may use — canonical names and aliases — // onto its catalog entry. // // Held atomically because the catalog is no longer fixed at startup: adding // a model used to mean restarting the control plane, and this is the // router's half of not having to. A request takes the map once and works // from it, so a swap mid-request is invisible rather than a race. models atomic.Pointer[map[string]catalog.Model] // adapters are the LoRA adapters by the name clients ask for them by. adapters atomic.Pointer[map[string]adapterRef] // extraMetrics are collectors other components publish on /metrics. extraMetrics []prometheus.Collector // activator loads a model that is configured but not currently running. activator Activator // metricsHandler is built with the admin registry and reused by any // listener that should serve metrics without serving pprof. metricsHandler http.Handler // landing configures the public page, which the router serves but does not // otherwise care about. landing config.Landing // auth, store, acct, recorder and accounts are nil when authentication is // disabled. auth *auth.Authenticator accounts *account.Service store *storepkg.DB // usageRetention is how long request records are kept; zero is for ever. usageRetention time.Duration // isLeader says whether this process leads; nil is always. isLeader atomic.Pointer[func() bool] acct *accounting.Accountant // threads holds conversations and memory notes. It is store when auth is on, // its own file when only the threads block is configured, and nil when // nothing is kept. Separate from store because a non-nil store also switches // on the accounts, web pages and usage recording, none of which belong to a // gateway that only wants to remember things. threads *storepkg.DB // anonymous means threads has no project to scope to: everything shares the // one pool named by anonymousProject. anonymous bool // personalOff refuses personal use: the chat page then only offers the // projects a person is a member of. personalOff bool // personalDefaults is what personal use is held to when nobody wrote a // limit for the person, for the quota page to say so. personalDefaults *storepkg.Limits // llamaVersion is the llama.cpp build the nodes install, so the mirror page // can refuse weights that build cannot load before fetching them. llamaVersion string // ownsThreads is true when threads is not store and so must be closed here. ownsThreads bool recorder *accounting.Recorder // prefix is nil unless approximate prefix routing is turned on. prefix *scheduler.PrefixScorer // clusterView is nil until the control plane hands it over, and stays nil // in a router-only deployment. clusterView web.Source clusterUsage web.UsageSource // mirror is nil unless a model mirror is configured, which leaves // /admin/mirror unmounted — the right page for a deployment with no mirror // is no page, not an empty one pointing at nothing. mirror web.MirrorAdmin // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry // catalogAdmin puts a mirrored model into service. Nil without a store, // where a model added at runtime would not survive a restart. catalogAdmin web.CatalogAdmin // history is the metrics store the dashboard charts read from. Nil where // none is configured, which leaves the charts off the page. history web.History histWindow time.Duration } // SetClusterView gives the router the data the /admin page renders. // // Set by the server command once the control plane exists, because the router // is built first. Nil leaves /admin unmounted, which is what a router-only // deployment should have: no cluster to show. func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) { r.clusterView, r.clusterUsage = src, usage } // SetMirror gives the router a model mirror to administer. // // Set by the server command when one is configured, for the same reason as the // cluster view: the router is built before the things its admin pages show. func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m } // SetRegistry gives the mirror page somewhere to look models up. func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg } // SetCatalogAdmin lets the mirror page put a model into service. func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c } // SetHistory gives the dashboard somewhere to read the cluster's past. func (r *Router) SetHistory(h web.History, window time.Duration) { r.history, r.histWindow = h, window } // Auth exposes the authenticator so ctl and the batch writer can reach it. // Nil when authentication is off. func (r *Router) Auth() *auth.Authenticator { return r.auth } // Store exposes the key store, or nil when authentication is off. func (r *Router) Store() *storepkg.DB { return r.store } // New builds a router from configuration. Start must be called to begin scraping. func New(cfg config.Server, log *slog.Logger) (*Router, error) { if log == nil { log = slog.Default() } reg := endpoint.NewRegistry(endpoint.Config{ StaleAfter: cfg.Router.StaleAfter, Breaker: breaker.Config{ FailureThreshold: cfg.Router.FailureThreshold, BaseCooldown: cfg.Router.Cooldown, }, }) reg.Replace(toInfos(cfg.Endpoints)) sc := scheduler.ProfileConfig{ TargetWaiting: cfg.Router.Scoring.TargetWaiting, TargetInflight: cfg.Router.Scoring.TargetInflight, KVCeiling: cfg.Router.Scoring.KVCeiling, SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor, } // The metrics come first because the prefix scorer reports into them, and // the decision metric is the whole difference between tuning this and // guessing at it. metrics := newMetrics() metrics.tenant.level = cfg.Router.TenantLabel var prefixScorer *scheduler.PrefixScorer if cfg.Router.Prefix.Enabled { prefixScorer = scheduler.NewPrefixScorer( prefix.New(prefix.Config{ MaxNodes: cfg.Router.Prefix.MaxNodes, MinMatch: cfg.Router.Prefix.MinMatch, TTL: cfg.Router.Prefix.TTL, }), prefix.Thresholds{ MatchRatio: cfg.Router.Prefix.MatchRatio, BalanceAbs: cfg.Router.Prefix.BalanceAbs, BalanceRel: cfg.Router.Prefix.BalanceRel, }, metrics.observePrefix) sc.Prefix = prefixScorer } profiles := map[string]*scheduler.Profile{} for _, name := range []string{"chat", "batch", "long-context", "debug"} { profiles[name] = scheduler.Builtin(name, sc) } if _, ok := profiles[cfg.Router.DefaultProfile]; !ok { return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile) } r := &Router{ cfg: cfg.Router, landing: cfg.Landing, log: log, reg: reg, profiles: profiles, prefix: prefixScorer, m: metrics, px: proxy.New(proxy.Config{ DialTimeout: cfg.Router.Timeouts.Dial, ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader, GenerationTimeout: cfg.Router.Timeouts.Generation, FirstTokenTimeout: cfg.Router.Timeouts.FirstToken, IdleTimeout: cfg.Router.Timeouts.Idle, TotalTimeout: cfg.Router.Timeouts.Total, MaxResponseBytes: cfg.Router.MaxBodyBytes, }), } r.poller = scrape.NewPoller(reg, scrape.Config{ Interval: cfg.Router.ScrapeInterval, Timeout: cfg.Router.ScrapeTimeout, }, log) if !cfg.Auth.Enabled && !loopbackOnly(cfg.Listen) { log.Warn("auth is off and the API listens beyond this machine: anyone who can reach "+ cfg.Listen+" can use every model; set auth.enabled, or listen on 127.0.0.1", "listen", cfg.Listen) } if cfg.Auth.Enabled { st, err := storepkg.Open(context.Background(), cfg.Auth.Store) if err != nil { return nil, fmt.Errorf("auth store: %w", err) } r.store = st r.usageRetention = cfg.Auth.UsageRetention // Applied where no scope set one, so a project created without limits // is still metered. Zero leaves it unlimited, which was the old // behaviour and is now a choice rather than an omission. var defRPM, defTPM *int if cfg.Auth.DefaultRPM > 0 { defRPM = &cfg.Auth.DefaultRPM } if cfg.Auth.DefaultTPM > 0 { defTPM = &cfg.Auth.DefaultTPM } st.SetDefaultLimits(defRPM, defTPM) st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal)) r.personalOff = cfg.Auth.Personal.Disabled r.personalDefaults = personalLimits(cfg.Auth.Personal) r.llamaVersion = cfg.WantedEngines()["llamacpp"] r.auth = auth.New(st, auth.Config{ TTL: cfg.Auth.CacheTTL, NegativeTTL: cfg.Auth.NegativeTTL, }) prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing)) for name, p := range cfg.Auth.Pricing { prices[name] = accounting.Price{ PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, } } r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{ FlushInterval: cfg.Auth.FlushInterval, BatchSize: cfg.Auth.BatchSize, Capacity: cfg.Auth.BufferSize, }, log) r.accounts = account.New(st, account.Config{ SignupOpen: cfg.Auth.SignupOpen, }, log) r.acct = accounting.New( accounting.NewPricing(prices, accounting.Price{ PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k, CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k, }), r.recorder, log) r.threads = r.store } else if cfg.Threads.Store != "" { st, err := storepkg.Open(context.Background(), cfg.Threads.Store) if err != nil { return nil, fmt.Errorf("threads store: %w", err) } if err := st.EnsureAnonymousProject(context.Background(), anonymousProject, cfg.Threads.RetainHours); err != nil { _ = st.Close() return nil, fmt.Errorf("threads store: %w", err) } r.threads, r.anonymous, r.ownsThreads = st, true, true } if cfg.Router.Admission.Enabled { adm, err := admission.New(admissionConfig(cfg.Router), r.saturation) if err != nil { return nil, fmt.Errorf("admission: %w", err) } r.adm = adm } // Seeded from configuration, not only pushed. A router running without a // control plane — a static endpoint list, which is a supported mode — has // nothing to push it a catalog, and would otherwise resolve no model at // all. A control plane pushes over the top of this at startup and on every // change after. if len(cfg.Models) > 0 { r.storeModels(cfg.Models) } return r, nil } func admissionConfig(rc config.Router) admission.Config { bands := make([]admission.Band, 0, len(rc.Admission.Bands)) for _, b := range rc.Admission.Bands { bands = append(bands, admission.Band{ Name: b.Name, Priority: b.Priority, SaturationCeiling: b.SaturationCeiling, }) } return admission.Config{ Bands: bands, DefaultBand: rc.Admission.DefaultBand, Thresholds: admission.Thresholds{ // Admission and the queue scorer must agree on what a healthy // buffer is, or the two layers fight each other. TargetWaitingPerReplica: rc.Scoring.TargetWaiting, KVCeiling: rc.Scoring.KVCeiling, }, QueueTTL: rc.Admission.QueueTTL, QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand, QueueCapacityTotal: rc.Admission.QueueCapacityTotal, Interval: rc.Admission.Interval, } } // saturation reports the pool pressure for a model, for the admission layer. // // outstanding is what admission has let through. It is compared against the // registry's inflight counters rather than the scraped engine stats, because // inflight updates the instant a request is dispatched while scraped stats lag // by a whole interval. Comparing against scraped data would make every // dispatched-but-unscraped request look like queue depth and throttle the // steady state down to a few requests per scrape. // // What is left over — admitted but not yet dispatched — is the thundering herd: // a burst that passed the check together before any of it became visible. func (r *Router) saturation(model string, outstanding int) (float64, bool) { eps := r.reg.Snapshots(model) if len(eps) == 0 { return 0, false } dispatched := 0 for _, e := range eps { dispatched += e.Inflight } return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{ TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting, KVCeiling: r.cfg.Scoring.KVCeiling, }), true } func toInfos(eps []config.Endpoint) []endpoint.Info { out := make([]endpoint.Info, 0, len(eps)) for _, e := range eps { out = append(out, endpoint.Info{ ID: e.ID, URL: strings.TrimRight(e.URL, "/"), APIKey: e.APIKey, Models: e.Models, Engine: e.Engine, Pool: e.Pool, Weight: e.Weight, Priority: e.Priority, }) } return out } // Start begins scraping endpoint metrics and dispatching admissions. // It returns immediately. func (r *Router) Start(ctx context.Context) { r.poller.Sync(ctx) if r.adm != nil { r.adm.Start() } if r.recorder != nil { r.recorder.Start(ctx) } if r.acct != nil && r.store != nil { // Budgets are counted in memory; the store is what makes them survive a // restart. A failure here is not fatal — refusing to start because last // month's spend could not be read would be worse than starting with // counters that are low. if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil { r.log.Warn("could not restore budgets; they start from zero this period", "err", err) } } if r.threads != nil { go r.pruneThreads(ctx) } if r.store != nil && r.usageRetention > 0 { go r.pruneUsage(ctx, r.usageRetention) } if r.store != nil { go r.refreshTenants(ctx) } } // refreshTenants keeps the tenant names the request series are labelled with, // and the budget gauges, a minute fresh. func (r *Router) refreshTenants(ctx context.Context) { refresh := func() { if err := r.m.tenant.refresh(ctx, r.store, time.Now()); err != nil && ctx.Err() == nil { r.log.Warn("could not refresh tenant names and budget gauges", "err", err) } } refresh() t := time.NewTicker(time.Minute) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: refresh() } } } // pruneUsage deletes request records older than the retention, hourly and // once at startup, as pruneThreads does. The five-minute sums stay, so no // total or chart moves; what goes is the trail of each old request. func (r *Router) pruneUsage(ctx context.Context, keep time.Duration) { prune := func() { n, err := r.store.PruneUsage(ctx, time.Now().Add(-keep)) if err != nil { r.log.Warn("could not prune old request records", "err", err) return } if n > 0 { r.log.Info("removed request records past their retention", "rows", n, "retention", keep) } } prune() t := time.NewTicker(time.Hour) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: prune() } } } // pruneThreads enforces each project's retention window. // // Without something running this, "we keep conversations for 24 hours" is a // comment rather than a promise — the rows would simply accumulate, and the // number a project chose would describe nothing. // // Hourly, and once at startup. A gateway that has been down for a week should // not wait another hour before honouring a window that expired during it, and // an hour is fine granularity for a limit expressed in hours. func (r *Router) pruneThreads(ctx context.Context) { prune := func() { n, err := r.threads.PruneThreads(ctx, time.Now()) if err != nil { r.log.Warn("could not prune expired conversations", "err", err) return } if n > 0 { r.log.Info("removed conversations past their retention", "threads", n) } m, err := r.threads.PruneMemory(ctx, time.Now()) if err != nil { r.log.Warn("could not prune expired memory notes", "err", err) return } if m > 0 { r.log.Info("removed memory notes past their retention", "notes", m) } } prune() t := time.NewTicker(time.Hour) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: prune() } } } // Stop halts scraping and admission. func (r *Router) Stop() { r.poller.Stop() if r.adm != nil { r.adm.Close() } // Drain before closing the database, or the last batch has nowhere to go. if r.recorder != nil { r.recorder.Stop() } if r.store != nil { _ = r.store.Close() } if r.ownsThreads { _ = r.threads.Close() } } // SetLeader tells /leader how to know whether this process leads. func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) } // Registry exposes the endpoint registry, for tests and admin endpoints. func (r *Router) Registry() *endpoint.Registry { return r.reg } // SetEndpoints replaces the endpoint list and resynchronizes metric scraping. // // This is how the control plane feeds the router from phase 3 onwards, taking // over from the static configuration list (plan §6.4). Endpoints that survive // the update keep their live state — inflight counts, breaker state, drain rate // — so a replica that was already serving does not start from cold just // because another one moved. // modelMap returns the name→model map in force. // // Never nil: a router built before any catalog arrived resolves nothing, which // is the same answer it gave for an unknown name before. func (r *Router) modelMap() map[string]catalog.Model { if m := r.models.Load(); m != nil { return *m } return nil } // SetCatalog replaces the models this router knows about. // // The catalog's counterpart to SetEndpoints, and pushed for the same reason: // the control plane owns what should exist, the router needs to know, and // neither has to be in the other's process for that to work. func (r *Router) SetCatalog(c *catalog.Catalog) { r.storeModels(c.Models()) } // storeModels makes models what names resolve to: each model by its name, its // aliases and its adapters. One place for it, whether the models came from // configuration at start or from the control plane since. func (r *Router) storeModels(models []catalog.Model) { m := map[string]catalog.Model{} ads := map[string]adapterRef{} for _, entry := range models { m[entry.Name] = entry for _, a := range entry.Aliases { m[a] = entry } // An adapter resolves to the model it runs on, as an alias does, and // is applied to the request on the way there. for i, a := range entry.Adapters { m[a.Name] = entry ads[a.Name] = adapterRef{base: entry.Name, index: i, scale: a.EffectiveScale()} } } r.models.Store(&m) r.adapters.Store(&ads) } // adapterRef is where an adapter runs and how it is applied. type adapterRef struct { base string index int scale float64 } // adapter finds one by the name a client asked for. func (r *Router) adapter(name string) (adapterRef, bool) { if p := r.adapters.Load(); p != nil { a, ok := (*p)[name] return a, ok } return adapterRef{}, false } // LastUsed reports when each endpoint was last sent a request, satisfying // control.UsageSource so the control plane can tell a busy replica from one // holding its cards out of habit. func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() } // ModelLoad sums what each model's replicas report: how many there are, and // requests running and waiting at their engines. Keyed by the catalog name, // which the control plane publishes first among each endpoint's names. func (r *Router) ModelLoad() map[string]reconciler.Load { out := map[string]reconciler.Load{} for _, s := range r.reg.Snapshots("") { if len(s.Models) == 0 { continue } l := out[s.Models[0]] l.Replicas++ l.Running += s.Stats.Running l.Waiting += s.Stats.Waiting out[s.Models[0]] = l } return out } func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) { if r.prefix != nil { // A replica that has gone takes its cache with it. Leaving its entries // in the tree would steer traffic towards something that is not there, // and since nothing reuses a replica id the entries would never be // corrected — only expire, minutes later. before := map[string]struct{}{} for _, s := range r.reg.Snapshots("") { before[s.ID] = struct{}{} } for _, i := range infos { delete(before, i.ID) } for id := range before { r.prefix.Forget(id) } } r.reg.Replace(infos) r.poller.Sync(ctx) } // --- model resolution --- // resolve maps a client-supplied model name onto the catalog. Unknown names are // passed through unchanged: an endpoint may serve a model the catalog does not // list, and refusing it would be unhelpful. func (r *Router) resolve(name string) (canonical string, m catalog.Model) { if e, ok := r.modelMap()[name]; ok { return e.Name, e } return name, catalog.Model{Name: name} } func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile { for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} { if p, ok := r.profiles[name]; ok && name != "" { return p } } return r.profiles["chat"] } // --- dispatch --- // attempt describes one upstream try, for logging and metrics. type dispatchResult struct { endpointID string pool string servedModel outcome proxy.Outcome err error } type servedModel struct { model string // fallback is true when the served model is not the one that was requested. fallback bool } // dispatch resolves the model, then tries endpoints until one works, the client // has been written to, or the attempt budget runs out. // // The retry rule (plan §7.7) is strict: retry only while nothing has reached the // client. Once bytes are out, a retry would splice two different responses // together, which is worse than the error the client would otherwise see. func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult { canonical, entry := r.resolve(req.Model) // The requested model first, then its fallbacks. A fallback is a different // model, so the client is told about it via x-llmgw-served-model. chain := append([]string{canonical}, entry.Fallbacks...) var last dispatchResult for i, model := range chain { if i > 0 { // Before dispatching, because dispatchOne writes the response and // a header set after that is a header nobody receives. It was set // in finish until now, which is after the proxy has already called // WriteHeader — so this has announced a fallback to nobody since it // was written. w.Header().Set("x-llmgw-served-model", model) } res := r.dispatchOne(ctx, w, req, model, entry) res.model = model res.fallback = i > 0 if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) { return res } last = res // Only a total absence of capacity justifies moving to another model; // a bad request would fail identically there. if !errors.Is(res.err, errNoCapacity) { return res } } return last } var errNoCapacity = errors.New("no endpoint available") // errLoading means the model is being started and the wait ran out. // // Distinct from having no capacity because the right client behaviour differs: // this one succeeds on a retry, and the other does not. var errLoading = errors.New("model loading") func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult { candidates := r.reg.Snapshots(model) if len(candidates) == 0 { // Nothing is serving it. That may be because nothing ever will, or // because it loads when asked and nobody has asked yet — and the second // is not an error, it is the feature working. if r.activate(ctx, model) { candidates = r.reg.Snapshots(model) } } if len(candidates) == 0 { if why, no := r.whyUnservable(model); no && r.activator != nil { // Not "loading; retry": it will not load, and a client told to // retry keeps retrying something that never arrives. return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, why)} } if r.activator != nil && r.activator.Want(model) { // Asked for, still loading. Saying "no endpoint serves this" would // be a lie, and a client told to retry will succeed where one told // the model does not exist gives up. return dispatchResult{err: fmt.Errorf("%w: %s", errLoading, "the model is loading; retry shortly")} } return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))} } profile := r.profileFor(entry, req.ProfileOverride) sctx := &scheduler.Ctx{ Now: time.Now(), Req: scheduler.Request{ Model: model, SessionID: req.SessionID, PromptPrefix: req.PromptPrefix, PromptTokens: req.PromptTokens, MaxTokens: req.MaxTokens, }, } attempts := max(r.cfg.Retries+1, 1) var last dispatchResult for range attempts { if len(candidates) == 0 { break } _, sspan := tracer().Start(ctx, "schedule", trace.WithAttributes( attribute.String("llmgw.model", model), attribute.Int("llmgw.candidates", len(candidates)))) dec, err := profile.Schedule(sctx, candidates) if err == nil { sspan.SetAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID)) } endSpan(sspan, err) if err != nil { last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)} break } release, granted := r.reg.Acquire(dec.Endpoint.ID) if !granted { // The breaker opened between scoring and dispatch; drop it and // score again rather than failing the request. candidates = without(candidates, dec.Endpoint.ID) continue } uctx, uspan := tracer().Start(ctx, "upstream", trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(attribute.String("llmgw.endpoint", dec.Endpoint.ID), attribute.String("url.full", dec.Endpoint.URL+req.Path))) body := req.UpstreamBody if dec.Endpoint.UpstreamModel != "" { body = withModel(body, dec.Endpoint.UpstreamModel) } // An adapter asked for by name, served by its model's replicas: the // engine is asked for the model, with the adapter applied. Not on a // fallback to another model, which does not have it. if a, ok := r.adapter(req.Model); ok && a.base == model { body = withAdapter(body, model, a.index, a.scale) } // What computes the answer, said before the proxy writes the headers. // A model can have replicas on cards and on CPUs at once, and the two // differ tenfold in speed: a reply that came slowly is otherwise // indistinguishable from a model that is slow everywhere. Set again on // a retry, which only happens before anything was written. if hw := hardwareOf(dec.Endpoint.Pool); hw != "" { w.Header().Set(hardwareHeader, hw) } else { w.Header().Del(hardwareHeader) } out, perr := r.px.Do(uctx, w, proxy.Request{ URL: dec.Endpoint.URL + req.Path, APIKey: dec.Endpoint.APIKey, Body: body, Header: withTraceparent(uctx, req.Header), Stream: req.Stream, SuppressUsageChunk: req.SuppressUsageChunk, CaptureReply: req.captureBytes(), }) uspan.SetAttributes(attribute.Int("http.response.status_code", out.Status)) endSpan(uspan, perr) // A retryable failure is the endpoint's fault; a client disconnect is not. release(perr == nil || errors.Is(perr, proxy.ErrClientGone)) res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr} r.m.observeScores(dec) // Recorded after the decision, not during it: the scorer proposes and // the picker disposes, so recording the proposal would fill the tree // with replicas that were never sent anything — the one thing that // would make the estimate worse than no estimate at all. if r.prefix != nil { r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID) } if perr == nil || out.Wrote || !out.Retryable { return res } last = res r.log.Warn("upstream attempt failed, trying another endpoint", "endpoint", dec.Endpoint.ID, "model", model, "err", perr) candidates = without(candidates, dec.Endpoint.ID) } if last.err == nil { last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model) } else if !errors.Is(last.err, errNoCapacity) { last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err) } return last } // hardwareHeader says what the replica that answered computes on: "cpu", or // the GPU class, such as "6000-24g". const hardwareHeader = "x-llmgw-hardware" // hardwareOf reads it from a pool, which placement names fabric/class — // "none/cpu", "none/6000-24g", "ib-a/h100-80g". Empty for an endpoint // configured by hand, which has no pool and says nothing about its hardware. func hardwareOf(pool string) string { i := strings.LastIndexByte(pool, '/') if i < 0 || i == len(pool)-1 { return "" } return pool[i+1:] } func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot { return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id }) } // personalLimits turns the personal block into limits, nil when it sets none. func personalLimits(p config.Personal) *storepkg.Limits { var l storepkg.Limits if p.RPM > 0 { v := p.RPM l.RPM = &v } if p.TPM > 0 { v := p.TPM l.TPM = &v } if p.Budget > 0 { v := int64(math.Round(p.Budget * 1e6)) l.Budget = &v l.BudgetPeriod = storepkg.PeriodMonth if p.Period == "day" { l.BudgetPeriod = storepkg.PeriodDay } } if l.RPM == nil && l.TPM == nil && l.Budget == nil { return nil } return &l } // loopbackOnly reports whether a listen address is reachable from this // machine alone. An empty host (":8080") is every interface. func loopbackOnly(listen string) bool { host, _, err := net.SplitHostPort(listen) if err != nil { return false } if host == "localhost" { return true } a, err := netip.ParseAddr(host) return err == nil && a.IsLoopback() }