CI / test (push) Successful in 1m50s
The dashboard charted what the cluster had been billed for — requests, tokens, spend, all from llmgw's own store — and nothing about whether it was healthy. Saturation, queue depth, cache occupancy and replica counts were numbers in a table at the instant the page loaded, with no way to see that one of them had been climbing for an hour. Four line charts now, read back from wherever the metrics were scraped to. Read back rather than kept twice. llmgw publishes these already; what a gauge cannot do is remember, and a ring buffer in this process would be a rival copy of the same numbers with its own retention, lost on every restart and disagreeing with the one an operator has on screen elsewhere. monitoring.prometheus is empty by default and the panels are then absent rather than empty. One axis per chart, always. Two measures of different scale belong in two charts: a second y-axis lets any two lines be made to cross wherever the author likes and the reader cannot tell it happened. Four colours, handed out in fixed order, never cycled. A fifth series is summed into "other" and drawn in grey — dropping it would make a chart that does not add up, and inventing a hue for it would put two lines on the page that nobody can tell apart. The fold is by timestamp, not by position: the tail's series do not share sample times, and adding them by index slides points sideways onto each other's clock. A legend for every chart with more than one line, because colour alone is not an encoding a reader can rely on. A flat zero is drawn along the bottom rather than left blank: nothing happening is a real answer, and an empty frame reads as a broken chart. A single sample is not a line and says so instead of putting a dot in the corner. One failed query fails all four panels. Three drawn and one blank invites reading the blank one as "nothing is happening there". NaN is dropped rather than plotted. Prometheus sends it as a string for a sample it has no value for, Go's ParseFloat accepts it happily, and one NaN coordinate is an SVG path that does not draw at all — taking the rest of the line with it. Axis maxima are rounded to a number a person would pick. An axis topping out at 0.8347 makes every gridline unreadable, and the shape is what a chart is for. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
702 lines
23 KiB
Go
702 lines
23 KiB
Go
// Package router wires the pieces into a working OpenAI-compatible gateway:
|
|
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
|
|
//
|
|
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
|
|
// breaker); this package holds the orchestration and the HTTP surface.
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// Router serves the OpenAI API in front of a set of engine replicas.
|
|
type Router struct {
|
|
cfg config.Router
|
|
log *slog.Logger
|
|
|
|
reg *endpoint.Registry
|
|
poller *scrape.Poller
|
|
px *proxy.Proxy
|
|
m *metrics
|
|
// adm is nil when admission control is disabled.
|
|
adm *admission.Controller
|
|
|
|
profiles map[string]*scheduler.Profile
|
|
// models maps every name a client may use — canonical names and aliases —
|
|
// onto its catalog entry.
|
|
//
|
|
// Held atomically because the catalog is no longer fixed at startup: adding
|
|
// a model used to mean restarting the control plane, and this is the
|
|
// router's half of not having to. A request takes the map once and works
|
|
// from it, so a swap mid-request is invisible rather than a race.
|
|
models atomic.Pointer[map[string]catalog.Model]
|
|
|
|
// extraMetrics are collectors other components publish on /metrics.
|
|
extraMetrics []prometheus.Collector
|
|
|
|
// activator loads a model that is configured but not currently running.
|
|
activator Activator
|
|
|
|
// metricsHandler is built with the admin registry and reused by any
|
|
// listener that should serve metrics without serving pprof.
|
|
metricsHandler http.Handler
|
|
|
|
// landing configures the public page, which the router serves but does not
|
|
// otherwise care about.
|
|
landing config.Landing
|
|
|
|
// auth, store, acct, recorder and accounts are nil when authentication is
|
|
// disabled.
|
|
auth *auth.Authenticator
|
|
accounts *account.Service
|
|
store *storepkg.DB
|
|
acct *accounting.Accountant
|
|
|
|
// threads holds conversations and memory notes. It is store when auth is on,
|
|
// its own file when only the threads block is configured, and nil when
|
|
// nothing is kept. Separate from store because a non-nil store also switches
|
|
// on the accounts, web pages and usage recording, none of which belong to a
|
|
// gateway that only wants to remember things.
|
|
threads *storepkg.DB
|
|
// anonymous means threads has no project to scope to: everything shares the
|
|
// one pool named by anonymousProject.
|
|
anonymous bool
|
|
// ownsThreads is true when threads is not store and so must be closed here.
|
|
ownsThreads bool
|
|
recorder *accounting.Recorder
|
|
|
|
// prefix is nil unless approximate prefix routing is turned on.
|
|
prefix *scheduler.PrefixScorer
|
|
|
|
// clusterView is nil until the control plane hands it over, and stays nil
|
|
// in a router-only deployment.
|
|
clusterView web.Source
|
|
clusterUsage web.UsageSource
|
|
|
|
// mirror is nil unless a model mirror is configured, which leaves
|
|
// /admin/mirror unmounted — the right page for a deployment with no mirror
|
|
// is no page, not an empty one pointing at nothing.
|
|
mirror web.MirrorAdmin
|
|
// registry browses a model registry from the mirror page. Nil where the
|
|
// deployment has no way out to the internet.
|
|
registry web.Registry
|
|
// catalogAdmin puts a mirrored model into service. Nil without a store,
|
|
// where a model added at runtime would not survive a restart.
|
|
catalogAdmin web.CatalogAdmin
|
|
// history is the metrics store the dashboard charts read from. Nil where
|
|
// none is configured, which leaves the charts off the page.
|
|
history web.History
|
|
histWindow time.Duration
|
|
}
|
|
|
|
// SetClusterView gives the router the data the /admin page renders.
|
|
//
|
|
// Set by the server command once the control plane exists, because the router
|
|
// is built first. Nil leaves /admin unmounted, which is what a router-only
|
|
// deployment should have: no cluster to show.
|
|
func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
|
|
r.clusterView, r.clusterUsage = src, usage
|
|
}
|
|
|
|
// SetMirror gives the router a model mirror to administer.
|
|
//
|
|
// Set by the server command when one is configured, for the same reason as the
|
|
// cluster view: the router is built before the things its admin pages show.
|
|
func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m }
|
|
|
|
// SetRegistry gives the mirror page somewhere to look models up.
|
|
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
|
|
|
|
// SetCatalogAdmin lets the mirror page put a model into service.
|
|
func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c }
|
|
|
|
// SetHistory gives the dashboard somewhere to read the cluster's past.
|
|
func (r *Router) SetHistory(h web.History, window time.Duration) {
|
|
r.history, r.histWindow = h, window
|
|
}
|
|
|
|
// Auth exposes the authenticator so ctl and the batch writer can reach it.
|
|
// Nil when authentication is off.
|
|
func (r *Router) Auth() *auth.Authenticator { return r.auth }
|
|
|
|
// Store exposes the key store, or nil when authentication is off.
|
|
func (r *Router) Store() *storepkg.DB { return r.store }
|
|
|
|
// New builds a router from configuration. Start must be called to begin scraping.
|
|
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
|
|
reg := endpoint.NewRegistry(endpoint.Config{
|
|
StaleAfter: cfg.Router.StaleAfter,
|
|
Breaker: breaker.Config{
|
|
FailureThreshold: cfg.Router.FailureThreshold,
|
|
BaseCooldown: cfg.Router.Cooldown,
|
|
},
|
|
})
|
|
reg.Replace(toInfos(cfg.Endpoints))
|
|
|
|
sc := scheduler.ProfileConfig{
|
|
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
|
|
TargetInflight: cfg.Router.Scoring.TargetInflight,
|
|
KVCeiling: cfg.Router.Scoring.KVCeiling,
|
|
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
|
|
}
|
|
// The metrics come first because the prefix scorer reports into them, and
|
|
// the decision metric is the whole difference between tuning this and
|
|
// guessing at it.
|
|
metrics := newMetrics()
|
|
var prefixScorer *scheduler.PrefixScorer
|
|
if cfg.Router.Prefix.Enabled {
|
|
prefixScorer = scheduler.NewPrefixScorer(
|
|
prefix.New(prefix.Config{
|
|
MaxNodes: cfg.Router.Prefix.MaxNodes,
|
|
MinMatch: cfg.Router.Prefix.MinMatch,
|
|
TTL: cfg.Router.Prefix.TTL,
|
|
}),
|
|
prefix.Thresholds{
|
|
MatchRatio: cfg.Router.Prefix.MatchRatio,
|
|
BalanceAbs: cfg.Router.Prefix.BalanceAbs,
|
|
BalanceRel: cfg.Router.Prefix.BalanceRel,
|
|
},
|
|
metrics.observePrefix)
|
|
sc.Prefix = prefixScorer
|
|
}
|
|
|
|
profiles := map[string]*scheduler.Profile{}
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
profiles[name] = scheduler.Builtin(name, sc)
|
|
}
|
|
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
|
|
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
|
|
}
|
|
|
|
r := &Router{
|
|
cfg: cfg.Router,
|
|
landing: cfg.Landing,
|
|
log: log,
|
|
reg: reg,
|
|
profiles: profiles,
|
|
prefix: prefixScorer,
|
|
m: metrics,
|
|
px: proxy.New(proxy.Config{
|
|
DialTimeout: cfg.Router.Timeouts.Dial,
|
|
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
|
|
GenerationTimeout: cfg.Router.Timeouts.Generation,
|
|
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
|
|
IdleTimeout: cfg.Router.Timeouts.Idle,
|
|
TotalTimeout: cfg.Router.Timeouts.Total,
|
|
MaxResponseBytes: cfg.Router.MaxBodyBytes,
|
|
}),
|
|
}
|
|
r.poller = scrape.NewPoller(reg, scrape.Config{
|
|
Interval: cfg.Router.ScrapeInterval,
|
|
Timeout: cfg.Router.ScrapeTimeout,
|
|
}, log)
|
|
|
|
if cfg.Auth.Enabled {
|
|
st, err := storepkg.Open(context.Background(), cfg.Auth.Store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("auth store: %w", err)
|
|
}
|
|
r.store = st
|
|
// Applied where no scope set one, so a project created without limits
|
|
// is still metered. Zero leaves it unlimited, which was the old
|
|
// behaviour and is now a choice rather than an omission.
|
|
var defRPM, defTPM *int
|
|
if cfg.Auth.DefaultRPM > 0 {
|
|
defRPM = &cfg.Auth.DefaultRPM
|
|
}
|
|
if cfg.Auth.DefaultTPM > 0 {
|
|
defTPM = &cfg.Auth.DefaultTPM
|
|
}
|
|
st.SetDefaultLimits(defRPM, defTPM)
|
|
|
|
r.auth = auth.New(st, auth.Config{
|
|
TTL: cfg.Auth.CacheTTL,
|
|
NegativeTTL: cfg.Auth.NegativeTTL,
|
|
})
|
|
|
|
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
|
|
for name, p := range cfg.Auth.Pricing {
|
|
prices[name] = accounting.Price{
|
|
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k,
|
|
}
|
|
}
|
|
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
|
|
FlushInterval: cfg.Auth.FlushInterval,
|
|
BatchSize: cfg.Auth.BatchSize,
|
|
Capacity: cfg.Auth.BufferSize,
|
|
}, log)
|
|
r.accounts = account.New(st, account.Config{
|
|
SignupOpen: cfg.Auth.SignupOpen,
|
|
}, log)
|
|
|
|
r.acct = accounting.New(
|
|
accounting.NewPricing(prices, accounting.Price{
|
|
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
|
|
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
|
|
}),
|
|
r.recorder, log)
|
|
r.threads = r.store
|
|
} else if cfg.Threads.Store != "" {
|
|
st, err := storepkg.Open(context.Background(), cfg.Threads.Store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("threads store: %w", err)
|
|
}
|
|
if err := st.EnsureAnonymousProject(context.Background(), anonymousProject, cfg.Threads.RetainHours); err != nil {
|
|
_ = st.Close()
|
|
return nil, fmt.Errorf("threads store: %w", err)
|
|
}
|
|
r.threads, r.anonymous, r.ownsThreads = st, true, true
|
|
}
|
|
|
|
if cfg.Router.Admission.Enabled {
|
|
adm, err := admission.New(admissionConfig(cfg.Router), r.saturation)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("admission: %w", err)
|
|
}
|
|
r.adm = adm
|
|
}
|
|
|
|
// Seeded from configuration, not only pushed. A router running without a
|
|
// control plane — a static endpoint list, which is a supported mode — has
|
|
// nothing to push it a catalog, and would otherwise resolve no model at
|
|
// all. A control plane pushes over the top of this at startup and on every
|
|
// change after.
|
|
if len(cfg.Models) > 0 {
|
|
seed := map[string]catalog.Model{}
|
|
for _, m := range cfg.Models {
|
|
seed[m.Name] = m
|
|
for _, a := range m.Aliases {
|
|
seed[a] = m
|
|
}
|
|
}
|
|
r.models.Store(&seed)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func admissionConfig(rc config.Router) admission.Config {
|
|
bands := make([]admission.Band, 0, len(rc.Admission.Bands))
|
|
for _, b := range rc.Admission.Bands {
|
|
bands = append(bands, admission.Band{
|
|
Name: b.Name,
|
|
Priority: b.Priority,
|
|
SaturationCeiling: b.SaturationCeiling,
|
|
})
|
|
}
|
|
return admission.Config{
|
|
Bands: bands,
|
|
DefaultBand: rc.Admission.DefaultBand,
|
|
Thresholds: admission.Thresholds{
|
|
// Admission and the queue scorer must agree on what a healthy
|
|
// buffer is, or the two layers fight each other.
|
|
TargetWaitingPerReplica: rc.Scoring.TargetWaiting,
|
|
KVCeiling: rc.Scoring.KVCeiling,
|
|
},
|
|
QueueTTL: rc.Admission.QueueTTL,
|
|
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
|
|
Interval: rc.Admission.Interval,
|
|
}
|
|
}
|
|
|
|
// saturation reports the pool pressure for a model, for the admission layer.
|
|
//
|
|
// outstanding is what admission has let through. It is compared against the
|
|
// registry's inflight counters rather than the scraped engine stats, because
|
|
// inflight updates the instant a request is dispatched while scraped stats lag
|
|
// by a whole interval. Comparing against scraped data would make every
|
|
// dispatched-but-unscraped request look like queue depth and throttle the
|
|
// steady state down to a few requests per scrape.
|
|
//
|
|
// What is left over — admitted but not yet dispatched — is the thundering herd:
|
|
// a burst that passed the check together before any of it became visible.
|
|
func (r *Router) saturation(model string, outstanding int) (float64, bool) {
|
|
eps := r.reg.Snapshots(model)
|
|
if len(eps) == 0 {
|
|
return 0, false
|
|
}
|
|
dispatched := 0
|
|
for _, e := range eps {
|
|
dispatched += e.Inflight
|
|
}
|
|
return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{
|
|
TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting,
|
|
KVCeiling: r.cfg.Scoring.KVCeiling,
|
|
}), true
|
|
}
|
|
|
|
func toInfos(eps []config.Endpoint) []endpoint.Info {
|
|
out := make([]endpoint.Info, 0, len(eps))
|
|
for _, e := range eps {
|
|
out = append(out, endpoint.Info{
|
|
ID: e.ID,
|
|
URL: strings.TrimRight(e.URL, "/"),
|
|
Models: e.Models,
|
|
Engine: e.Engine,
|
|
Pool: e.Pool,
|
|
Weight: e.Weight,
|
|
Priority: e.Priority,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Start begins scraping endpoint metrics and dispatching admissions.
|
|
// It returns immediately.
|
|
func (r *Router) Start(ctx context.Context) {
|
|
r.poller.Sync(ctx)
|
|
if r.adm != nil {
|
|
r.adm.Start()
|
|
}
|
|
if r.recorder != nil {
|
|
r.recorder.Start(ctx)
|
|
}
|
|
if r.acct != nil && r.store != nil {
|
|
// Budgets are counted in memory; the store is what makes them survive a
|
|
// restart. A failure here is not fatal — refusing to start because last
|
|
// month's spend could not be read would be worse than starting with
|
|
// counters that are low.
|
|
if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil {
|
|
r.log.Warn("could not restore budgets; they start from zero this period", "err", err)
|
|
}
|
|
}
|
|
if r.threads != nil {
|
|
go r.pruneThreads(ctx)
|
|
}
|
|
}
|
|
|
|
// pruneThreads enforces each project's retention window.
|
|
//
|
|
// Without something running this, "we keep conversations for 24 hours" is a
|
|
// comment rather than a promise — the rows would simply accumulate, and the
|
|
// number a project chose would describe nothing.
|
|
//
|
|
// Hourly, and once at startup. A gateway that has been down for a week should
|
|
// not wait another hour before honouring a window that expired during it, and
|
|
// an hour is fine granularity for a limit expressed in hours.
|
|
func (r *Router) pruneThreads(ctx context.Context) {
|
|
prune := func() {
|
|
n, err := r.threads.PruneThreads(ctx, time.Now())
|
|
if err != nil {
|
|
r.log.Warn("could not prune expired conversations", "err", err)
|
|
return
|
|
}
|
|
if n > 0 {
|
|
r.log.Info("removed conversations past their retention", "threads", n)
|
|
}
|
|
m, err := r.threads.PruneMemory(ctx, time.Now())
|
|
if err != nil {
|
|
r.log.Warn("could not prune expired memory notes", "err", err)
|
|
return
|
|
}
|
|
if m > 0 {
|
|
r.log.Info("removed memory notes past their retention", "notes", m)
|
|
}
|
|
}
|
|
prune()
|
|
|
|
t := time.NewTicker(time.Hour)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
prune()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop halts scraping and admission.
|
|
func (r *Router) Stop() {
|
|
r.poller.Stop()
|
|
if r.adm != nil {
|
|
r.adm.Close()
|
|
}
|
|
// Drain before closing the database, or the last batch has nowhere to go.
|
|
if r.recorder != nil {
|
|
r.recorder.Stop()
|
|
}
|
|
if r.store != nil {
|
|
_ = r.store.Close()
|
|
}
|
|
if r.ownsThreads {
|
|
_ = r.threads.Close()
|
|
}
|
|
}
|
|
|
|
// Registry exposes the endpoint registry, for tests and admin endpoints.
|
|
func (r *Router) Registry() *endpoint.Registry { return r.reg }
|
|
|
|
// SetEndpoints replaces the endpoint list and resynchronizes metric scraping.
|
|
//
|
|
// This is how the control plane feeds the router from phase 3 onwards, taking
|
|
// over from the static configuration list (plan §6.4). Endpoints that survive
|
|
// the update keep their live state — inflight counts, breaker state, drain rate
|
|
// — so a replica that was already serving does not start from cold just
|
|
// because another one moved.
|
|
// modelMap returns the name→model map in force.
|
|
//
|
|
// Never nil: a router built before any catalog arrived resolves nothing, which
|
|
// is the same answer it gave for an unknown name before.
|
|
func (r *Router) modelMap() map[string]catalog.Model {
|
|
if m := r.models.Load(); m != nil {
|
|
return *m
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetCatalog replaces the models this router knows about.
|
|
//
|
|
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
|
|
// the control plane owns what should exist, the router needs to know, and
|
|
// neither has to be in the other's process for that to work.
|
|
func (r *Router) SetCatalog(c *catalog.Catalog) {
|
|
m := map[string]catalog.Model{}
|
|
for _, entry := range c.Models() {
|
|
m[entry.Name] = entry
|
|
for _, a := range entry.Aliases {
|
|
m[a] = entry
|
|
}
|
|
}
|
|
r.models.Store(&m)
|
|
}
|
|
|
|
// LastUsed reports when each endpoint was last sent a request, satisfying
|
|
// control.UsageSource so the control plane can tell a busy replica from one
|
|
// holding its cards out of habit.
|
|
func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() }
|
|
|
|
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
|
|
if r.prefix != nil {
|
|
// A replica that has gone takes its cache with it. Leaving its entries
|
|
// in the tree would steer traffic towards something that is not there,
|
|
// and since nothing reuses a replica id the entries would never be
|
|
// corrected — only expire, minutes later.
|
|
before := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
before[s.ID] = struct{}{}
|
|
}
|
|
for _, i := range infos {
|
|
delete(before, i.ID)
|
|
}
|
|
for id := range before {
|
|
r.prefix.Forget(id)
|
|
}
|
|
}
|
|
|
|
r.reg.Replace(infos)
|
|
r.poller.Sync(ctx)
|
|
}
|
|
|
|
// --- model resolution ---
|
|
|
|
// resolve maps a client-supplied model name onto the catalog. Unknown names are
|
|
// passed through unchanged: an endpoint may serve a model the catalog does not
|
|
// list, and refusing it would be unhelpful.
|
|
func (r *Router) resolve(name string) (canonical string, m catalog.Model) {
|
|
if e, ok := r.modelMap()[name]; ok {
|
|
return e.Name, e
|
|
}
|
|
return name, catalog.Model{Name: name}
|
|
}
|
|
|
|
func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile {
|
|
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
|
|
if p, ok := r.profiles[name]; ok && name != "" {
|
|
return p
|
|
}
|
|
}
|
|
return r.profiles["chat"]
|
|
}
|
|
|
|
// --- dispatch ---
|
|
|
|
// attempt describes one upstream try, for logging and metrics.
|
|
type dispatchResult struct {
|
|
endpointID string
|
|
pool string
|
|
servedModel
|
|
outcome proxy.Outcome
|
|
err error
|
|
}
|
|
|
|
type servedModel struct {
|
|
model string
|
|
// fallback is true when the served model is not the one that was requested.
|
|
fallback bool
|
|
}
|
|
|
|
// dispatch resolves the model, then tries endpoints until one works, the client
|
|
// has been written to, or the attempt budget runs out.
|
|
//
|
|
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
|
|
// client. Once bytes are out, a retry would splice two different responses
|
|
// together, which is worse than the error the client would otherwise see.
|
|
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
|
|
canonical, entry := r.resolve(req.Model)
|
|
|
|
// The requested model first, then its fallbacks. A fallback is a different
|
|
// model, so the client is told about it via x-llmgw-served-model.
|
|
chain := append([]string{canonical}, entry.Fallbacks...)
|
|
|
|
var last dispatchResult
|
|
for i, model := range chain {
|
|
if i > 0 {
|
|
// Before dispatching, because dispatchOne writes the response and
|
|
// a header set after that is a header nobody receives. It was set
|
|
// in finish until now, which is after the proxy has already called
|
|
// WriteHeader — so this has announced a fallback to nobody since it
|
|
// was written.
|
|
w.Header().Set("x-llmgw-served-model", model)
|
|
}
|
|
res := r.dispatchOne(ctx, w, req, model, entry)
|
|
res.model = model
|
|
res.fallback = i > 0
|
|
|
|
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
|
|
return res
|
|
}
|
|
last = res
|
|
// Only a total absence of capacity justifies moving to another model;
|
|
// a bad request would fail identically there.
|
|
if !errors.Is(res.err, errNoCapacity) {
|
|
return res
|
|
}
|
|
}
|
|
return last
|
|
}
|
|
|
|
var errNoCapacity = errors.New("no endpoint available")
|
|
|
|
// errLoading means the model is being started and the wait ran out.
|
|
//
|
|
// Distinct from having no capacity because the right client behaviour differs:
|
|
// this one succeeds on a retry, and the other does not.
|
|
var errLoading = errors.New("model loading")
|
|
|
|
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
|
|
candidates := r.reg.Snapshots(model)
|
|
if len(candidates) == 0 {
|
|
// Nothing is serving it. That may be because nothing ever will, or
|
|
// because it loads when asked and nobody has asked yet — and the second
|
|
// is not an error, it is the feature working.
|
|
if r.activate(ctx, model) {
|
|
candidates = r.reg.Snapshots(model)
|
|
}
|
|
}
|
|
if len(candidates) == 0 {
|
|
if r.activator != nil && r.activator.Want(model) {
|
|
// Asked for, still loading. Saying "no endpoint serves this" would
|
|
// be a lie, and a client told to retry will succeed where one told
|
|
// the model does not exist gives up.
|
|
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
|
|
"the model is loading; retry shortly")}
|
|
}
|
|
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
|
|
}
|
|
|
|
profile := r.profileFor(entry, req.ProfileOverride)
|
|
sctx := &scheduler.Ctx{
|
|
Now: time.Now(),
|
|
Req: scheduler.Request{
|
|
Model: model,
|
|
SessionID: req.SessionID,
|
|
PromptPrefix: req.PromptPrefix,
|
|
PromptTokens: req.PromptTokens,
|
|
MaxTokens: req.MaxTokens,
|
|
},
|
|
}
|
|
|
|
attempts := max(r.cfg.Retries+1, 1)
|
|
var last dispatchResult
|
|
|
|
for range attempts {
|
|
if len(candidates) == 0 {
|
|
break
|
|
}
|
|
dec, err := profile.Schedule(sctx, candidates)
|
|
if err != nil {
|
|
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
|
|
break
|
|
}
|
|
|
|
release, granted := r.reg.Acquire(dec.Endpoint.ID)
|
|
if !granted {
|
|
// The breaker opened between scoring and dispatch; drop it and
|
|
// score again rather than failing the request.
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
continue
|
|
}
|
|
|
|
out, perr := r.px.Do(ctx, w, proxy.Request{
|
|
URL: dec.Endpoint.URL + req.Path,
|
|
Body: req.UpstreamBody,
|
|
Header: req.Header,
|
|
Stream: req.Stream,
|
|
SuppressUsageChunk: req.SuppressUsageChunk,
|
|
CaptureReply: req.captureBytes(),
|
|
})
|
|
// A retryable failure is the endpoint's fault; a client disconnect is not.
|
|
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
|
|
|
|
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
|
|
r.m.observeScores(dec)
|
|
|
|
// Recorded after the decision, not during it: the scorer proposes and
|
|
// the picker disposes, so recording the proposal would fill the tree
|
|
// with replicas that were never sent anything — the one thing that
|
|
// would make the estimate worse than no estimate at all.
|
|
if r.prefix != nil {
|
|
r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID)
|
|
}
|
|
|
|
if perr == nil || out.Wrote || !out.Retryable {
|
|
return res
|
|
}
|
|
last = res
|
|
r.log.Warn("upstream attempt failed, trying another endpoint",
|
|
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
}
|
|
|
|
if last.err == nil {
|
|
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
|
|
} else if !errors.Is(last.err, errNoCapacity) {
|
|
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
|
|
}
|
|
return last
|
|
}
|
|
|
|
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
|
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
|
}
|