A message to a model that was not running waited 45 seconds and came back as "the model is loading; retry shortly". The page now asks for the model with /chat/wake, shows where it stands from /chat/model-status - waiting for a node, downloading with bytes and a percentage, starting the engine - and sends the message when it answers. Also passes the nodes' llama.cpp build to the pages, which the mirror's checks for loadable weights needed and were never given. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
741 lines
25 KiB
Go
741 lines
25 KiB
Go
// Package router wires the pieces into a working OpenAI-compatible gateway:
|
|
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
|
|
//
|
|
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
|
|
// breaker); this package holds the orchestration and the HTTP surface.
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"math"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/account"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/prefix"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
|
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// Router serves the OpenAI API in front of a set of engine replicas.
|
|
type Router struct {
|
|
cfg config.Router
|
|
log *slog.Logger
|
|
|
|
reg *endpoint.Registry
|
|
poller *scrape.Poller
|
|
px *proxy.Proxy
|
|
m *metrics
|
|
// adm is nil when admission control is disabled.
|
|
adm *admission.Controller
|
|
|
|
profiles map[string]*scheduler.Profile
|
|
// models maps every name a client may use — canonical names and aliases —
|
|
// onto its catalog entry.
|
|
//
|
|
// Held atomically because the catalog is no longer fixed at startup: adding
|
|
// a model used to mean restarting the control plane, and this is the
|
|
// router's half of not having to. A request takes the map once and works
|
|
// from it, so a swap mid-request is invisible rather than a race.
|
|
models atomic.Pointer[map[string]catalog.Model]
|
|
|
|
// extraMetrics are collectors other components publish on /metrics.
|
|
extraMetrics []prometheus.Collector
|
|
|
|
// activator loads a model that is configured but not currently running.
|
|
activator Activator
|
|
|
|
// metricsHandler is built with the admin registry and reused by any
|
|
// listener that should serve metrics without serving pprof.
|
|
metricsHandler http.Handler
|
|
|
|
// landing configures the public page, which the router serves but does not
|
|
// otherwise care about.
|
|
landing config.Landing
|
|
|
|
// auth, store, acct, recorder and accounts are nil when authentication is
|
|
// disabled.
|
|
auth *auth.Authenticator
|
|
accounts *account.Service
|
|
store *storepkg.DB
|
|
acct *accounting.Accountant
|
|
|
|
// threads holds conversations and memory notes. It is store when auth is on,
|
|
// its own file when only the threads block is configured, and nil when
|
|
// nothing is kept. Separate from store because a non-nil store also switches
|
|
// on the accounts, web pages and usage recording, none of which belong to a
|
|
// gateway that only wants to remember things.
|
|
threads *storepkg.DB
|
|
// anonymous means threads has no project to scope to: everything shares the
|
|
// one pool named by anonymousProject.
|
|
anonymous bool
|
|
// personalOff refuses personal use: the chat page then only offers the
|
|
// projects a person is a member of.
|
|
personalOff bool
|
|
// personalDefaults is what personal use is held to when nobody wrote a
|
|
// limit for the person, for the quota page to say so.
|
|
personalDefaults *storepkg.Limits
|
|
// llamaVersion is the llama.cpp build the nodes install, so the mirror page
|
|
// can refuse weights that build cannot load before fetching them.
|
|
llamaVersion string
|
|
// ownsThreads is true when threads is not store and so must be closed here.
|
|
ownsThreads bool
|
|
recorder *accounting.Recorder
|
|
|
|
// prefix is nil unless approximate prefix routing is turned on.
|
|
prefix *scheduler.PrefixScorer
|
|
|
|
// clusterView is nil until the control plane hands it over, and stays nil
|
|
// in a router-only deployment.
|
|
clusterView web.Source
|
|
clusterUsage web.UsageSource
|
|
|
|
// mirror is nil unless a model mirror is configured, which leaves
|
|
// /admin/mirror unmounted — the right page for a deployment with no mirror
|
|
// is no page, not an empty one pointing at nothing.
|
|
mirror web.MirrorAdmin
|
|
// registry browses a model registry from the mirror page. Nil where the
|
|
// deployment has no way out to the internet.
|
|
registry web.Registry
|
|
// catalogAdmin puts a mirrored model into service. Nil without a store,
|
|
// where a model added at runtime would not survive a restart.
|
|
catalogAdmin web.CatalogAdmin
|
|
// history is the metrics store the dashboard charts read from. Nil where
|
|
// none is configured, which leaves the charts off the page.
|
|
history web.History
|
|
histWindow time.Duration
|
|
}
|
|
|
|
// SetClusterView gives the router the data the /admin page renders.
|
|
//
|
|
// Set by the server command once the control plane exists, because the router
|
|
// is built first. Nil leaves /admin unmounted, which is what a router-only
|
|
// deployment should have: no cluster to show.
|
|
func (r *Router) SetClusterView(src web.Source, usage web.UsageSource) {
|
|
r.clusterView, r.clusterUsage = src, usage
|
|
}
|
|
|
|
// SetMirror gives the router a model mirror to administer.
|
|
//
|
|
// Set by the server command when one is configured, for the same reason as the
|
|
// cluster view: the router is built before the things its admin pages show.
|
|
func (r *Router) SetMirror(m web.MirrorAdmin) { r.mirror = m }
|
|
|
|
// SetRegistry gives the mirror page somewhere to look models up.
|
|
func (r *Router) SetRegistry(reg web.Registry) { r.registry = reg }
|
|
|
|
// SetCatalogAdmin lets the mirror page put a model into service.
|
|
func (r *Router) SetCatalogAdmin(c web.CatalogAdmin) { r.catalogAdmin = c }
|
|
|
|
// SetHistory gives the dashboard somewhere to read the cluster's past.
|
|
func (r *Router) SetHistory(h web.History, window time.Duration) {
|
|
r.history, r.histWindow = h, window
|
|
}
|
|
|
|
// Auth exposes the authenticator so ctl and the batch writer can reach it.
|
|
// Nil when authentication is off.
|
|
func (r *Router) Auth() *auth.Authenticator { return r.auth }
|
|
|
|
// Store exposes the key store, or nil when authentication is off.
|
|
func (r *Router) Store() *storepkg.DB { return r.store }
|
|
|
|
// New builds a router from configuration. Start must be called to begin scraping.
|
|
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
|
|
reg := endpoint.NewRegistry(endpoint.Config{
|
|
StaleAfter: cfg.Router.StaleAfter,
|
|
Breaker: breaker.Config{
|
|
FailureThreshold: cfg.Router.FailureThreshold,
|
|
BaseCooldown: cfg.Router.Cooldown,
|
|
},
|
|
})
|
|
reg.Replace(toInfos(cfg.Endpoints))
|
|
|
|
sc := scheduler.ProfileConfig{
|
|
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
|
|
TargetInflight: cfg.Router.Scoring.TargetInflight,
|
|
KVCeiling: cfg.Router.Scoring.KVCeiling,
|
|
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
|
|
}
|
|
// The metrics come first because the prefix scorer reports into them, and
|
|
// the decision metric is the whole difference between tuning this and
|
|
// guessing at it.
|
|
metrics := newMetrics()
|
|
var prefixScorer *scheduler.PrefixScorer
|
|
if cfg.Router.Prefix.Enabled {
|
|
prefixScorer = scheduler.NewPrefixScorer(
|
|
prefix.New(prefix.Config{
|
|
MaxNodes: cfg.Router.Prefix.MaxNodes,
|
|
MinMatch: cfg.Router.Prefix.MinMatch,
|
|
TTL: cfg.Router.Prefix.TTL,
|
|
}),
|
|
prefix.Thresholds{
|
|
MatchRatio: cfg.Router.Prefix.MatchRatio,
|
|
BalanceAbs: cfg.Router.Prefix.BalanceAbs,
|
|
BalanceRel: cfg.Router.Prefix.BalanceRel,
|
|
},
|
|
metrics.observePrefix)
|
|
sc.Prefix = prefixScorer
|
|
}
|
|
|
|
profiles := map[string]*scheduler.Profile{}
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
profiles[name] = scheduler.Builtin(name, sc)
|
|
}
|
|
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
|
|
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
|
|
}
|
|
|
|
r := &Router{
|
|
cfg: cfg.Router,
|
|
landing: cfg.Landing,
|
|
log: log,
|
|
reg: reg,
|
|
profiles: profiles,
|
|
prefix: prefixScorer,
|
|
m: metrics,
|
|
px: proxy.New(proxy.Config{
|
|
DialTimeout: cfg.Router.Timeouts.Dial,
|
|
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
|
|
GenerationTimeout: cfg.Router.Timeouts.Generation,
|
|
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
|
|
IdleTimeout: cfg.Router.Timeouts.Idle,
|
|
TotalTimeout: cfg.Router.Timeouts.Total,
|
|
MaxResponseBytes: cfg.Router.MaxBodyBytes,
|
|
}),
|
|
}
|
|
r.poller = scrape.NewPoller(reg, scrape.Config{
|
|
Interval: cfg.Router.ScrapeInterval,
|
|
Timeout: cfg.Router.ScrapeTimeout,
|
|
}, log)
|
|
|
|
if cfg.Auth.Enabled {
|
|
st, err := storepkg.Open(context.Background(), cfg.Auth.Store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("auth store: %w", err)
|
|
}
|
|
r.store = st
|
|
// Applied where no scope set one, so a project created without limits
|
|
// is still metered. Zero leaves it unlimited, which was the old
|
|
// behaviour and is now a choice rather than an omission.
|
|
var defRPM, defTPM *int
|
|
if cfg.Auth.DefaultRPM > 0 {
|
|
defRPM = &cfg.Auth.DefaultRPM
|
|
}
|
|
if cfg.Auth.DefaultTPM > 0 {
|
|
defTPM = &cfg.Auth.DefaultTPM
|
|
}
|
|
st.SetDefaultLimits(defRPM, defTPM)
|
|
st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal))
|
|
r.personalOff = cfg.Auth.Personal.Disabled
|
|
r.personalDefaults = personalLimits(cfg.Auth.Personal)
|
|
r.llamaVersion = cfg.WantedEngines()["llamacpp"]
|
|
|
|
r.auth = auth.New(st, auth.Config{
|
|
TTL: cfg.Auth.CacheTTL,
|
|
NegativeTTL: cfg.Auth.NegativeTTL,
|
|
})
|
|
|
|
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
|
|
for name, p := range cfg.Auth.Pricing {
|
|
prices[name] = accounting.Price{
|
|
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k,
|
|
}
|
|
}
|
|
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
|
|
FlushInterval: cfg.Auth.FlushInterval,
|
|
BatchSize: cfg.Auth.BatchSize,
|
|
Capacity: cfg.Auth.BufferSize,
|
|
}, log)
|
|
r.accounts = account.New(st, account.Config{
|
|
SignupOpen: cfg.Auth.SignupOpen,
|
|
}, log)
|
|
|
|
r.acct = accounting.New(
|
|
accounting.NewPricing(prices, accounting.Price{
|
|
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
|
|
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
|
|
}),
|
|
r.recorder, log)
|
|
r.threads = r.store
|
|
} else if cfg.Threads.Store != "" {
|
|
st, err := storepkg.Open(context.Background(), cfg.Threads.Store)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("threads store: %w", err)
|
|
}
|
|
if err := st.EnsureAnonymousProject(context.Background(), anonymousProject, cfg.Threads.RetainHours); err != nil {
|
|
_ = st.Close()
|
|
return nil, fmt.Errorf("threads store: %w", err)
|
|
}
|
|
r.threads, r.anonymous, r.ownsThreads = st, true, true
|
|
}
|
|
|
|
if cfg.Router.Admission.Enabled {
|
|
adm, err := admission.New(admissionConfig(cfg.Router), r.saturation)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("admission: %w", err)
|
|
}
|
|
r.adm = adm
|
|
}
|
|
|
|
// Seeded from configuration, not only pushed. A router running without a
|
|
// control plane — a static endpoint list, which is a supported mode — has
|
|
// nothing to push it a catalog, and would otherwise resolve no model at
|
|
// all. A control plane pushes over the top of this at startup and on every
|
|
// change after.
|
|
if len(cfg.Models) > 0 {
|
|
seed := map[string]catalog.Model{}
|
|
for _, m := range cfg.Models {
|
|
seed[m.Name] = m
|
|
for _, a := range m.Aliases {
|
|
seed[a] = m
|
|
}
|
|
}
|
|
r.models.Store(&seed)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func admissionConfig(rc config.Router) admission.Config {
|
|
bands := make([]admission.Band, 0, len(rc.Admission.Bands))
|
|
for _, b := range rc.Admission.Bands {
|
|
bands = append(bands, admission.Band{
|
|
Name: b.Name,
|
|
Priority: b.Priority,
|
|
SaturationCeiling: b.SaturationCeiling,
|
|
})
|
|
}
|
|
return admission.Config{
|
|
Bands: bands,
|
|
DefaultBand: rc.Admission.DefaultBand,
|
|
Thresholds: admission.Thresholds{
|
|
// Admission and the queue scorer must agree on what a healthy
|
|
// buffer is, or the two layers fight each other.
|
|
TargetWaitingPerReplica: rc.Scoring.TargetWaiting,
|
|
KVCeiling: rc.Scoring.KVCeiling,
|
|
},
|
|
QueueTTL: rc.Admission.QueueTTL,
|
|
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
|
|
Interval: rc.Admission.Interval,
|
|
}
|
|
}
|
|
|
|
// saturation reports the pool pressure for a model, for the admission layer.
|
|
//
|
|
// outstanding is what admission has let through. It is compared against the
|
|
// registry's inflight counters rather than the scraped engine stats, because
|
|
// inflight updates the instant a request is dispatched while scraped stats lag
|
|
// by a whole interval. Comparing against scraped data would make every
|
|
// dispatched-but-unscraped request look like queue depth and throttle the
|
|
// steady state down to a few requests per scrape.
|
|
//
|
|
// What is left over — admitted but not yet dispatched — is the thundering herd:
|
|
// a burst that passed the check together before any of it became visible.
|
|
func (r *Router) saturation(model string, outstanding int) (float64, bool) {
|
|
eps := r.reg.Snapshots(model)
|
|
if len(eps) == 0 {
|
|
return 0, false
|
|
}
|
|
dispatched := 0
|
|
for _, e := range eps {
|
|
dispatched += e.Inflight
|
|
}
|
|
return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{
|
|
TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting,
|
|
KVCeiling: r.cfg.Scoring.KVCeiling,
|
|
}), true
|
|
}
|
|
|
|
func toInfos(eps []config.Endpoint) []endpoint.Info {
|
|
out := make([]endpoint.Info, 0, len(eps))
|
|
for _, e := range eps {
|
|
out = append(out, endpoint.Info{
|
|
ID: e.ID,
|
|
URL: strings.TrimRight(e.URL, "/"),
|
|
Models: e.Models,
|
|
Engine: e.Engine,
|
|
Pool: e.Pool,
|
|
Weight: e.Weight,
|
|
Priority: e.Priority,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Start begins scraping endpoint metrics and dispatching admissions.
|
|
// It returns immediately.
|
|
func (r *Router) Start(ctx context.Context) {
|
|
r.poller.Sync(ctx)
|
|
if r.adm != nil {
|
|
r.adm.Start()
|
|
}
|
|
if r.recorder != nil {
|
|
r.recorder.Start(ctx)
|
|
}
|
|
if r.acct != nil && r.store != nil {
|
|
// Budgets are counted in memory; the store is what makes them survive a
|
|
// restart. A failure here is not fatal — refusing to start because last
|
|
// month's spend could not be read would be worse than starting with
|
|
// counters that are low.
|
|
if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil {
|
|
r.log.Warn("could not restore budgets; they start from zero this period", "err", err)
|
|
}
|
|
}
|
|
if r.threads != nil {
|
|
go r.pruneThreads(ctx)
|
|
}
|
|
}
|
|
|
|
// pruneThreads enforces each project's retention window.
|
|
//
|
|
// Without something running this, "we keep conversations for 24 hours" is a
|
|
// comment rather than a promise — the rows would simply accumulate, and the
|
|
// number a project chose would describe nothing.
|
|
//
|
|
// Hourly, and once at startup. A gateway that has been down for a week should
|
|
// not wait another hour before honouring a window that expired during it, and
|
|
// an hour is fine granularity for a limit expressed in hours.
|
|
func (r *Router) pruneThreads(ctx context.Context) {
|
|
prune := func() {
|
|
n, err := r.threads.PruneThreads(ctx, time.Now())
|
|
if err != nil {
|
|
r.log.Warn("could not prune expired conversations", "err", err)
|
|
return
|
|
}
|
|
if n > 0 {
|
|
r.log.Info("removed conversations past their retention", "threads", n)
|
|
}
|
|
m, err := r.threads.PruneMemory(ctx, time.Now())
|
|
if err != nil {
|
|
r.log.Warn("could not prune expired memory notes", "err", err)
|
|
return
|
|
}
|
|
if m > 0 {
|
|
r.log.Info("removed memory notes past their retention", "notes", m)
|
|
}
|
|
}
|
|
prune()
|
|
|
|
t := time.NewTicker(time.Hour)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
prune()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop halts scraping and admission.
|
|
func (r *Router) Stop() {
|
|
r.poller.Stop()
|
|
if r.adm != nil {
|
|
r.adm.Close()
|
|
}
|
|
// Drain before closing the database, or the last batch has nowhere to go.
|
|
if r.recorder != nil {
|
|
r.recorder.Stop()
|
|
}
|
|
if r.store != nil {
|
|
_ = r.store.Close()
|
|
}
|
|
if r.ownsThreads {
|
|
_ = r.threads.Close()
|
|
}
|
|
}
|
|
|
|
// Registry exposes the endpoint registry, for tests and admin endpoints.
|
|
func (r *Router) Registry() *endpoint.Registry { return r.reg }
|
|
|
|
// SetEndpoints replaces the endpoint list and resynchronizes metric scraping.
|
|
//
|
|
// This is how the control plane feeds the router from phase 3 onwards, taking
|
|
// over from the static configuration list (plan §6.4). Endpoints that survive
|
|
// the update keep their live state — inflight counts, breaker state, drain rate
|
|
// — so a replica that was already serving does not start from cold just
|
|
// because another one moved.
|
|
// modelMap returns the name→model map in force.
|
|
//
|
|
// Never nil: a router built before any catalog arrived resolves nothing, which
|
|
// is the same answer it gave for an unknown name before.
|
|
func (r *Router) modelMap() map[string]catalog.Model {
|
|
if m := r.models.Load(); m != nil {
|
|
return *m
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// SetCatalog replaces the models this router knows about.
|
|
//
|
|
// The catalog's counterpart to SetEndpoints, and pushed for the same reason:
|
|
// the control plane owns what should exist, the router needs to know, and
|
|
// neither has to be in the other's process for that to work.
|
|
func (r *Router) SetCatalog(c *catalog.Catalog) {
|
|
m := map[string]catalog.Model{}
|
|
for _, entry := range c.Models() {
|
|
m[entry.Name] = entry
|
|
for _, a := range entry.Aliases {
|
|
m[a] = entry
|
|
}
|
|
}
|
|
r.models.Store(&m)
|
|
}
|
|
|
|
// LastUsed reports when each endpoint was last sent a request, satisfying
|
|
// control.UsageSource so the control plane can tell a busy replica from one
|
|
// holding its cards out of habit.
|
|
func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() }
|
|
|
|
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
|
|
if r.prefix != nil {
|
|
// A replica that has gone takes its cache with it. Leaving its entries
|
|
// in the tree would steer traffic towards something that is not there,
|
|
// and since nothing reuses a replica id the entries would never be
|
|
// corrected — only expire, minutes later.
|
|
before := map[string]struct{}{}
|
|
for _, s := range r.reg.Snapshots("") {
|
|
before[s.ID] = struct{}{}
|
|
}
|
|
for _, i := range infos {
|
|
delete(before, i.ID)
|
|
}
|
|
for id := range before {
|
|
r.prefix.Forget(id)
|
|
}
|
|
}
|
|
|
|
r.reg.Replace(infos)
|
|
r.poller.Sync(ctx)
|
|
}
|
|
|
|
// --- model resolution ---
|
|
|
|
// resolve maps a client-supplied model name onto the catalog. Unknown names are
|
|
// passed through unchanged: an endpoint may serve a model the catalog does not
|
|
// list, and refusing it would be unhelpful.
|
|
func (r *Router) resolve(name string) (canonical string, m catalog.Model) {
|
|
if e, ok := r.modelMap()[name]; ok {
|
|
return e.Name, e
|
|
}
|
|
return name, catalog.Model{Name: name}
|
|
}
|
|
|
|
func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile {
|
|
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
|
|
if p, ok := r.profiles[name]; ok && name != "" {
|
|
return p
|
|
}
|
|
}
|
|
return r.profiles["chat"]
|
|
}
|
|
|
|
// --- dispatch ---
|
|
|
|
// attempt describes one upstream try, for logging and metrics.
|
|
type dispatchResult struct {
|
|
endpointID string
|
|
pool string
|
|
servedModel
|
|
outcome proxy.Outcome
|
|
err error
|
|
}
|
|
|
|
type servedModel struct {
|
|
model string
|
|
// fallback is true when the served model is not the one that was requested.
|
|
fallback bool
|
|
}
|
|
|
|
// dispatch resolves the model, then tries endpoints until one works, the client
|
|
// has been written to, or the attempt budget runs out.
|
|
//
|
|
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
|
|
// client. Once bytes are out, a retry would splice two different responses
|
|
// together, which is worse than the error the client would otherwise see.
|
|
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
|
|
canonical, entry := r.resolve(req.Model)
|
|
|
|
// The requested model first, then its fallbacks. A fallback is a different
|
|
// model, so the client is told about it via x-llmgw-served-model.
|
|
chain := append([]string{canonical}, entry.Fallbacks...)
|
|
|
|
var last dispatchResult
|
|
for i, model := range chain {
|
|
if i > 0 {
|
|
// Before dispatching, because dispatchOne writes the response and
|
|
// a header set after that is a header nobody receives. It was set
|
|
// in finish until now, which is after the proxy has already called
|
|
// WriteHeader — so this has announced a fallback to nobody since it
|
|
// was written.
|
|
w.Header().Set("x-llmgw-served-model", model)
|
|
}
|
|
res := r.dispatchOne(ctx, w, req, model, entry)
|
|
res.model = model
|
|
res.fallback = i > 0
|
|
|
|
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
|
|
return res
|
|
}
|
|
last = res
|
|
// Only a total absence of capacity justifies moving to another model;
|
|
// a bad request would fail identically there.
|
|
if !errors.Is(res.err, errNoCapacity) {
|
|
return res
|
|
}
|
|
}
|
|
return last
|
|
}
|
|
|
|
var errNoCapacity = errors.New("no endpoint available")
|
|
|
|
// errLoading means the model is being started and the wait ran out.
|
|
//
|
|
// Distinct from having no capacity because the right client behaviour differs:
|
|
// this one succeeds on a retry, and the other does not.
|
|
var errLoading = errors.New("model loading")
|
|
|
|
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
|
|
candidates := r.reg.Snapshots(model)
|
|
if len(candidates) == 0 {
|
|
// Nothing is serving it. That may be because nothing ever will, or
|
|
// because it loads when asked and nobody has asked yet — and the second
|
|
// is not an error, it is the feature working.
|
|
if r.activate(ctx, model) {
|
|
candidates = r.reg.Snapshots(model)
|
|
}
|
|
}
|
|
if len(candidates) == 0 {
|
|
if r.activator != nil && r.activator.Want(model) {
|
|
// Asked for, still loading. Saying "no endpoint serves this" would
|
|
// be a lie, and a client told to retry will succeed where one told
|
|
// the model does not exist gives up.
|
|
return dispatchResult{err: fmt.Errorf("%w: %s", errLoading,
|
|
"the model is loading; retry shortly")}
|
|
}
|
|
return dispatchResult{err: fmt.Errorf("%w: %s", errNoCapacity, noEndpointFor(model, r.servedModels()))}
|
|
}
|
|
|
|
profile := r.profileFor(entry, req.ProfileOverride)
|
|
sctx := &scheduler.Ctx{
|
|
Now: time.Now(),
|
|
Req: scheduler.Request{
|
|
Model: model,
|
|
SessionID: req.SessionID,
|
|
PromptPrefix: req.PromptPrefix,
|
|
PromptTokens: req.PromptTokens,
|
|
MaxTokens: req.MaxTokens,
|
|
},
|
|
}
|
|
|
|
attempts := max(r.cfg.Retries+1, 1)
|
|
var last dispatchResult
|
|
|
|
for range attempts {
|
|
if len(candidates) == 0 {
|
|
break
|
|
}
|
|
dec, err := profile.Schedule(sctx, candidates)
|
|
if err != nil {
|
|
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
|
|
break
|
|
}
|
|
|
|
release, granted := r.reg.Acquire(dec.Endpoint.ID)
|
|
if !granted {
|
|
// The breaker opened between scoring and dispatch; drop it and
|
|
// score again rather than failing the request.
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
continue
|
|
}
|
|
|
|
out, perr := r.px.Do(ctx, w, proxy.Request{
|
|
URL: dec.Endpoint.URL + req.Path,
|
|
Body: req.UpstreamBody,
|
|
Header: req.Header,
|
|
Stream: req.Stream,
|
|
SuppressUsageChunk: req.SuppressUsageChunk,
|
|
CaptureReply: req.captureBytes(),
|
|
})
|
|
// A retryable failure is the endpoint's fault; a client disconnect is not.
|
|
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
|
|
|
|
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
|
|
r.m.observeScores(dec)
|
|
|
|
// Recorded after the decision, not during it: the scorer proposes and
|
|
// the picker disposes, so recording the proposal would fill the tree
|
|
// with replicas that were never sent anything — the one thing that
|
|
// would make the estimate worse than no estimate at all.
|
|
if r.prefix != nil {
|
|
r.prefix.Observe(req.PromptPrefix, dec.Endpoint.ID)
|
|
}
|
|
|
|
if perr == nil || out.Wrote || !out.Retryable {
|
|
return res
|
|
}
|
|
last = res
|
|
r.log.Warn("upstream attempt failed, trying another endpoint",
|
|
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
}
|
|
|
|
if last.err == nil {
|
|
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
|
|
} else if !errors.Is(last.err, errNoCapacity) {
|
|
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
|
|
}
|
|
return last
|
|
}
|
|
|
|
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
|
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
|
}
|
|
|
|
// personalLimits turns the personal block into limits, nil when it sets none.
|
|
func personalLimits(p config.Personal) *storepkg.Limits {
|
|
var l storepkg.Limits
|
|
if p.RPM > 0 {
|
|
v := p.RPM
|
|
l.RPM = &v
|
|
}
|
|
if p.TPM > 0 {
|
|
v := p.TPM
|
|
l.TPM = &v
|
|
}
|
|
if p.Budget > 0 {
|
|
v := int64(math.Round(p.Budget * 1e6))
|
|
l.Budget = &v
|
|
l.BudgetPeriod = storepkg.PeriodMonth
|
|
if p.Period == "day" {
|
|
l.BudgetPeriod = storepkg.PeriodDay
|
|
}
|
|
}
|
|
if l.RPM == nil && l.TPM == nil && l.Budget == nil {
|
|
return nil
|
|
}
|
|
return &l
|
|
}
|