Files
llmgw/internal/router/router.go
T
Ilker ManapandClaude Opus 5 8d35faf6f8 Give the landing page somewhere to send people
The page explained the API, said keys come from an operator, and left the
reader there. Someone who reaches the address without a key had no route to
getting one, which is where onboarding stopped.

`landing.contact` is that route, because the gateway cannot guess who to ask.
An address becomes a mailto link, an http(s) URL stays a link, and anything
else — a channel, a queue, a person — is shown as text. Other schemes are
never linked: this string comes from configuration, configuration is not
always written by whoever runs the gateway, and a javascript: href on a page
served to every visitor is a short path to a stored XSS. Escaping the text
does not make a scheme safe.

Unconfigured, it now says something rather than nothing, and the page explains
that a key arrives as a one-time link so the reader knows what to expect
before it lands.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 09:22:42 +02:00

426 lines
13 KiB
Go

// Package router wires the pieces into a working OpenAI-compatible gateway:
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
//
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
// breaker); this package holds the orchestration and the HTTP surface.
package router
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Router serves the OpenAI API in front of a set of engine replicas.
type Router struct {
cfg config.Router
log *slog.Logger
reg *endpoint.Registry
poller *scrape.Poller
px *proxy.Proxy
m *metrics
// adm is nil when admission control is disabled.
adm *admission.Controller
profiles map[string]*scheduler.Profile
// models maps every name a client may use — canonical names and aliases —
// onto its catalog entry.
models map[string]catalog.Model
// landing configures the public page, which the router serves but does not
// otherwise care about.
landing config.Landing
// auth, store, acct and recorder are nil when authentication is disabled.
auth *auth.Authenticator
store *storepkg.SQLite
acct *accounting.Accountant
recorder *accounting.Recorder
}
// Auth exposes the authenticator so ctl and the batch writer can reach it.
// Nil when authentication is off.
func (r *Router) Auth() *auth.Authenticator { return r.auth }
// Store exposes the key store, or nil when authentication is off.
func (r *Router) Store() *storepkg.SQLite { return r.store }
// New builds a router from configuration. Start must be called to begin scraping.
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
if log == nil {
log = slog.Default()
}
reg := endpoint.NewRegistry(endpoint.Config{
StaleAfter: cfg.Router.StaleAfter,
Breaker: breaker.Config{
FailureThreshold: cfg.Router.FailureThreshold,
BaseCooldown: cfg.Router.Cooldown,
},
})
reg.Replace(toInfos(cfg.Endpoints))
sc := scheduler.ProfileConfig{
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
TargetInflight: cfg.Router.Scoring.TargetInflight,
KVCeiling: cfg.Router.Scoring.KVCeiling,
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
}
profiles := map[string]*scheduler.Profile{}
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
profiles[name] = scheduler.Builtin(name, sc)
}
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
}
models := map[string]catalog.Model{}
for _, m := range cfg.Models {
models[m.Name] = m
for _, a := range m.Aliases {
models[a] = m
}
}
r := &Router{
cfg: cfg.Router,
landing: cfg.Landing,
log: log,
reg: reg,
profiles: profiles,
models: models,
m: newMetrics(),
px: proxy.New(proxy.Config{
DialTimeout: cfg.Router.Timeouts.Dial,
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
IdleTimeout: cfg.Router.Timeouts.Idle,
TotalTimeout: cfg.Router.Timeouts.Total,
MaxResponseBytes: cfg.Router.MaxBodyBytes,
}),
}
r.poller = scrape.NewPoller(reg, scrape.Config{
Interval: cfg.Router.ScrapeInterval,
Timeout: cfg.Router.ScrapeTimeout,
}, log)
if cfg.Auth.Enabled {
st, err := storepkg.OpenSQLite(context.Background(), cfg.Auth.Store)
if err != nil {
return nil, fmt.Errorf("auth store: %w", err)
}
r.store = st
r.auth = auth.New(st, auth.Config{
TTL: cfg.Auth.CacheTTL,
NegativeTTL: cfg.Auth.NegativeTTL,
})
prices := make(map[string]accounting.Price, len(cfg.Auth.Pricing))
for name, p := range cfg.Auth.Pricing {
prices[name] = accounting.Price{
PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k,
}
}
r.recorder = accounting.NewRecorder(st, r.auth, accounting.RecorderConfig{
FlushInterval: cfg.Auth.FlushInterval,
BatchSize: cfg.Auth.BatchSize,
Capacity: cfg.Auth.BufferSize,
}, log)
r.acct = accounting.New(
accounting.NewPricing(prices, accounting.Price{
PromptPer1k: cfg.Auth.DefaultPrice.PromptPer1k,
CompletionPer1k: cfg.Auth.DefaultPrice.CompletionPer1k,
}),
r.recorder, log)
}
if cfg.Router.Admission.Enabled {
adm, err := admission.New(admissionConfig(cfg.Router), r.saturation)
if err != nil {
return nil, fmt.Errorf("admission: %w", err)
}
r.adm = adm
}
return r, nil
}
func admissionConfig(rc config.Router) admission.Config {
bands := make([]admission.Band, 0, len(rc.Admission.Bands))
for _, b := range rc.Admission.Bands {
bands = append(bands, admission.Band{
Name: b.Name,
Priority: b.Priority,
SaturationCeiling: b.SaturationCeiling,
})
}
return admission.Config{
Bands: bands,
DefaultBand: rc.Admission.DefaultBand,
Thresholds: admission.Thresholds{
// Admission and the queue scorer must agree on what a healthy
// buffer is, or the two layers fight each other.
TargetWaitingPerReplica: rc.Scoring.TargetWaiting,
KVCeiling: rc.Scoring.KVCeiling,
},
QueueTTL: rc.Admission.QueueTTL,
QueueCapacityPerBand: rc.Admission.QueueCapacityPerBand,
Interval: rc.Admission.Interval,
}
}
// saturation reports the pool pressure for a model, for the admission layer.
//
// outstanding is what admission has let through. It is compared against the
// registry's inflight counters rather than the scraped engine stats, because
// inflight updates the instant a request is dispatched while scraped stats lag
// by a whole interval. Comparing against scraped data would make every
// dispatched-but-unscraped request look like queue depth and throttle the
// steady state down to a few requests per scrape.
//
// What is left over — admitted but not yet dispatched — is the thundering herd:
// a burst that passed the check together before any of it became visible.
func (r *Router) saturation(model string, outstanding int) (float64, bool) {
eps := r.reg.Snapshots(model)
if len(eps) == 0 {
return 0, false
}
dispatched := 0
for _, e := range eps {
dispatched += e.Inflight
}
return admission.Saturation(eps, max(0, outstanding-dispatched), admission.Thresholds{
TargetWaitingPerReplica: r.cfg.Scoring.TargetWaiting,
KVCeiling: r.cfg.Scoring.KVCeiling,
}), true
}
func toInfos(eps []config.Endpoint) []endpoint.Info {
out := make([]endpoint.Info, 0, len(eps))
for _, e := range eps {
out = append(out, endpoint.Info{
ID: e.ID,
URL: strings.TrimRight(e.URL, "/"),
Models: e.Models,
Engine: e.Engine,
Pool: e.Pool,
Weight: e.Weight,
Priority: e.Priority,
})
}
return out
}
// Start begins scraping endpoint metrics and dispatching admissions.
// It returns immediately.
func (r *Router) Start(ctx context.Context) {
r.poller.Sync(ctx)
if r.adm != nil {
r.adm.Start()
}
if r.recorder != nil {
r.recorder.Start(ctx)
}
if r.acct != nil && r.store != nil {
// Budgets are counted in memory; the store is what makes them survive a
// restart. A failure here is not fatal — refusing to start because last
// month's spend could not be read would be worse than starting with
// counters that are low.
if err := r.acct.RestoreBudgets(ctx, r.store, storepkg.PeriodMonth); err != nil {
r.log.Warn("could not restore budgets; they start from zero this period", "err", err)
}
}
}
// Stop halts scraping and admission.
func (r *Router) Stop() {
r.poller.Stop()
if r.adm != nil {
r.adm.Close()
}
// Drain before closing the database, or the last batch has nowhere to go.
if r.recorder != nil {
r.recorder.Stop()
}
if r.store != nil {
_ = r.store.Close()
}
}
// Registry exposes the endpoint registry, for tests and admin endpoints.
func (r *Router) Registry() *endpoint.Registry { return r.reg }
// SetEndpoints replaces the endpoint list and resynchronizes metric scraping.
//
// This is how the control plane feeds the router from phase 3 onwards, taking
// over from the static configuration list (plan §6.4). Endpoints that survive
// the update keep their live state — inflight counts, breaker state, drain rate
// — so a replica that was already serving does not start from cold just
// because another one moved.
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
r.reg.Replace(infos)
r.poller.Sync(ctx)
}
// --- model resolution ---
// resolve maps a client-supplied model name onto the catalog. Unknown names are
// passed through unchanged: an endpoint may serve a model the catalog does not
// list, and refusing it would be unhelpful.
func (r *Router) resolve(name string) (canonical string, m catalog.Model) {
if e, ok := r.models[name]; ok {
return e.Name, e
}
return name, catalog.Model{Name: name}
}
func (r *Router) profileFor(m catalog.Model, override string) *scheduler.Profile {
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
if p, ok := r.profiles[name]; ok && name != "" {
return p
}
}
return r.profiles["chat"]
}
// --- dispatch ---
// attempt describes one upstream try, for logging and metrics.
type dispatchResult struct {
endpointID string
pool string
servedModel
outcome proxy.Outcome
err error
}
type servedModel struct {
model string
// fallback is true when the served model is not the one that was requested.
fallback bool
}
// dispatch resolves the model, then tries endpoints until one works, the client
// has been written to, or the attempt budget runs out.
//
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
// client. Once bytes are out, a retry would splice two different responses
// together, which is worse than the error the client would otherwise see.
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
canonical, entry := r.resolve(req.Model)
// The requested model first, then its fallbacks. A fallback is a different
// model, so the client is told about it via x-llmgw-served-model.
chain := append([]string{canonical}, entry.Fallbacks...)
var last dispatchResult
for i, model := range chain {
res := r.dispatchOne(ctx, w, req, model, entry)
res.model = model
res.fallback = i > 0
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
return res
}
last = res
// Only a total absence of capacity justifies moving to another model;
// a bad request would fail identically there.
if !errors.Is(res.err, errNoCapacity) {
return res
}
}
return last
}
var errNoCapacity = errors.New("no endpoint available")
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry catalog.Model) dispatchResult {
candidates := r.reg.Snapshots(model)
if len(candidates) == 0 {
return dispatchResult{err: fmt.Errorf("%w: no endpoint serves %q", errNoCapacity, model)}
}
profile := r.profileFor(entry, req.ProfileOverride)
sctx := &scheduler.Ctx{
Now: time.Now(),
Req: scheduler.Request{
Model: model,
SessionID: req.SessionID,
PromptPrefix: req.PromptPrefix,
PromptTokens: req.PromptTokens,
MaxTokens: req.MaxTokens,
},
}
attempts := max(r.cfg.Retries+1, 1)
var last dispatchResult
for range attempts {
if len(candidates) == 0 {
break
}
dec, err := profile.Schedule(sctx, candidates)
if err != nil {
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
break
}
release, granted := r.reg.Acquire(dec.Endpoint.ID)
if !granted {
// The breaker opened between scoring and dispatch; drop it and
// score again rather than failing the request.
candidates = without(candidates, dec.Endpoint.ID)
continue
}
out, perr := r.px.Do(ctx, w, proxy.Request{
URL: dec.Endpoint.URL + req.Path,
Body: req.UpstreamBody,
Header: req.Header,
Stream: req.Stream,
SuppressUsageChunk: req.SuppressUsageChunk,
})
// A retryable failure is the endpoint's fault; a client disconnect is not.
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
r.m.observeScores(dec)
if perr == nil || out.Wrote || !out.Retryable {
return res
}
last = res
r.log.Warn("upstream attempt failed, trying another endpoint",
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
candidates = without(candidates, dec.Endpoint.ID)
}
if last.err == nil {
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
} else if !errors.Is(last.err, errNoCapacity) {
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
}
return last
}
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
}