CI / test (push) Successful in 1m1s
llmgw server is real now. It fronts a static set of engine replicas with an OpenAI-compatible API, scoring endpoints through the pipeline added in the previous commit. - proxy: SSE relayed event by event with a flush after each, cancellation propagated upstream, and separate deadlines for dial, headers, first token, inter-token idle and total. The idle watchdog is what distinguishes a wedged engine from a slow one. - router: dispatch with retry and model fallback. Retry only happens while nothing has reached the client; after that, splicing two responses together would be worse than the short stream the client already sees. - config: YAML via koanf overlaid on defaults, with LLMGW_ env overrides. Validation rejects duplicate endpoint ids and aliases that shadow a model, because both misroute silently at request time rather than failing loudly. - metrics: requests, duration, TTFT, tokens, plus per-scorer values and per-filter drops on a separate admin listener with health and pprof. Phase 1 exit criterion, measured in test/e2e: 1000 concurrent streams across 3 sim-engines complete at ~6800 req/s; every client disconnect frees the KV blocks on every engine; router overhead p99 is about 0.9 ms against the plan's 2 ms budget. The handlers live in internal/router rather than an internal/router/openai package as the plan's tree sketched: they need the dispatch loop, and splitting them apart would have meant an interface with exactly one implementation. Plan §4 updated to match. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
271 lines
8.1 KiB
Go
271 lines
8.1 KiB
Go
// Package router wires the pieces into a working OpenAI-compatible gateway:
|
|
// resolve the model, pick an endpoint, proxy the stream, retry when it is safe.
|
|
//
|
|
// The sub-packages hold the reusable parts (scheduler, endpoint, scrape, proxy,
|
|
// breaker); this package holds the orchestration and the HTTP surface.
|
|
package router
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/breaker"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scheduler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/scrape"
|
|
)
|
|
|
|
// Router serves the OpenAI API in front of a set of engine replicas.
|
|
type Router struct {
|
|
cfg config.Router
|
|
log *slog.Logger
|
|
|
|
reg *endpoint.Registry
|
|
poller *scrape.Poller
|
|
px *proxy.Proxy
|
|
m *metrics
|
|
|
|
profiles map[string]*scheduler.Profile
|
|
// models maps every name a client may use — canonical names and aliases —
|
|
// onto its catalog entry.
|
|
models map[string]config.Model
|
|
}
|
|
|
|
// New builds a router from configuration. Start must be called to begin scraping.
|
|
func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
|
|
reg := endpoint.NewRegistry(endpoint.Config{
|
|
StaleAfter: cfg.Router.StaleAfter,
|
|
Breaker: breaker.Config{
|
|
FailureThreshold: cfg.Router.FailureThreshold,
|
|
BaseCooldown: cfg.Router.Cooldown,
|
|
},
|
|
})
|
|
reg.Replace(toInfos(cfg.Endpoints))
|
|
|
|
sc := scheduler.ProfileConfig{
|
|
TargetWaiting: cfg.Router.Scoring.TargetWaiting,
|
|
TargetInflight: cfg.Router.Scoring.TargetInflight,
|
|
KVCeiling: cfg.Router.Scoring.KVCeiling,
|
|
SessionLoadFactor: cfg.Router.Scoring.SessionLoadFactor,
|
|
}
|
|
profiles := map[string]*scheduler.Profile{}
|
|
for _, name := range []string{"chat", "batch", "long-context", "debug"} {
|
|
profiles[name] = scheduler.Builtin(name, sc)
|
|
}
|
|
if _, ok := profiles[cfg.Router.DefaultProfile]; !ok {
|
|
return nil, fmt.Errorf("unknown default_profile %q", cfg.Router.DefaultProfile)
|
|
}
|
|
|
|
models := map[string]config.Model{}
|
|
for _, m := range cfg.Models {
|
|
models[m.Name] = m
|
|
for _, a := range m.Aliases {
|
|
models[a] = m
|
|
}
|
|
}
|
|
|
|
r := &Router{
|
|
cfg: cfg.Router,
|
|
log: log,
|
|
reg: reg,
|
|
profiles: profiles,
|
|
models: models,
|
|
m: newMetrics(),
|
|
px: proxy.New(proxy.Config{
|
|
DialTimeout: cfg.Router.Timeouts.Dial,
|
|
ResponseHeaderTimeout: cfg.Router.Timeouts.ResponseHeader,
|
|
FirstTokenTimeout: cfg.Router.Timeouts.FirstToken,
|
|
IdleTimeout: cfg.Router.Timeouts.Idle,
|
|
TotalTimeout: cfg.Router.Timeouts.Total,
|
|
MaxResponseBytes: cfg.Router.MaxBodyBytes,
|
|
}),
|
|
}
|
|
r.poller = scrape.NewPoller(reg, scrape.Config{
|
|
Interval: cfg.Router.ScrapeInterval,
|
|
Timeout: cfg.Router.ScrapeTimeout,
|
|
}, log)
|
|
|
|
return r, nil
|
|
}
|
|
|
|
func toInfos(eps []config.Endpoint) []endpoint.Info {
|
|
out := make([]endpoint.Info, 0, len(eps))
|
|
for _, e := range eps {
|
|
out = append(out, endpoint.Info{
|
|
ID: e.ID,
|
|
URL: strings.TrimRight(e.URL, "/"),
|
|
Models: e.Models,
|
|
Pool: e.Pool,
|
|
Weight: e.Weight,
|
|
Priority: e.Priority,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Start begins scraping endpoint metrics. It returns immediately.
|
|
func (r *Router) Start(ctx context.Context) { r.poller.Sync(ctx) }
|
|
|
|
// Stop halts scraping.
|
|
func (r *Router) Stop() { r.poller.Stop() }
|
|
|
|
// Registry exposes the endpoint registry, for tests and admin endpoints.
|
|
func (r *Router) Registry() *endpoint.Registry { return r.reg }
|
|
|
|
// --- model resolution ---
|
|
|
|
// resolve maps a client-supplied model name onto the catalog. Unknown names are
|
|
// passed through unchanged: an endpoint may serve a model the catalog does not
|
|
// list, and refusing it would be unhelpful.
|
|
func (r *Router) resolve(name string) (canonical string, m config.Model) {
|
|
if e, ok := r.models[name]; ok {
|
|
return e.Name, e
|
|
}
|
|
return name, config.Model{Name: name}
|
|
}
|
|
|
|
func (r *Router) profileFor(m config.Model, override string) *scheduler.Profile {
|
|
for _, name := range []string{override, m.RoutingProfile, r.cfg.DefaultProfile} {
|
|
if p, ok := r.profiles[name]; ok && name != "" {
|
|
return p
|
|
}
|
|
}
|
|
return r.profiles["chat"]
|
|
}
|
|
|
|
// --- dispatch ---
|
|
|
|
// attempt describes one upstream try, for logging and metrics.
|
|
type dispatchResult struct {
|
|
endpointID string
|
|
pool string
|
|
servedModel
|
|
outcome proxy.Outcome
|
|
err error
|
|
}
|
|
|
|
type servedModel struct {
|
|
model string
|
|
// fallback is true when the served model is not the one that was requested.
|
|
fallback bool
|
|
}
|
|
|
|
// dispatch resolves the model, then tries endpoints until one works, the client
|
|
// has been written to, or the attempt budget runs out.
|
|
//
|
|
// The retry rule (plan §7.7) is strict: retry only while nothing has reached the
|
|
// client. Once bytes are out, a retry would splice two different responses
|
|
// together, which is worse than the error the client would otherwise see.
|
|
func (r *Router) dispatch(ctx context.Context, w http.ResponseWriter, req *parsedRequest) dispatchResult {
|
|
canonical, entry := r.resolve(req.Model)
|
|
|
|
// The requested model first, then its fallbacks. A fallback is a different
|
|
// model, so the client is told about it via x-llmgw-served-model.
|
|
chain := append([]string{canonical}, entry.Fallbacks...)
|
|
|
|
var last dispatchResult
|
|
for i, model := range chain {
|
|
res := r.dispatchOne(ctx, w, req, model, entry)
|
|
res.model = model
|
|
res.fallback = i > 0
|
|
|
|
if res.err == nil || res.outcome.Wrote || errors.Is(res.err, proxy.ErrClientGone) {
|
|
return res
|
|
}
|
|
last = res
|
|
// Only a total absence of capacity justifies moving to another model;
|
|
// a bad request would fail identically there.
|
|
if !errors.Is(res.err, errNoCapacity) {
|
|
return res
|
|
}
|
|
}
|
|
return last
|
|
}
|
|
|
|
var errNoCapacity = errors.New("no endpoint available")
|
|
|
|
func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *parsedRequest, model string, entry config.Model) dispatchResult {
|
|
candidates := r.reg.Snapshots(model)
|
|
if len(candidates) == 0 {
|
|
return dispatchResult{err: fmt.Errorf("%w: no endpoint serves %q", errNoCapacity, model)}
|
|
}
|
|
|
|
profile := r.profileFor(entry, req.ProfileOverride)
|
|
sctx := &scheduler.Ctx{
|
|
Now: time.Now(),
|
|
Req: scheduler.Request{
|
|
Model: model,
|
|
SessionID: req.SessionID,
|
|
PromptPrefix: req.PromptPrefix,
|
|
PromptTokens: req.PromptTokens,
|
|
MaxTokens: req.MaxTokens,
|
|
},
|
|
}
|
|
|
|
attempts := max(r.cfg.Retries+1, 1)
|
|
var last dispatchResult
|
|
|
|
for range attempts {
|
|
if len(candidates) == 0 {
|
|
break
|
|
}
|
|
dec, err := profile.Schedule(sctx, candidates)
|
|
if err != nil {
|
|
last = dispatchResult{err: fmt.Errorf("%w: %v", errNoCapacity, err)}
|
|
break
|
|
}
|
|
|
|
release, granted := r.reg.Acquire(dec.Endpoint.ID)
|
|
if !granted {
|
|
// The breaker opened between scoring and dispatch; drop it and
|
|
// score again rather than failing the request.
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
continue
|
|
}
|
|
|
|
out, perr := r.px.Do(ctx, w, proxy.Request{
|
|
URL: dec.Endpoint.URL + req.Path,
|
|
Body: req.UpstreamBody,
|
|
Header: req.Header,
|
|
Stream: req.Stream,
|
|
SuppressUsageChunk: req.SuppressUsageChunk,
|
|
})
|
|
// A retryable failure is the endpoint's fault; a client disconnect is not.
|
|
release(perr == nil || errors.Is(perr, proxy.ErrClientGone))
|
|
|
|
res := dispatchResult{endpointID: dec.Endpoint.ID, pool: dec.Endpoint.Pool, outcome: out, err: perr}
|
|
r.m.observeScores(dec)
|
|
|
|
if perr == nil || out.Wrote || !out.Retryable {
|
|
return res
|
|
}
|
|
last = res
|
|
r.log.Warn("upstream attempt failed, trying another endpoint",
|
|
"endpoint", dec.Endpoint.ID, "model", model, "err", perr)
|
|
candidates = without(candidates, dec.Endpoint.ID)
|
|
}
|
|
|
|
if last.err == nil {
|
|
last.err = fmt.Errorf("%w: every endpoint for %q failed", errNoCapacity, model)
|
|
} else if !errors.Is(last.err, errNoCapacity) {
|
|
last.err = fmt.Errorf("%w: %v", errNoCapacity, last.err)
|
|
}
|
|
return last
|
|
}
|
|
|
|
func without(eps []endpoint.Snapshot, id string) []endpoint.Snapshot {
|
|
return slices.DeleteFunc(slices.Clone(eps), func(e endpoint.Snapshot) bool { return e.ID == id })
|
|
}
|