CI / test (push) Successful in 1m23s
response_header sounds like a check that the endpoint is alive, and for a streaming request it is: headers arrive before generation starts. For a non-streaming request the engine sends nothing until the completion is finished, so the same setting is the total generation budget wearing a liveness timeout's name. At 60s that was invisible on GPUs and a hard cap of roughly a hundred tokens on qwen2.5-coder-14b at 1.7 tokens a second. A request for 120 tokens failed at exactly one minute, and neither the setting's name nor the error said why. Two transports now: the short header timeout for streaming, where it still catches a sick endpoint, and a generation budget for non-streaming, defaulting to ten minutes. One number could not mean both things. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
402 lines
15 KiB
Go
402 lines
15 KiB
Go
// Package config loads llmgw's YAML configuration.
|
|
//
|
|
// Phase 1 only needs the router's half of it: the endpoint list is static here
|
|
// and will come from the control plane in phase 3 (plan §6.4). The shape is
|
|
// already the one the plan describes, so that later change stays additive.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/knadh/koanf/parsers/yaml"
|
|
env "github.com/knadh/koanf/providers/env/v2"
|
|
"github.com/knadh/koanf/providers/file"
|
|
"github.com/knadh/koanf/v2"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
)
|
|
|
|
// Server is the whole server configuration.
|
|
type Server struct {
|
|
// Listen is the client-facing OpenAI API address.
|
|
Listen string `koanf:"listen"`
|
|
// AdminListen serves metrics, health and pprof. Kept separate so the
|
|
// client-facing port can be exposed without exposing diagnostics.
|
|
AdminListen string `koanf:"admin_listen"`
|
|
|
|
// AgentListen is where node agents connect. Separate from the client port
|
|
// so the fleet-facing surface can be firewalled independently.
|
|
AgentListen string `koanf:"agent_listen"`
|
|
|
|
// Auth turns on API keys on the client-facing port.
|
|
Auth Auth `koanf:"auth"`
|
|
|
|
// Landing configures the page served at "/".
|
|
Landing Landing `koanf:"landing"`
|
|
|
|
// AgentTLS authenticates the fleet-facing port. It deliberately does not
|
|
// cover the client-facing one: that surface is fronted by a publicly
|
|
// trusted certificate, while this one is mutual and private.
|
|
AgentTLS tlsconf.Config `koanf:"agent_tls"`
|
|
|
|
Router Router `koanf:"router"`
|
|
Control Control `koanf:"control"`
|
|
Models []catalog.Model `koanf:"models"`
|
|
|
|
// Endpoints is the phase-1 static replica list. It is only consulted when
|
|
// no model declares a profile, which is what makes a router-only smoke test
|
|
// against sim-engines possible without a control plane.
|
|
Endpoints []Endpoint `koanf:"endpoints"`
|
|
}
|
|
|
|
// Landing configures the public page at "/".
|
|
//
|
|
// It exists for one thing the gateway cannot know: who to ask for a key. A page
|
|
// that explains the API and then leaves the reader with no route to using it is
|
|
// where onboarding stops.
|
|
type Landing struct {
|
|
// Contact is where someone without a key should go — an address, a URL, a
|
|
// channel name. Rendered as a link when it is one, as text otherwise.
|
|
Contact string `koanf:"contact"`
|
|
// Notice is an optional line of your own, for whatever the reader needs to
|
|
// know that this page cannot guess.
|
|
Notice string `koanf:"notice"`
|
|
}
|
|
|
|
// Auth configures client authentication (plan §7.9).
|
|
//
|
|
// Off by default, because a gateway on a trusted network with no keys issued
|
|
// still has to serve. Turning it on is what makes the client-facing port safe
|
|
// to expose.
|
|
type Auth struct {
|
|
Enabled bool `koanf:"enabled"`
|
|
// Store is the SQLite file holding keys, tenancy and usage.
|
|
Store string `koanf:"store"`
|
|
// CacheTTL bounds how long a revoked key keeps working, traded against a
|
|
// database read per key per TTL.
|
|
CacheTTL time.Duration `koanf:"cache_ttl"`
|
|
// NegativeTTL is how long a rejected key is remembered as rejected, so a
|
|
// client looping with a bad key cannot become a database read per request.
|
|
NegativeTTL time.Duration `koanf:"negative_ttl"`
|
|
|
|
// SignupOpen serves the signup form. Accounts it creates are pending until
|
|
// an operator approves them, so this is not a way in — but turning it off
|
|
// means accounts are made with `llmgw users create` instead.
|
|
SignupOpen bool `koanf:"signup_open"`
|
|
|
|
// Pricing is what each model costs, for budgets. A model absent from the
|
|
// table costs whatever DefaultPrice says, which is nothing unless set.
|
|
Pricing map[string]Price `koanf:"pricing"`
|
|
DefaultPrice Price `koanf:"default_price"`
|
|
|
|
// FlushInterval and BatchSize decide when usage records are written.
|
|
FlushInterval time.Duration `koanf:"flush_interval"`
|
|
BatchSize int `koanf:"batch_size"`
|
|
// BufferSize bounds the queue. Beyond it records are dropped rather than
|
|
// blocking a request: losing accounting is bad, turning a stalled database
|
|
// into an outage is worse.
|
|
BufferSize int `koanf:"buffer_size"`
|
|
}
|
|
|
|
// Price is a model's cost in micro-units per thousand tokens.
|
|
//
|
|
// What a unit means is the operator's choice — self-hosted models have no
|
|
// per-token price, so it may be GPU-seconds, a notional rate, or 1000
|
|
// everywhere, which turns a budget into a token quota (plan §7.9.4).
|
|
type Price struct {
|
|
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
|
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
|
}
|
|
|
|
// Control configures the control plane.
|
|
type Control struct {
|
|
// Interval is how often desired and actual state are compared.
|
|
Interval time.Duration `koanf:"interval"`
|
|
// BaseBackoff and MaxBackoff bound how fast a failing model is retried.
|
|
BaseBackoff time.Duration `koanf:"base_backoff"`
|
|
MaxBackoff time.Duration `koanf:"max_backoff"`
|
|
// UnreachableAfter and LostAfter stage node expiry. They are deliberately
|
|
// far apart: a network blip must not tear down replicas that still serve,
|
|
// and rebuilding a multi-node replica costs tens of minutes of loading.
|
|
UnreachableAfter time.Duration `koanf:"unreachable_after"`
|
|
LostAfter time.Duration `koanf:"lost_after"`
|
|
// PortRange is the per-node range for engine HTTP and dist-init ports.
|
|
PortRange []int `koanf:"port_range"`
|
|
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
|
MgmtInterface string `koanf:"mgmt_interface"`
|
|
}
|
|
|
|
// Router holds routing behaviour.
|
|
type Router struct {
|
|
ScrapeInterval time.Duration `koanf:"scrape_interval"`
|
|
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
|
|
// StaleAfter is how long an endpoint may go unscraped before being scored
|
|
// down (plan §7.6).
|
|
StaleAfter time.Duration `koanf:"stale_after"`
|
|
|
|
// Retries is how many *additional* endpoints may be tried after a failure,
|
|
// and only while nothing has been written to the client (plan §7.7).
|
|
Retries int `koanf:"retries"`
|
|
// Cooldown is how long a failing endpoint leaves the candidate pool.
|
|
Cooldown time.Duration `koanf:"cooldown"`
|
|
// FailureThreshold is the consecutive failure count that opens a breaker.
|
|
// 1 turns the breaker into a pure cooldown.
|
|
FailureThreshold int `koanf:"failure_threshold"`
|
|
|
|
// DefaultProfile names the routing profile used when a model picks none.
|
|
DefaultProfile string `koanf:"default_profile"`
|
|
// Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the
|
|
// router uses session affinity alone, which is tier 0 and already most of
|
|
// the win for multi-turn traffic.
|
|
Prefix Prefix `koanf:"prefix"`
|
|
|
|
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
|
|
SessionHeader string `koanf:"session_header"`
|
|
// InjectUsage asks engines for usage even when the client did not, so token
|
|
// accounting works regardless of client behaviour.
|
|
InjectUsage bool `koanf:"inject_usage"`
|
|
|
|
// MaxBodyBytes bounds the request body the router will read.
|
|
MaxBodyBytes int64 `koanf:"max_body_bytes"`
|
|
|
|
Timeouts Timeouts `koanf:"timeouts"`
|
|
Scoring Scoring `koanf:"scoring"`
|
|
Admission Admission `koanf:"admission"`
|
|
}
|
|
|
|
// Admission configures the flow control layer (plan §7.2).
|
|
type Admission struct {
|
|
// Enabled turns the layer on. Off, every request goes straight through,
|
|
// which is the v0.2.0 behaviour.
|
|
Enabled bool `koanf:"enabled"`
|
|
|
|
Bands []Band `koanf:"bands"`
|
|
DefaultBand string `koanf:"default_band"`
|
|
// BandHeader lets a client declare its band. Until authentication lands in
|
|
// v1.0.0 this is trusted, so the port must not be exposed to callers who
|
|
// should not be able to promote themselves.
|
|
BandHeader string `koanf:"band_header"`
|
|
|
|
QueueTTL time.Duration `koanf:"queue_ttl"`
|
|
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
|
Interval time.Duration `koanf:"interval"`
|
|
}
|
|
|
|
// Prefix configures approximate prefix routing.
|
|
type Prefix struct {
|
|
Enabled bool `koanf:"enabled"`
|
|
// MaxNodes bounds the tree, MinMatch is the shortest match worth acting on
|
|
// in bytes, and TTL is how long an entry is believed — an engine evicts on
|
|
// its own schedule, so an old record may describe a cache that is gone.
|
|
MaxNodes int `koanf:"max_nodes"`
|
|
MinMatch int `koanf:"min_match"`
|
|
TTL time.Duration `koanf:"ttl"`
|
|
|
|
// MatchRatio is the share of the prompt that must match to be worth acting
|
|
// on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip
|
|
// before affinity is abandoned.
|
|
MatchRatio float64 `koanf:"match_ratio"`
|
|
BalanceAbs float64 `koanf:"balance_abs"`
|
|
BalanceRel float64 `koanf:"balance_rel"`
|
|
}
|
|
|
|
// Band is one priority class.
|
|
type Band struct {
|
|
Name string `koanf:"name"`
|
|
Priority int `koanf:"priority"`
|
|
SaturationCeiling float64 `koanf:"saturation_ceiling"`
|
|
}
|
|
|
|
// Timeouts bounds each phase of an upstream request.
|
|
type Timeouts struct {
|
|
Dial time.Duration `koanf:"dial"`
|
|
// ResponseHeader bounds the wait for headers on a streaming request. It is
|
|
// a liveness check there, because headers arrive before generation starts.
|
|
ResponseHeader time.Duration `koanf:"response_header"`
|
|
// Generation bounds a non-streaming request, where the engine sends no
|
|
// headers until the completion is done. Setting response_header alone used
|
|
// to cap both, so 60s read as a liveness guard and behaved as a limit of
|
|
// about a hundred tokens on a slow model.
|
|
Generation time.Duration `koanf:"generation"`
|
|
FirstToken time.Duration `koanf:"first_token"`
|
|
Idle time.Duration `koanf:"idle"`
|
|
Total time.Duration `koanf:"total"`
|
|
}
|
|
|
|
// Scoring holds the thresholds the scorers normalize against. These want to be
|
|
// measured with ctl bench (plan §10), not guessed.
|
|
type Scoring struct {
|
|
TargetWaiting float64 `koanf:"target_waiting"`
|
|
TargetInflight float64 `koanf:"target_inflight"`
|
|
KVCeiling float64 `koanf:"kv_ceiling"`
|
|
SessionLoadFactor float64 `koanf:"session_load_factor"`
|
|
}
|
|
|
|
// Endpoint is one engine replica. Static in phase 1.
|
|
type Endpoint struct {
|
|
ID string `koanf:"id"`
|
|
URL string `koanf:"url"`
|
|
Models []string `koanf:"models"`
|
|
// Engine names the inference engine, which decides what its metrics are
|
|
// called. Empty means vLLM's names.
|
|
Engine string `koanf:"engine"`
|
|
Pool string `koanf:"pool"`
|
|
Weight float64 `koanf:"weight"`
|
|
Priority int `koanf:"priority"`
|
|
}
|
|
|
|
// Default returns the configuration used when nothing is specified.
|
|
func Default() Server {
|
|
return Server{
|
|
Listen: ":8080",
|
|
AdminListen: ":9090",
|
|
AgentListen: ":7443",
|
|
Control: Control{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
PortRange: []int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
},
|
|
Router: Router{
|
|
ScrapeInterval: 2 * time.Second,
|
|
ScrapeTimeout: time.Second,
|
|
StaleAfter: 10 * time.Second,
|
|
Retries: 2,
|
|
Cooldown: 15 * time.Second,
|
|
FailureThreshold: 3,
|
|
DefaultProfile: "chat",
|
|
SessionHeader: "x-session-id",
|
|
InjectUsage: true,
|
|
MaxBodyBytes: 32 << 20,
|
|
Timeouts: Timeouts{
|
|
Dial: 5 * time.Second,
|
|
ResponseHeader: 60 * time.Second,
|
|
FirstToken: 120 * time.Second,
|
|
Idle: 60 * time.Second,
|
|
Total: 30 * time.Minute,
|
|
},
|
|
Scoring: Scoring{
|
|
TargetWaiting: 4,
|
|
TargetInflight: 8,
|
|
KVCeiling: 0.9,
|
|
SessionLoadFactor: 1.25,
|
|
},
|
|
Admission: Admission{
|
|
Enabled: true,
|
|
BandHeader: "x-llmgw-band",
|
|
DefaultBand: "gold",
|
|
Bands: []Band{
|
|
{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
|
|
{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
|
|
{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
|
|
{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
|
|
},
|
|
QueueTTL: 30 * time.Second,
|
|
QueueCapacityPerBand: 2000,
|
|
Interval: 10 * time.Millisecond,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load reads configuration from path, overlaying it on the defaults, then
|
|
// applies LLMGW_-prefixed environment overrides.
|
|
//
|
|
// An empty path loads the defaults, which is what makes `llmgw server` with a
|
|
// couple of --endpoint flags usable for a smoke test.
|
|
func Load(path string) (Server, error) {
|
|
cfg := Default()
|
|
k := koanf.New(".")
|
|
|
|
if path != "" {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return cfg, fmt.Errorf("config file: %w", err)
|
|
}
|
|
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
|
return cfg, fmt.Errorf("parsing %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
|
|
err := k.Load(env.Provider(".", env.Opt{
|
|
Prefix: "LLMGW_",
|
|
TransformFunc: func(key, value string) (string, any) {
|
|
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
|
|
return strings.ReplaceAll(key, "__", "."), value
|
|
},
|
|
}), nil)
|
|
if err != nil {
|
|
return cfg, fmt.Errorf("environment overrides: %w", err)
|
|
}
|
|
|
|
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
|
return cfg, fmt.Errorf("decoding configuration: %w", err)
|
|
}
|
|
return cfg, cfg.Validate()
|
|
}
|
|
|
|
// Validate rejects configurations that would otherwise fail confusingly at
|
|
// request time.
|
|
func (s Server) Validate() error {
|
|
if s.Listen == "" {
|
|
return fmt.Errorf("listen is empty")
|
|
}
|
|
if err := s.AgentTLS.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if s.Auth.Enabled && s.Auth.Store == "" {
|
|
return fmt.Errorf("auth.store is required when auth is enabled")
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(s.Endpoints))
|
|
for i, e := range s.Endpoints {
|
|
switch {
|
|
case e.ID == "":
|
|
return fmt.Errorf("endpoints[%d]: id is required", i)
|
|
case e.URL == "":
|
|
return fmt.Errorf("endpoint %q: url is required", e.ID)
|
|
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
|
|
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
|
|
case len(e.Models) == 0:
|
|
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
|
|
}
|
|
if _, dup := seen[e.ID]; dup {
|
|
return fmt.Errorf("endpoint id %q is used twice", e.ID)
|
|
}
|
|
seen[e.ID] = struct{}{}
|
|
}
|
|
|
|
// The catalog enforces the rest — aliases, fallbacks, parallelism
|
|
// arithmetic — so there is one place those rules live.
|
|
if _, err := s.Catalog(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Catalog builds the model catalog from the configuration.
|
|
func (s Server) Catalog() (*catalog.Catalog, error) {
|
|
return catalog.New(s.Models)
|
|
}
|
|
|
|
// ControlPlaneMode reports whether the server should run a control plane.
|
|
//
|
|
// A model with no profiles cannot be placed, so a configuration of bare names
|
|
// plus a static endpoint list is the router-only mode used for testing against
|
|
// sim-engines.
|
|
func (s Server) ControlPlaneMode() bool {
|
|
for _, m := range s.Models {
|
|
if len(m.Profiles) > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|