CI / test (push) Successful in 2m34s
Until now anything that could reach the client port could use every model in the catalog. A key now resolves to an identity, and that identity decides what it may do. Authentication wraps the whole mux rather than checking inside each handler, so adding a route cannot add an unauthenticated one by omission. Health stays open, because a load balancer has no key and a health check that needs one reports the gateway down. Nothing on the request path reads the database. A key is resolved once and answered from memory until the entry expires; rejections are cached too, so a client looping with a bad key cannot turn into a database read per request. The price is that a revocation takes up to cache_ttl to take effect, which is the trade stated in the configuration next to the knob. A store failure is not an authentication answer: it is never cached as one and returns 503, not 401 — sending someone hunting for a key problem that does not exist wastes the one thing an incident is short of. A revoked key is refused with the same message as an unknown one, because saying otherwise confirms the key was real. With auth on the band comes from the key and the band header is ignored. The header was trusted only while there was nothing to consult, and it let any client promote itself into the top priority band. Tenant fairness within a band is now per key rather than per session, so one caller cannot take a band by opening many sessions. Allowlists are decided per model, not per string: a model's canonical name and every alias name the same thing, so either form works in an allowlist and no alias can reach a model that was not allowed. The first version of that test passed without the resolution being there at all, which is why it now checks all four combinations. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
327 lines
11 KiB
Go
327 lines
11 KiB
Go
// Package config loads llmgw's YAML configuration.
|
|
//
|
|
// Phase 1 only needs the router's half of it: the endpoint list is static here
|
|
// and will come from the control plane in phase 3 (plan §6.4). The shape is
|
|
// already the one the plan describes, so that later change stays additive.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/knadh/koanf/parsers/yaml"
|
|
env "github.com/knadh/koanf/providers/env/v2"
|
|
"github.com/knadh/koanf/providers/file"
|
|
"github.com/knadh/koanf/v2"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
)
|
|
|
|
// Server is the whole server configuration.
|
|
type Server struct {
|
|
// Listen is the client-facing OpenAI API address.
|
|
Listen string `koanf:"listen"`
|
|
// AdminListen serves metrics, health and pprof. Kept separate so the
|
|
// client-facing port can be exposed without exposing diagnostics.
|
|
AdminListen string `koanf:"admin_listen"`
|
|
|
|
// AgentListen is where node agents connect. Separate from the client port
|
|
// so the fleet-facing surface can be firewalled independently.
|
|
AgentListen string `koanf:"agent_listen"`
|
|
|
|
// Auth turns on API keys on the client-facing port.
|
|
Auth Auth `koanf:"auth"`
|
|
|
|
// AgentTLS authenticates the fleet-facing port. It deliberately does not
|
|
// cover the client-facing one: that surface is fronted by a publicly
|
|
// trusted certificate, while this one is mutual and private.
|
|
AgentTLS tlsconf.Config `koanf:"agent_tls"`
|
|
|
|
Router Router `koanf:"router"`
|
|
Control Control `koanf:"control"`
|
|
Models []catalog.Model `koanf:"models"`
|
|
|
|
// Endpoints is the phase-1 static replica list. It is only consulted when
|
|
// no model declares a profile, which is what makes a router-only smoke test
|
|
// against sim-engines possible without a control plane.
|
|
Endpoints []Endpoint `koanf:"endpoints"`
|
|
}
|
|
|
|
// Auth configures client authentication (plan §7.9).
|
|
//
|
|
// Off by default, because a gateway on a trusted network with no keys issued
|
|
// still has to serve. Turning it on is what makes the client-facing port safe
|
|
// to expose.
|
|
type Auth struct {
|
|
Enabled bool `koanf:"enabled"`
|
|
// Store is the SQLite file holding keys, tenancy and usage.
|
|
Store string `koanf:"store"`
|
|
// CacheTTL bounds how long a revoked key keeps working, traded against a
|
|
// database read per key per TTL.
|
|
CacheTTL time.Duration `koanf:"cache_ttl"`
|
|
// NegativeTTL is how long a rejected key is remembered as rejected, so a
|
|
// client looping with a bad key cannot become a database read per request.
|
|
NegativeTTL time.Duration `koanf:"negative_ttl"`
|
|
}
|
|
|
|
// Control configures the control plane.
|
|
type Control struct {
|
|
// Interval is how often desired and actual state are compared.
|
|
Interval time.Duration `koanf:"interval"`
|
|
// BaseBackoff and MaxBackoff bound how fast a failing model is retried.
|
|
BaseBackoff time.Duration `koanf:"base_backoff"`
|
|
MaxBackoff time.Duration `koanf:"max_backoff"`
|
|
// UnreachableAfter and LostAfter stage node expiry. They are deliberately
|
|
// far apart: a network blip must not tear down replicas that still serve,
|
|
// and rebuilding a multi-node replica costs tens of minutes of loading.
|
|
UnreachableAfter time.Duration `koanf:"unreachable_after"`
|
|
LostAfter time.Duration `koanf:"lost_after"`
|
|
// PortRange is the per-node range for engine HTTP and dist-init ports.
|
|
PortRange []int `koanf:"port_range"`
|
|
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
|
MgmtInterface string `koanf:"mgmt_interface"`
|
|
}
|
|
|
|
// Router holds routing behaviour.
|
|
type Router struct {
|
|
ScrapeInterval time.Duration `koanf:"scrape_interval"`
|
|
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
|
|
// StaleAfter is how long an endpoint may go unscraped before being scored
|
|
// down (plan §7.6).
|
|
StaleAfter time.Duration `koanf:"stale_after"`
|
|
|
|
// Retries is how many *additional* endpoints may be tried after a failure,
|
|
// and only while nothing has been written to the client (plan §7.7).
|
|
Retries int `koanf:"retries"`
|
|
// Cooldown is how long a failing endpoint leaves the candidate pool.
|
|
Cooldown time.Duration `koanf:"cooldown"`
|
|
// FailureThreshold is the consecutive failure count that opens a breaker.
|
|
// 1 turns the breaker into a pure cooldown.
|
|
FailureThreshold int `koanf:"failure_threshold"`
|
|
|
|
// DefaultProfile names the routing profile used when a model picks none.
|
|
DefaultProfile string `koanf:"default_profile"`
|
|
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
|
|
SessionHeader string `koanf:"session_header"`
|
|
// InjectUsage asks engines for usage even when the client did not, so token
|
|
// accounting works regardless of client behaviour.
|
|
InjectUsage bool `koanf:"inject_usage"`
|
|
|
|
// MaxBodyBytes bounds the request body the router will read.
|
|
MaxBodyBytes int64 `koanf:"max_body_bytes"`
|
|
|
|
Timeouts Timeouts `koanf:"timeouts"`
|
|
Scoring Scoring `koanf:"scoring"`
|
|
Admission Admission `koanf:"admission"`
|
|
}
|
|
|
|
// Admission configures the flow control layer (plan §7.2).
|
|
type Admission struct {
|
|
// Enabled turns the layer on. Off, every request goes straight through,
|
|
// which is the v0.2.0 behaviour.
|
|
Enabled bool `koanf:"enabled"`
|
|
|
|
Bands []Band `koanf:"bands"`
|
|
DefaultBand string `koanf:"default_band"`
|
|
// BandHeader lets a client declare its band. Until authentication lands in
|
|
// v1.0.0 this is trusted, so the port must not be exposed to callers who
|
|
// should not be able to promote themselves.
|
|
BandHeader string `koanf:"band_header"`
|
|
|
|
QueueTTL time.Duration `koanf:"queue_ttl"`
|
|
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
|
Interval time.Duration `koanf:"interval"`
|
|
}
|
|
|
|
// Band is one priority class.
|
|
type Band struct {
|
|
Name string `koanf:"name"`
|
|
Priority int `koanf:"priority"`
|
|
SaturationCeiling float64 `koanf:"saturation_ceiling"`
|
|
}
|
|
|
|
// Timeouts bounds each phase of an upstream request.
|
|
type Timeouts struct {
|
|
Dial time.Duration `koanf:"dial"`
|
|
ResponseHeader time.Duration `koanf:"response_header"`
|
|
FirstToken time.Duration `koanf:"first_token"`
|
|
Idle time.Duration `koanf:"idle"`
|
|
Total time.Duration `koanf:"total"`
|
|
}
|
|
|
|
// Scoring holds the thresholds the scorers normalize against. These want to be
|
|
// measured with ctl bench (plan §10), not guessed.
|
|
type Scoring struct {
|
|
TargetWaiting float64 `koanf:"target_waiting"`
|
|
TargetInflight float64 `koanf:"target_inflight"`
|
|
KVCeiling float64 `koanf:"kv_ceiling"`
|
|
SessionLoadFactor float64 `koanf:"session_load_factor"`
|
|
}
|
|
|
|
// Endpoint is one engine replica. Static in phase 1.
|
|
type Endpoint struct {
|
|
ID string `koanf:"id"`
|
|
URL string `koanf:"url"`
|
|
Models []string `koanf:"models"`
|
|
// Engine names the inference engine, which decides what its metrics are
|
|
// called. Empty means vLLM's names.
|
|
Engine string `koanf:"engine"`
|
|
Pool string `koanf:"pool"`
|
|
Weight float64 `koanf:"weight"`
|
|
Priority int `koanf:"priority"`
|
|
}
|
|
|
|
// Default returns the configuration used when nothing is specified.
|
|
func Default() Server {
|
|
return Server{
|
|
Listen: ":8080",
|
|
AdminListen: ":9090",
|
|
AgentListen: ":7443",
|
|
Control: Control{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
PortRange: []int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
},
|
|
Router: Router{
|
|
ScrapeInterval: 2 * time.Second,
|
|
ScrapeTimeout: time.Second,
|
|
StaleAfter: 10 * time.Second,
|
|
Retries: 2,
|
|
Cooldown: 15 * time.Second,
|
|
FailureThreshold: 3,
|
|
DefaultProfile: "chat",
|
|
SessionHeader: "x-session-id",
|
|
InjectUsage: true,
|
|
MaxBodyBytes: 32 << 20,
|
|
Timeouts: Timeouts{
|
|
Dial: 5 * time.Second,
|
|
ResponseHeader: 60 * time.Second,
|
|
FirstToken: 120 * time.Second,
|
|
Idle: 60 * time.Second,
|
|
Total: 30 * time.Minute,
|
|
},
|
|
Scoring: Scoring{
|
|
TargetWaiting: 4,
|
|
TargetInflight: 8,
|
|
KVCeiling: 0.9,
|
|
SessionLoadFactor: 1.25,
|
|
},
|
|
Admission: Admission{
|
|
Enabled: true,
|
|
BandHeader: "x-llmgw-band",
|
|
DefaultBand: "gold",
|
|
Bands: []Band{
|
|
{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
|
|
{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
|
|
{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
|
|
{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
|
|
},
|
|
QueueTTL: 30 * time.Second,
|
|
QueueCapacityPerBand: 2000,
|
|
Interval: 10 * time.Millisecond,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load reads configuration from path, overlaying it on the defaults, then
|
|
// applies LLMGW_-prefixed environment overrides.
|
|
//
|
|
// An empty path loads the defaults, which is what makes `llmgw server` with a
|
|
// couple of --endpoint flags usable for a smoke test.
|
|
func Load(path string) (Server, error) {
|
|
cfg := Default()
|
|
k := koanf.New(".")
|
|
|
|
if path != "" {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return cfg, fmt.Errorf("config file: %w", err)
|
|
}
|
|
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
|
return cfg, fmt.Errorf("parsing %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
|
|
err := k.Load(env.Provider(".", env.Opt{
|
|
Prefix: "LLMGW_",
|
|
TransformFunc: func(key, value string) (string, any) {
|
|
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
|
|
return strings.ReplaceAll(key, "__", "."), value
|
|
},
|
|
}), nil)
|
|
if err != nil {
|
|
return cfg, fmt.Errorf("environment overrides: %w", err)
|
|
}
|
|
|
|
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
|
return cfg, fmt.Errorf("decoding configuration: %w", err)
|
|
}
|
|
return cfg, cfg.Validate()
|
|
}
|
|
|
|
// Validate rejects configurations that would otherwise fail confusingly at
|
|
// request time.
|
|
func (s Server) Validate() error {
|
|
if s.Listen == "" {
|
|
return fmt.Errorf("listen is empty")
|
|
}
|
|
if err := s.AgentTLS.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if s.Auth.Enabled && s.Auth.Store == "" {
|
|
return fmt.Errorf("auth.store is required when auth is enabled")
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(s.Endpoints))
|
|
for i, e := range s.Endpoints {
|
|
switch {
|
|
case e.ID == "":
|
|
return fmt.Errorf("endpoints[%d]: id is required", i)
|
|
case e.URL == "":
|
|
return fmt.Errorf("endpoint %q: url is required", e.ID)
|
|
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
|
|
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
|
|
case len(e.Models) == 0:
|
|
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
|
|
}
|
|
if _, dup := seen[e.ID]; dup {
|
|
return fmt.Errorf("endpoint id %q is used twice", e.ID)
|
|
}
|
|
seen[e.ID] = struct{}{}
|
|
}
|
|
|
|
// The catalog enforces the rest — aliases, fallbacks, parallelism
|
|
// arithmetic — so there is one place those rules live.
|
|
if _, err := s.Catalog(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Catalog builds the model catalog from the configuration.
|
|
func (s Server) Catalog() (*catalog.Catalog, error) {
|
|
return catalog.New(s.Models)
|
|
}
|
|
|
|
// ControlPlaneMode reports whether the server should run a control plane.
|
|
//
|
|
// A model with no profiles cannot be placed, so a configuration of bare names
|
|
// plus a static endpoint list is the router-only mode used for testing against
|
|
// sim-engines.
|
|
func (s Server) ControlPlaneMode() bool {
|
|
for _, m := range s.Models {
|
|
if len(m.Profiles) > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|