Files
llmgw/internal/config/config.go
T
Ilker ManapandClaude Opus 5 ec1b6b04df
CI / test (push) Successful in 54s
Fix an agent that could neither explain itself nor stop
Two bugs found while verifying mTLS against real machines.

session cancelled its own derived context and then asked that context
whether the agent was shutting down. The answer was always yes, so every
stream error read as a clean close: the log said err=<nil>, the backoff
reset, and an agent whose certificate the control plane refused hammered it
forever at the base interval with nothing to diagnose. It now keeps the
parent context to answer that question, and a refusal reaches the operator
naming both the certificate and the claim.

Cancelling the context also did not stop the agent. HTTP/2 watches the
context while the request body is being written, and a bidirectional stream
spends its life blocked on the read side instead, so the read never returned.
Measured on a deployed node: `systemctl stop llmgw-agent` took 90 seconds and
ended in SIGKILL, on every restart. Closing the response when the context
ends is what actually unblocks it.

TestAgentStopsWhenContextEnds hangs reliably without the second fix, which is
the only reason to trust that it tests it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 08:03:21 +02:00

301 lines
10 KiB
Go

// Package config loads llmgw's YAML configuration.
//
// Phase 1 only needs the router's half of it: the endpoint list is static here
// and will come from the control plane in phase 3 (plan §6.4). The shape is
// already the one the plan describes, so that later change stays additive.
package config
import (
"fmt"
"os"
"strings"
"time"
"github.com/knadh/koanf/parsers/yaml"
env "github.com/knadh/koanf/providers/env/v2"
"github.com/knadh/koanf/providers/file"
"github.com/knadh/koanf/v2"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
// Server is the whole server configuration.
type Server struct {
// Listen is the client-facing OpenAI API address.
Listen string `koanf:"listen"`
// AdminListen serves metrics, health and pprof. Kept separate so the
// client-facing port can be exposed without exposing diagnostics.
AdminListen string `koanf:"admin_listen"`
// AgentListen is where node agents connect. Separate from the client port
// so the fleet-facing surface can be firewalled independently.
AgentListen string `koanf:"agent_listen"`
// AgentTLS authenticates the fleet-facing port. It deliberately does not
// cover the client-facing one: that surface is fronted by a publicly
// trusted certificate, while this one is mutual and private.
AgentTLS tlsconf.Config `koanf:"agent_tls"`
Router Router `koanf:"router"`
Control Control `koanf:"control"`
Models []catalog.Model `koanf:"models"`
// Endpoints is the phase-1 static replica list. It is only consulted when
// no model declares a profile, which is what makes a router-only smoke test
// against sim-engines possible without a control plane.
Endpoints []Endpoint `koanf:"endpoints"`
}
// Control configures the control plane.
type Control struct {
// Interval is how often desired and actual state are compared.
Interval time.Duration `koanf:"interval"`
// BaseBackoff and MaxBackoff bound how fast a failing model is retried.
BaseBackoff time.Duration `koanf:"base_backoff"`
MaxBackoff time.Duration `koanf:"max_backoff"`
// UnreachableAfter and LostAfter stage node expiry. They are deliberately
// far apart: a network blip must not tear down replicas that still serve,
// and rebuilding a multi-node replica costs tens of minutes of loading.
UnreachableAfter time.Duration `koanf:"unreachable_after"`
LostAfter time.Duration `koanf:"lost_after"`
// PortRange is the per-node range for engine HTTP and dist-init ports.
PortRange []int `koanf:"port_range"`
// MgmtInterface is the ethernet interface NCCL bootstraps over.
MgmtInterface string `koanf:"mgmt_interface"`
}
// Router holds routing behaviour.
type Router struct {
ScrapeInterval time.Duration `koanf:"scrape_interval"`
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
// StaleAfter is how long an endpoint may go unscraped before being scored
// down (plan §7.6).
StaleAfter time.Duration `koanf:"stale_after"`
// Retries is how many *additional* endpoints may be tried after a failure,
// and only while nothing has been written to the client (plan §7.7).
Retries int `koanf:"retries"`
// Cooldown is how long a failing endpoint leaves the candidate pool.
Cooldown time.Duration `koanf:"cooldown"`
// FailureThreshold is the consecutive failure count that opens a breaker.
// 1 turns the breaker into a pure cooldown.
FailureThreshold int `koanf:"failure_threshold"`
// DefaultProfile names the routing profile used when a model picks none.
DefaultProfile string `koanf:"default_profile"`
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
SessionHeader string `koanf:"session_header"`
// InjectUsage asks engines for usage even when the client did not, so token
// accounting works regardless of client behaviour.
InjectUsage bool `koanf:"inject_usage"`
// MaxBodyBytes bounds the request body the router will read.
MaxBodyBytes int64 `koanf:"max_body_bytes"`
Timeouts Timeouts `koanf:"timeouts"`
Scoring Scoring `koanf:"scoring"`
Admission Admission `koanf:"admission"`
}
// Admission configures the flow control layer (plan §7.2).
type Admission struct {
// Enabled turns the layer on. Off, every request goes straight through,
// which is the v0.2.0 behaviour.
Enabled bool `koanf:"enabled"`
Bands []Band `koanf:"bands"`
DefaultBand string `koanf:"default_band"`
// BandHeader lets a client declare its band. Until authentication lands in
// v1.0.0 this is trusted, so the port must not be exposed to callers who
// should not be able to promote themselves.
BandHeader string `koanf:"band_header"`
QueueTTL time.Duration `koanf:"queue_ttl"`
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
Interval time.Duration `koanf:"interval"`
}
// Band is one priority class.
type Band struct {
Name string `koanf:"name"`
Priority int `koanf:"priority"`
SaturationCeiling float64 `koanf:"saturation_ceiling"`
}
// Timeouts bounds each phase of an upstream request.
type Timeouts struct {
Dial time.Duration `koanf:"dial"`
ResponseHeader time.Duration `koanf:"response_header"`
FirstToken time.Duration `koanf:"first_token"`
Idle time.Duration `koanf:"idle"`
Total time.Duration `koanf:"total"`
}
// Scoring holds the thresholds the scorers normalize against. These want to be
// measured with ctl bench (plan §10), not guessed.
type Scoring struct {
TargetWaiting float64 `koanf:"target_waiting"`
TargetInflight float64 `koanf:"target_inflight"`
KVCeiling float64 `koanf:"kv_ceiling"`
SessionLoadFactor float64 `koanf:"session_load_factor"`
}
// Endpoint is one engine replica. Static in phase 1.
type Endpoint struct {
ID string `koanf:"id"`
URL string `koanf:"url"`
Models []string `koanf:"models"`
Pool string `koanf:"pool"`
Weight float64 `koanf:"weight"`
Priority int `koanf:"priority"`
}
// Default returns the configuration used when nothing is specified.
func Default() Server {
return Server{
Listen: ":8080",
AdminListen: ":9090",
AgentListen: ":7443",
Control: Control{
Interval: 5 * time.Second,
BaseBackoff: 10 * time.Second,
MaxBackoff: 10 * time.Minute,
UnreachableAfter: 15 * time.Second,
LostAfter: 60 * time.Second,
PortRange: []int{20000, 20999},
MgmtInterface: "eth0",
},
Router: Router{
ScrapeInterval: 2 * time.Second,
ScrapeTimeout: time.Second,
StaleAfter: 10 * time.Second,
Retries: 2,
Cooldown: 15 * time.Second,
FailureThreshold: 3,
DefaultProfile: "chat",
SessionHeader: "x-session-id",
InjectUsage: true,
MaxBodyBytes: 32 << 20,
Timeouts: Timeouts{
Dial: 5 * time.Second,
ResponseHeader: 60 * time.Second,
FirstToken: 120 * time.Second,
Idle: 60 * time.Second,
Total: 30 * time.Minute,
},
Scoring: Scoring{
TargetWaiting: 4,
TargetInflight: 8,
KVCeiling: 0.9,
SessionLoadFactor: 1.25,
},
Admission: Admission{
Enabled: true,
BandHeader: "x-llmgw-band",
DefaultBand: "gold",
Bands: []Band{
{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
},
QueueTTL: 30 * time.Second,
QueueCapacityPerBand: 2000,
Interval: 10 * time.Millisecond,
},
},
}
}
// Load reads configuration from path, overlaying it on the defaults, then
// applies LLMGW_-prefixed environment overrides.
//
// An empty path loads the defaults, which is what makes `llmgw server` with a
// couple of --endpoint flags usable for a smoke test.
func Load(path string) (Server, error) {
cfg := Default()
k := koanf.New(".")
if path != "" {
if _, err := os.Stat(path); err != nil {
return cfg, fmt.Errorf("config file: %w", err)
}
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
return cfg, fmt.Errorf("parsing %s: %w", path, err)
}
}
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
err := k.Load(env.Provider(".", env.Opt{
Prefix: "LLMGW_",
TransformFunc: func(key, value string) (string, any) {
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
return strings.ReplaceAll(key, "__", "."), value
},
}), nil)
if err != nil {
return cfg, fmt.Errorf("environment overrides: %w", err)
}
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
return cfg, fmt.Errorf("decoding configuration: %w", err)
}
return cfg, cfg.Validate()
}
// Validate rejects configurations that would otherwise fail confusingly at
// request time.
func (s Server) Validate() error {
if s.Listen == "" {
return fmt.Errorf("listen is empty")
}
if err := s.AgentTLS.Validate(); err != nil {
return err
}
seen := make(map[string]struct{}, len(s.Endpoints))
for i, e := range s.Endpoints {
switch {
case e.ID == "":
return fmt.Errorf("endpoints[%d]: id is required", i)
case e.URL == "":
return fmt.Errorf("endpoint %q: url is required", e.ID)
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
case len(e.Models) == 0:
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
}
if _, dup := seen[e.ID]; dup {
return fmt.Errorf("endpoint id %q is used twice", e.ID)
}
seen[e.ID] = struct{}{}
}
// The catalog enforces the rest — aliases, fallbacks, parallelism
// arithmetic — so there is one place those rules live.
if _, err := s.Catalog(); err != nil {
return err
}
return nil
}
// Catalog builds the model catalog from the configuration.
func (s Server) Catalog() (*catalog.Catalog, error) {
return catalog.New(s.Models)
}
// ControlPlaneMode reports whether the server should run a control plane.
//
// A model with no profiles cannot be placed, so a configuration of bare names
// plus a static endpoint list is the router-only mode used for testing against
// sim-engines.
func (s Server) ControlPlaneMode() bool {
for _, m := range s.Models {
if len(m.Profiles) > 0 {
return true
}
}
return false
}