A model served from the registry records its origin: repository, commit and each file's checksum; one served before is worked out from the mirror's record of where it fetched the file. Every mirror.update_check (6h) the control plane asks whether those files changed there. ctl model updates lists it, ctl model update fetches beside the old files, checks, swaps and replaces replicas one at a time, ctl model rollback goes back. The console shows it on Models and offers the same. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
742 lines
30 KiB
Go
742 lines
30 KiB
Go
// Package config loads llmgw's YAML configuration.
|
|
//
|
|
// Phase 1 only needs the router's half of it: the endpoint list is static here
|
|
// and will come from the control plane in phase 3 (plan §6.4). The shape is
|
|
// already the one the plan describes, so that later change stays additive.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/knadh/koanf/parsers/yaml"
|
|
env "github.com/knadh/koanf/providers/env/v2"
|
|
"github.com/knadh/koanf/providers/file"
|
|
"github.com/knadh/koanf/v2"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
|
)
|
|
|
|
// Server is the whole server configuration.
|
|
type Server struct {
|
|
// Listen is the client-facing OpenAI API address.
|
|
Listen string `koanf:"listen"`
|
|
// AdminListen serves the dashboard, the management API, health and pprof.
|
|
//
|
|
// Loopback by default, and it should stay there: pprof can dump the process
|
|
// heap, and API keys are in that heap in plaintext. Every comment beside
|
|
// this setting has always said not to expose it, while the default bound
|
|
// every interface — a default that contradicts its own documentation is a
|
|
// footgun, so the default now matches the advice.
|
|
AdminListen string `koanf:"admin_listen"`
|
|
|
|
// MetricsListen serves Prometheus metrics and nothing else.
|
|
//
|
|
// Separate from AdminListen because the reason that one is loopback is
|
|
// pprof, not metrics — and metrics are the one thing on it that something
|
|
// else genuinely needs to reach. Sharing a listener meant a fleet could
|
|
// either be monitored or keep its heap private, which is not a choice
|
|
// anybody should have to make.
|
|
//
|
|
// Empty means off, and metrics remain available on AdminListen either way,
|
|
// so nothing that scrapes it today has to change.
|
|
MetricsListen string `koanf:"metrics_listen"`
|
|
|
|
// AgentListen is where node agents connect. Separate from the client port
|
|
// so the fleet-facing surface can be firewalled independently.
|
|
AgentListen string `koanf:"agent_listen"`
|
|
|
|
// Auth turns on API keys on the client-facing port.
|
|
Auth Auth `koanf:"auth"`
|
|
|
|
// Threads keeps conversations and agent memory without API keys, for a
|
|
// gateway on one trusted machine. With auth on they live in the auth store,
|
|
// per project, and this block is ignored.
|
|
Threads Threads `koanf:"threads"`
|
|
|
|
// Landing configures the page served at "/".
|
|
Landing Landing `koanf:"landing"`
|
|
|
|
// AgentTLS authenticates the fleet-facing port. It deliberately does not
|
|
// cover the client-facing one: that surface is fronted by a publicly
|
|
// trusted certificate, while this one is mutual and private.
|
|
AgentTLS tlsconf.Config `koanf:"agent_tls"`
|
|
|
|
// Monitoring is where the dashboard reads its history from.
|
|
Monitoring Monitoring `koanf:"monitoring"`
|
|
// Tracing sends a trace of each inference request to an OpenTelemetry
|
|
// collector. Off unless an endpoint is set here or in
|
|
// OTEL_EXPORTER_OTLP_ENDPOINT.
|
|
Tracing Tracing `koanf:"tracing"`
|
|
|
|
// Mirror points the control plane at a model mirror it can administer.
|
|
// Empty means there is none, and /admin/mirror stays unmounted.
|
|
Mirror Mirror `koanf:"mirror"`
|
|
|
|
// Engines are the engines every node installs, and at which version. The
|
|
// agents install them themselves — the right build for their hardware,
|
|
// checked against its publisher's checksum and seen to run — so a node is
|
|
// never trusted to already have what somebody once put there.
|
|
//
|
|
// Unset keeps DefaultEngines. A version replaces the default; an empty
|
|
// one leaves that engine to be installed by hand.
|
|
Engines map[string]string `koanf:"engines"`
|
|
|
|
// EngineOverrides give some nodes different engines from the rest: a
|
|
// vLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at
|
|
// all on nodes meant only for llama.cpp. Each picks nodes and lays its
|
|
// engines over the cluster's, in order, later ones last; an empty version
|
|
// removes that engine from the nodes it picks.
|
|
EngineOverrides []EngineOverride `koanf:"engine_overrides"`
|
|
|
|
// BackupDir is where `llmgw server backup` writes, and where the newest
|
|
// archive's age is read for llmgw_backup_last_success_timestamp_seconds.
|
|
// Empty means /var/backups/llmgw-server.
|
|
BackupDir string `koanf:"backup_dir"`
|
|
|
|
// Releases is where `llmgw ctl upgrade` fetches release binaries from,
|
|
// for this machine and — through their agents — every node. Empty means
|
|
// the project's own releases.
|
|
Releases string `koanf:"releases"`
|
|
|
|
Router Router `koanf:"router"`
|
|
Control Control `koanf:"control"`
|
|
Models []catalog.Model `koanf:"models"`
|
|
|
|
// Endpoints is the phase-1 static replica list. It is only consulted when
|
|
// no model declares a profile, which is what makes a router-only smoke test
|
|
// against sim-engines possible without a control plane.
|
|
Endpoints []Endpoint `koanf:"endpoints"`
|
|
}
|
|
|
|
// Monitoring points the dashboard at a Prometheus holding this cluster's
|
|
// history.
|
|
//
|
|
// Read back rather than kept twice. llmgw publishes these metrics already; what
|
|
// a gauge cannot do is remember, and a ring buffer in this process would be a
|
|
// rival copy with its own retention, lost on every restart and disagreeing with
|
|
// the one an operator already has on screen.
|
|
type Monitoring struct {
|
|
// Prometheus is the server's address, for example
|
|
// http://monitoring.internal:9090. Empty leaves the charts off the
|
|
// dashboard rather than showing an error where one should be.
|
|
Prometheus string `koanf:"prometheus"`
|
|
// Window is how far back the charts look. Zero takes an hour.
|
|
Window time.Duration `koanf:"window"`
|
|
// Timeout bounds one query. It runs while somebody waits for a page, so a
|
|
// monitoring system that is slow today is skipped rather than waited for.
|
|
Timeout time.Duration `koanf:"timeout"`
|
|
}
|
|
|
|
// Tracing is where request traces go, over OTLP/HTTP.
|
|
//
|
|
// One span per inference request, with admission, scheduling and each attempt
|
|
// at an engine beneath it, and the trace carried on to the engine in a
|
|
// traceparent header. The request's X-Request-Id is on the span, so a trace
|
|
// and `llmgw ctl request` name the same request.
|
|
type Tracing struct {
|
|
// Endpoint is the collector, for example http://otel.internal:4318.
|
|
// Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.
|
|
Endpoint string `koanf:"endpoint"`
|
|
// SampleRatio is the share of requests traced, 0 to 1, when the caller
|
|
// did not decide it with a sampled traceparent of its own. Zero takes 1.
|
|
SampleRatio float64 `koanf:"sample_ratio"`
|
|
}
|
|
|
|
// Mirror is a model mirror this control plane administers over its API.
|
|
//
|
|
// The mirror is a separate machine with a large disk, not part of the gateway,
|
|
// so what is configured here is an address and how to authenticate to it —
|
|
// never a directory. A control plane that could write to the mirror's disk
|
|
// directly would be a way to replace a model with another one that every node
|
|
// would then fetch without complaint.
|
|
type Mirror struct {
|
|
// API is the mirror's management address, scheme included, for example
|
|
// https://models.internal:8082. Empty means no mirror is configured.
|
|
//
|
|
// The management port, not the one nodes fetch bytes from. Those are two
|
|
// listeners on the mirror precisely so this one can be locked down without
|
|
// re-certifying every agent's fetcher.
|
|
API string `koanf:"api"`
|
|
// TLS authenticates to the mirror's management API, with the same CA the
|
|
// agents use. Empty leaves the connection in plain HTTP, which is only
|
|
// acceptable on a network where nothing untrusted can reach that port —
|
|
// the mirror says as much in its own log when it starts that way.
|
|
TLS tlsconf.Config `koanf:"tls"`
|
|
// Registry is the model registry the mirror page browses, empty for the
|
|
// public Hugging Face. Set it to an internal mirror of the registry where
|
|
// the control plane cannot reach the internet — or leave the whole mirror
|
|
// block out, which turns the explorer off rather than giving it a search
|
|
// box that can only time out.
|
|
Registry string `koanf:"registry"`
|
|
// Timeout bounds a management call. It does not bound a download: the
|
|
// mirror answers a fetch as soon as it has started one, and the bytes keep
|
|
// arriving long after this deadline has passed.
|
|
Timeout time.Duration `koanf:"timeout"`
|
|
// UpdateCheck is how often the registry is asked whether the files of
|
|
// the models served from it have changed there: a repository can publish
|
|
// new weights under the same name. Only told, never applied — moving a
|
|
// model to them is `llmgw ctl model update`. 0 never asks.
|
|
UpdateCheck time.Duration `koanf:"update_check"`
|
|
}
|
|
|
|
// Landing configures the public page at "/".
|
|
//
|
|
// It exists for one thing the gateway cannot know: who to ask for a key. A page
|
|
// that explains the API and then leaves the reader with no route to using it is
|
|
// where onboarding stops.
|
|
type Landing struct {
|
|
// Contact is where someone without a key should go — an address, a URL, a
|
|
// channel name. Rendered as a link when it is one, as text otherwise.
|
|
Contact string `koanf:"contact"`
|
|
// Notice is an optional line of your own, for whatever the reader needs to
|
|
// know that this page cannot guess.
|
|
Notice string `koanf:"notice"`
|
|
}
|
|
|
|
// Threads configures conversation and memory storage for a gateway without
|
|
// authentication.
|
|
//
|
|
// Everything stored is in one pool with no owner, readable by anyone who can
|
|
// reach the client port and knows an id. That is acceptable on one machine and
|
|
// wrong on a shared network, which is why it takes a setting rather than being
|
|
// the default.
|
|
type Threads struct {
|
|
// Store is the SQLite file. Empty means nothing is kept.
|
|
Store string `koanf:"store"`
|
|
// RetainHours is how long idle conversations and notes are kept. Required
|
|
// with Store: a store with no expiry keeps everything anyone typed, by
|
|
// omission.
|
|
RetainHours int `koanf:"retain_hours"`
|
|
}
|
|
|
|
// Auth configures client authentication (plan §7.9).
|
|
//
|
|
// Off by default, because a gateway on a trusted network with no keys issued
|
|
// still has to serve. Turning it on is what makes the client-facing port safe
|
|
// to expose.
|
|
type Auth struct {
|
|
Enabled bool `koanf:"enabled"`
|
|
// Store is the SQLite file holding keys, tenancy and usage.
|
|
Store string `koanf:"store"`
|
|
// CacheTTL bounds how long a revoked key keeps working, traded against a
|
|
// database read per key per TTL.
|
|
CacheTTL time.Duration `koanf:"cache_ttl"`
|
|
// NegativeTTL is how long a rejected key is remembered as rejected, so a
|
|
// client looping with a bad key cannot become a database read per request.
|
|
NegativeTTL time.Duration `koanf:"negative_ttl"`
|
|
|
|
// DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to.
|
|
//
|
|
// Unlimited used to be the default, and a project created without limits is
|
|
// the easy path — the "default" project on this cluster still is one. The
|
|
// first sign of that is the bill or the outage. Zero keeps the old
|
|
// behaviour for anyone who wants it, but it should be a choice rather than
|
|
// an omission. A fallback, not a ceiling: an explicit limit at any scope
|
|
// wins, above or below.
|
|
DefaultRPM int `koanf:"default_rpm"`
|
|
DefaultTPM int `koanf:"default_tpm"`
|
|
|
|
// SignupOpen serves the signup form. Accounts it creates are pending until
|
|
// an operator approves them, so this is not a way in — but turning it off
|
|
// means accounts are made with `llmgw users create` instead.
|
|
SignupOpen bool `koanf:"signup_open"`
|
|
|
|
// Personal is what a person may do on their own, outside any project: make
|
|
// personal keys and use the chat page personally, within these limits.
|
|
Personal Personal `koanf:"personal"`
|
|
|
|
// Pricing is what each model costs, for budgets. A model absent from the
|
|
// table costs whatever DefaultPrice says, which is nothing unless set.
|
|
Pricing map[string]Price `koanf:"pricing"`
|
|
DefaultPrice Price `koanf:"default_price"`
|
|
|
|
// FlushInterval and BatchSize decide when usage records are written.
|
|
FlushInterval time.Duration `koanf:"flush_interval"`
|
|
BatchSize int `koanf:"batch_size"`
|
|
// BufferSize bounds the queue. Beyond it records are dropped rather than
|
|
// blocking a request: losing accounting is bad, turning a stalled database
|
|
// into an outage is worse.
|
|
BufferSize int `koanf:"buffer_size"`
|
|
|
|
// UsageRetention is how long the record of each request is kept: who sent
|
|
// it, which replica answered, how long it queued — what `ctl request`
|
|
// reads. Zero keeps it for ever. Totals, charts and budgets are read from
|
|
// five-minute sums kept beside the records, which are not pruned, so this
|
|
// bounds the table that grows by a row a request without changing any
|
|
// figure a page shows. At least a day when set.
|
|
UsageRetention time.Duration `koanf:"usage_retention"`
|
|
}
|
|
|
|
// Personal configures use that belongs to a person rather than a project.
|
|
//
|
|
// On by default: a person need not belong to anything to use the gateway. The
|
|
// limits are each person's own pool across all their personal keys and their
|
|
// personal chat, and apply only where nobody wrote a limit for that person.
|
|
type Personal struct {
|
|
// Disabled refuses personal keys and personal chat; people then use the
|
|
// gateway only through projects they are members of.
|
|
Disabled bool `koanf:"disabled"`
|
|
RPM int `koanf:"rpm"`
|
|
TPM int `koanf:"tpm"`
|
|
// Budget is in whole units per Period ("day" or "month").
|
|
Budget float64 `koanf:"budget"`
|
|
Period string `koanf:"period"`
|
|
}
|
|
|
|
// Price is a model's cost in micro-units per thousand tokens.
|
|
//
|
|
// What a unit means is the operator's choice — self-hosted models have no
|
|
// per-token price, so it may be GPU-seconds, a notional rate, or 1000
|
|
// everywhere, which turns a budget into a token quota (plan §7.9.4).
|
|
type Price struct {
|
|
PromptPer1k int64 `koanf:"prompt_per_1k"`
|
|
CompletionPer1k int64 `koanf:"completion_per_1k"`
|
|
}
|
|
|
|
// Control configures the control plane.
|
|
type Control struct {
|
|
// Interval is how often desired and actual state are compared.
|
|
Interval time.Duration `koanf:"interval"`
|
|
// BaseBackoff and MaxBackoff bound how fast a failing model is retried.
|
|
BaseBackoff time.Duration `koanf:"base_backoff"`
|
|
MaxBackoff time.Duration `koanf:"max_backoff"`
|
|
// UnreachableAfter and LostAfter stage node expiry. They are deliberately
|
|
// far apart: a network blip must not tear down replicas that still serve,
|
|
// and rebuilding a multi-node replica costs tens of minutes of loading.
|
|
UnreachableAfter time.Duration `koanf:"unreachable_after"`
|
|
LostAfter time.Duration `koanf:"lost_after"`
|
|
// PortRange is the per-node range for engine HTTP and dist-init ports.
|
|
PortRange []int `koanf:"port_range"`
|
|
// MgmtInterface is the ethernet interface NCCL bootstraps over.
|
|
MgmtInterface string `koanf:"mgmt_interface"`
|
|
|
|
// Enabled runs the control plane even when configuration declares no
|
|
// models.
|
|
//
|
|
// Without it, a deployment whose catalog lives entirely in the database
|
|
// cannot start: control plane mode is inferred from the config file, so an
|
|
// empty models block means "router only", the process refuses to start for
|
|
// want of anything to serve, and the models it would have loaded are never
|
|
// read. That is a cluster that cannot be managed at runtime because it is
|
|
// managed at runtime. AgentListen cannot carry this signal — it has a
|
|
// default, so every deployment would become a control plane.
|
|
Enabled bool `koanf:"enabled"`
|
|
|
|
// LeaseTTL is how long this control plane's claim to be the one placing
|
|
// work stands without being renewed.
|
|
//
|
|
// Only meaningful with a shared store, where a second control plane can
|
|
// see the claim. Long enough to survive a slow query or a brief network
|
|
// stall, short enough that a standby takes over before anyone has finished
|
|
// reading the alert. Zero takes the default.
|
|
LeaseTTL time.Duration `koanf:"lease_ttl"`
|
|
}
|
|
|
|
// Router holds routing behaviour.
|
|
type Router struct {
|
|
// ActivationWait is how long a request waits for an on-demand model to
|
|
// load before the client is told to come back. Zero takes the default.
|
|
//
|
|
// The one real decision in on-demand loading. Refusing outright makes the
|
|
// feature useless, since the first request for every model would fail;
|
|
// waiting without a bound turns a request into a job, and a client that
|
|
// budgeted thirty seconds ends up holding a connection for ten minutes.
|
|
ActivationWait time.Duration `koanf:"activation_wait"`
|
|
|
|
ScrapeInterval time.Duration `koanf:"scrape_interval"`
|
|
ScrapeTimeout time.Duration `koanf:"scrape_timeout"`
|
|
// StaleAfter is how long an endpoint may go unscraped before being scored
|
|
// down (plan §7.6).
|
|
StaleAfter time.Duration `koanf:"stale_after"`
|
|
|
|
// Retries is how many *additional* endpoints may be tried after a failure,
|
|
// and only while nothing has been written to the client (plan §7.7).
|
|
Retries int `koanf:"retries"`
|
|
// Cooldown is how long a failing endpoint leaves the candidate pool.
|
|
Cooldown time.Duration `koanf:"cooldown"`
|
|
// FailureThreshold is the consecutive failure count that opens a breaker.
|
|
// 1 turns the breaker into a pure cooldown.
|
|
FailureThreshold int `koanf:"failure_threshold"`
|
|
|
|
// DefaultProfile names the routing profile used when a model picks none.
|
|
DefaultProfile string `koanf:"default_profile"`
|
|
// Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the
|
|
// router uses session affinity alone, which is tier 0 and already most of
|
|
// the win for multi-turn traffic.
|
|
Prefix Prefix `koanf:"prefix"`
|
|
|
|
// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
|
|
SessionHeader string `koanf:"session_header"`
|
|
// InjectUsage asks engines for usage even when the client did not, so token
|
|
// accounting works regardless of client behaviour.
|
|
InjectUsage bool `koanf:"inject_usage"`
|
|
|
|
// TenantLabel is the level of the tree the per-tenant request series are
|
|
// labelled by — llmgw_tenant_requests_total and its siblings: "org",
|
|
// "team" or "project", or "none" for no such series. A label per project
|
|
// on a cluster of thousands is more series than Prometheus wants, and a
|
|
// team is what an alert is usually about; "team" is the default.
|
|
TenantLabel string `koanf:"tenant_label"`
|
|
|
|
// MaxBodyBytes bounds the request body the router will read.
|
|
MaxBodyBytes int64 `koanf:"max_body_bytes"`
|
|
|
|
Timeouts Timeouts `koanf:"timeouts"`
|
|
Scoring Scoring `koanf:"scoring"`
|
|
Admission Admission `koanf:"admission"`
|
|
}
|
|
|
|
// Admission configures the flow control layer (plan §7.2).
|
|
type Admission struct {
|
|
// Enabled turns the layer on. Off, every request goes straight through,
|
|
// which is the v0.2.0 behaviour.
|
|
Enabled bool `koanf:"enabled"`
|
|
|
|
Bands []Band `koanf:"bands"`
|
|
DefaultBand string `koanf:"default_band"`
|
|
// BandHeader lets a client declare its band. Until authentication lands in
|
|
// v1.0.0 this is trusted, so the port must not be exposed to callers who
|
|
// should not be able to promote themselves.
|
|
BandHeader string `koanf:"band_header"`
|
|
|
|
QueueTTL time.Duration `koanf:"queue_ttl"`
|
|
QueueCapacityPerBand int `koanf:"queue_capacity_per_band"`
|
|
// QueueCapacityTotal bounds everything queued across bands. Reached, a
|
|
// request is not refused while a lower band has any queued: the newest of
|
|
// the lowest band is turned away instead, with a 503 and Retry-After.
|
|
// Zero takes half of all the bands' capacities together.
|
|
QueueCapacityTotal int `koanf:"queue_capacity_total"`
|
|
Interval time.Duration `koanf:"interval"`
|
|
}
|
|
|
|
// Prefix configures approximate prefix routing.
|
|
type Prefix struct {
|
|
Enabled bool `koanf:"enabled"`
|
|
// MaxNodes bounds the tree, MinMatch is the shortest match worth acting on
|
|
// in bytes, and TTL is how long an entry is believed — an engine evicts on
|
|
// its own schedule, so an old record may describe a cache that is gone.
|
|
MaxNodes int `koanf:"max_nodes"`
|
|
MinMatch int `koanf:"min_match"`
|
|
TTL time.Duration `koanf:"ttl"`
|
|
|
|
// MatchRatio is the share of the prompt that must match to be worth acting
|
|
// on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip
|
|
// before affinity is abandoned.
|
|
MatchRatio float64 `koanf:"match_ratio"`
|
|
BalanceAbs float64 `koanf:"balance_abs"`
|
|
BalanceRel float64 `koanf:"balance_rel"`
|
|
}
|
|
|
|
// Band is one priority class.
|
|
type Band struct {
|
|
Name string `koanf:"name"`
|
|
Priority int `koanf:"priority"`
|
|
SaturationCeiling float64 `koanf:"saturation_ceiling"`
|
|
}
|
|
|
|
// Timeouts bounds each phase of an upstream request.
|
|
type Timeouts struct {
|
|
Dial time.Duration `koanf:"dial"`
|
|
// ResponseHeader bounds the wait for headers on a streaming request. It is
|
|
// a liveness check there, because headers arrive before generation starts.
|
|
ResponseHeader time.Duration `koanf:"response_header"`
|
|
// Generation bounds a non-streaming request, where the engine sends no
|
|
// headers until the completion is done. Setting response_header alone used
|
|
// to cap both, so 60s read as a liveness guard and behaved as a limit of
|
|
// about a hundred tokens on a slow model.
|
|
Generation time.Duration `koanf:"generation"`
|
|
FirstToken time.Duration `koanf:"first_token"`
|
|
Idle time.Duration `koanf:"idle"`
|
|
Total time.Duration `koanf:"total"`
|
|
}
|
|
|
|
// Scoring holds the thresholds the scorers normalize against. These want to be
|
|
// measured with ctl bench (plan §10), not guessed.
|
|
type Scoring struct {
|
|
TargetWaiting float64 `koanf:"target_waiting"`
|
|
TargetInflight float64 `koanf:"target_inflight"`
|
|
KVCeiling float64 `koanf:"kv_ceiling"`
|
|
SessionLoadFactor float64 `koanf:"session_load_factor"`
|
|
}
|
|
|
|
// Endpoint is one engine replica. Static in phase 1.
|
|
type Endpoint struct {
|
|
ID string `koanf:"id"`
|
|
URL string `koanf:"url"`
|
|
// APIKey is sent to the engine as a bearer token, for an engine started
|
|
// with one.
|
|
APIKey string `koanf:"api_key"`
|
|
Models []string `koanf:"models"`
|
|
// Engine names the inference engine, which decides what its metrics are
|
|
// called. Empty means vLLM's names.
|
|
Engine string `koanf:"engine"`
|
|
Pool string `koanf:"pool"`
|
|
Weight float64 `koanf:"weight"`
|
|
Priority int `koanf:"priority"`
|
|
}
|
|
|
|
// Default returns the configuration used when nothing is specified.
|
|
func Default() Server {
|
|
return Server{
|
|
Listen: ":8080",
|
|
AdminListen: "127.0.0.1:9090",
|
|
AgentListen: ":7443",
|
|
Mirror: Mirror{UpdateCheck: 6 * time.Hour},
|
|
Control: Control{
|
|
Interval: 5 * time.Second,
|
|
BaseBackoff: 10 * time.Second,
|
|
MaxBackoff: 10 * time.Minute,
|
|
UnreachableAfter: 15 * time.Second,
|
|
LostAfter: 60 * time.Second,
|
|
PortRange: []int{20000, 20999},
|
|
MgmtInterface: "eth0",
|
|
},
|
|
Router: Router{
|
|
ScrapeInterval: 2 * time.Second,
|
|
ScrapeTimeout: time.Second,
|
|
StaleAfter: 10 * time.Second,
|
|
Retries: 2,
|
|
Cooldown: 15 * time.Second,
|
|
FailureThreshold: 3,
|
|
DefaultProfile: "chat",
|
|
TenantLabel: "team",
|
|
SessionHeader: "x-session-id",
|
|
InjectUsage: true,
|
|
MaxBodyBytes: 32 << 20,
|
|
Timeouts: Timeouts{
|
|
Dial: 5 * time.Second,
|
|
ResponseHeader: 60 * time.Second,
|
|
FirstToken: 120 * time.Second,
|
|
Idle: 60 * time.Second,
|
|
Total: 30 * time.Minute,
|
|
},
|
|
Scoring: Scoring{
|
|
TargetWaiting: 4,
|
|
TargetInflight: 8,
|
|
KVCeiling: 0.9,
|
|
SessionLoadFactor: 1.25,
|
|
},
|
|
Admission: Admission{
|
|
Enabled: true,
|
|
BandHeader: "x-llmgw-band",
|
|
DefaultBand: "gold",
|
|
Bands: []Band{
|
|
{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
|
|
{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
|
|
{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
|
|
{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
|
|
},
|
|
QueueTTL: 30 * time.Second,
|
|
QueueCapacityPerBand: 2000,
|
|
Interval: 10 * time.Millisecond,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// Load reads configuration from path, overlaying it on the defaults, then
|
|
// applies LLMGW_-prefixed environment overrides.
|
|
//
|
|
// An empty path loads the defaults, which is what makes `llmgw server` with a
|
|
// couple of --endpoint flags usable for a smoke test.
|
|
func Load(path string) (Server, error) {
|
|
cfg := Default()
|
|
k := koanf.New(".")
|
|
|
|
if path != "" {
|
|
if _, err := os.Stat(path); err != nil {
|
|
return cfg, fmt.Errorf("config file: %w", err)
|
|
}
|
|
if err := k.Load(file.Provider(path), yaml.Parser()); err != nil {
|
|
return cfg, fmt.Errorf("parsing %s: %w", path, err)
|
|
}
|
|
}
|
|
|
|
// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
|
|
err := k.Load(env.Provider(".", env.Opt{
|
|
Prefix: "LLMGW_",
|
|
TransformFunc: func(key, value string) (string, any) {
|
|
key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
|
|
return strings.ReplaceAll(key, "__", "."), value
|
|
},
|
|
}), nil)
|
|
if err != nil {
|
|
return cfg, fmt.Errorf("environment overrides: %w", err)
|
|
}
|
|
|
|
if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
|
|
return cfg, fmt.Errorf("decoding configuration: %w", err)
|
|
}
|
|
return cfg, cfg.Validate()
|
|
}
|
|
|
|
// Validate rejects configurations that would otherwise fail confusingly at
|
|
// request time.
|
|
func (s Server) Validate() error {
|
|
for i, o := range s.EngineOverrides {
|
|
if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 {
|
|
return fmt.Errorf("engine_overrides[%d] picks no nodes: give it a gpu_class, nodes or labels", i)
|
|
}
|
|
if len(o.Engines) == 0 {
|
|
return fmt.Errorf("engine_overrides[%d] changes no engine: give it engines, with \"\" for none", i)
|
|
}
|
|
}
|
|
if s.Listen == "" {
|
|
return fmt.Errorf("listen is empty")
|
|
}
|
|
if err := s.AgentTLS.Validate(); err != nil {
|
|
return err
|
|
}
|
|
if s.Auth.Enabled && s.Auth.Store == "" {
|
|
return fmt.Errorf("auth.store is required when auth is enabled")
|
|
}
|
|
switch s.Router.TenantLabel {
|
|
case "", "none", "org", "team", "project":
|
|
default:
|
|
return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel)
|
|
}
|
|
if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour {
|
|
return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention)
|
|
}
|
|
if s.Threads.Store != "" && s.Threads.RetainHours <= 0 {
|
|
return fmt.Errorf("threads.retain_hours must be positive when threads.store is set")
|
|
}
|
|
|
|
seen := make(map[string]struct{}, len(s.Endpoints))
|
|
for i, e := range s.Endpoints {
|
|
switch {
|
|
case e.ID == "":
|
|
return fmt.Errorf("endpoints[%d]: id is required", i)
|
|
case e.URL == "":
|
|
return fmt.Errorf("endpoint %q: url is required", e.ID)
|
|
case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
|
|
return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
|
|
case len(e.Models) == 0:
|
|
return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
|
|
}
|
|
if _, dup := seen[e.ID]; dup {
|
|
return fmt.Errorf("endpoint id %q is used twice", e.ID)
|
|
}
|
|
seen[e.ID] = struct{}{}
|
|
}
|
|
|
|
// The catalog enforces the rest — aliases, fallbacks, parallelism
|
|
// arithmetic — so there is one place those rules live.
|
|
if _, err := s.Catalog(); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Catalog builds the model catalog from the configuration.
|
|
func (s Server) Catalog() (*catalog.Catalog, error) {
|
|
return catalog.New(s.Models)
|
|
}
|
|
|
|
// ControlPlaneMode reports whether the server should run a control plane.
|
|
//
|
|
// A model with no profiles cannot be placed, so a configuration of bare names
|
|
// plus a static endpoint list is the router-only mode used for testing against
|
|
// sim-engines.
|
|
//
|
|
// With no static endpoints there is nothing a router-only process could serve,
|
|
// so a configuration naming neither endpoints nor models is a control plane
|
|
// with an empty catalog — which is what every new cluster is. It used to be
|
|
// refused as having nothing to serve, and under systemd that is a unit
|
|
// restarting every two seconds with nothing on 8080.
|
|
func (s Server) ControlPlaneMode() bool {
|
|
if s.Control.Enabled || len(s.Endpoints) == 0 {
|
|
return true
|
|
}
|
|
for _, m := range s.Models {
|
|
if len(m.Profiles) > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// DefaultEngines are the engine versions this release was checked against:
|
|
// every flag llmgw builds for them is one they take.
|
|
//
|
|
// vLLM is installed on every node with an NVIDIA GPU it can run on and
|
|
// skipped elsewhere. SGLang is not here: it is opt-in, by naming a version
|
|
// under engines.
|
|
var DefaultEngines = map[string]string{
|
|
"llamacpp": "b10991",
|
|
"vllm": "0.30.0",
|
|
}
|
|
|
|
// EngineOverride gives the nodes it picks different engines. Every selector
|
|
// set must match; at least one must be set.
|
|
type EngineOverride struct {
|
|
// GPUClass picks nodes with a GPU of this class ("v100-32g").
|
|
GPUClass string `koanf:"gpu_class"`
|
|
// Nodes picks nodes by id.
|
|
Nodes []string `koanf:"nodes"`
|
|
// Labels picks nodes carrying all of these.
|
|
Labels map[string]string `koanf:"labels"`
|
|
// Engines are laid over the cluster's: a version, or "" for none.
|
|
Engines map[string]string `koanf:"engines"`
|
|
}
|
|
|
|
// Picks reports whether the override applies to a node with this id, these
|
|
// GPU classes and these labels.
|
|
func (o EngineOverride) Picks(node string, classes []string, labels map[string]string) bool {
|
|
if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 {
|
|
return false
|
|
}
|
|
if o.GPUClass != "" && !slices.Contains(classes, o.GPUClass) {
|
|
return false
|
|
}
|
|
if len(o.Nodes) > 0 && !slices.Contains(o.Nodes, node) {
|
|
return false
|
|
}
|
|
for k, v := range o.Labels {
|
|
if labels[k] != v {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// EnginesFor is what one node installs: WantedEngines with every override
|
|
// that picks it laid over, in order.
|
|
func (s Server) EnginesFor(node string, classes []string, labels map[string]string) map[string]string {
|
|
out := s.WantedEngines()
|
|
for _, o := range s.EngineOverrides {
|
|
if !o.Picks(node, classes, labels) {
|
|
continue
|
|
}
|
|
for k, v := range o.Engines {
|
|
if v == "" {
|
|
delete(out, k)
|
|
continue
|
|
}
|
|
out[k] = v
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// WantedEngines is DefaultEngines with the configuration laid over it.
|
|
func (s Server) WantedEngines() map[string]string {
|
|
out := map[string]string{}
|
|
for k, v := range DefaultEngines {
|
|
out[k] = v
|
|
}
|
|
for k, v := range s.Engines {
|
|
if v == "" {
|
|
delete(out, k)
|
|
continue
|
|
}
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|