// Package config loads llmgw's YAML configuration. // // Phase 1 only needs the router's half of it: the endpoint list is static here // and will come from the control plane in phase 3 (plan §6.4). The shape is // already the one the plan describes, so that later change stays additive. package config import ( "fmt" "os" "slices" "strings" "time" "github.com/knadh/koanf/parsers/yaml" env "github.com/knadh/koanf/providers/env/v2" "github.com/knadh/koanf/v2" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/notify" "gitea.mynodes.xyz/ilker/llmgw/internal/obs" "gitea.mynodes.xyz/ilker/llmgw/internal/secrets" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" ) // Server is the whole server configuration. type Server struct { // Listen is the client-facing OpenAI API address. Listen string `koanf:"listen"` // AdminListen serves the dashboard, the management API, health and pprof. // // Loopback by default, and it should stay there: pprof can dump the process // heap, and API keys are in that heap in plaintext. Every comment beside // this setting has always said not to expose it, while the default bound // every interface — a default that contradicts its own documentation is a // footgun, so the default now matches the advice. AdminListen string `koanf:"admin_listen"` // MetricsListen serves Prometheus metrics and nothing else. // // Separate from AdminListen because the reason that one is loopback is // pprof, not metrics — and metrics are the one thing on it that something // else genuinely needs to reach. Sharing a listener meant a fleet could // either be monitored or keep its heap private, which is not a choice // anybody should have to make. // // Empty means off, and metrics remain available on AdminListen either way, // so nothing that scrapes it today has to change. MetricsListen string `koanf:"metrics_listen"` // AgentListen is where node agents connect. Separate from the client port // so the fleet-facing surface can be firewalled independently. AgentListen string `koanf:"agent_listen"` // Auth turns on API keys on the client-facing port. Auth Auth `koanf:"auth"` // Threads keeps conversations and agent memory without API keys, for a // gateway on one trusted machine. With auth on they live in the auth store, // per project, and this block is ignored. Threads Threads `koanf:"threads"` // Landing configures the page served at "/". Landing Landing `koanf:"landing"` // AgentTLS authenticates the fleet-facing port. It deliberately does not // cover the client-facing one: that surface is fronted by a publicly // trusted certificate, while this one is mutual and private. AgentTLS tlsconf.Config `koanf:"agent_tls"` // MaxConnections bounds the client port's open connections; past it new // ones wait in the kernel's backlog. 0 is no bound. MaxConnections int `koanf:"max_connections"` // AdminTokenFile holds the token the admin API's /api and /debug paths // demand. Empty is admin-token in the systemd unit's state directory // (/var/lib/llmgw), or secrets/admin-token beside this file for a server // run by hand. Made, 0600, when the server starts and it does not exist; // `sudo llmgw ctl` and the console read it from there. AdminTokenFile string `koanf:"admin_token_file"` // AdminRemote allows admin_listen on an address other than loopback. The // token still guards the API, but it then crosses the network in clear. AdminRemote bool `koanf:"admin_remote"` // DebugPprof serves Go's profiler on the admin listener, behind the token. // Off by default: a heap profile holds the API keys requests presented. DebugPprof bool `koanf:"debug_pprof"` // AgentInsecure lets a control plane run its agent port without // agent_tls: plaintext, and any machine that reaches it may register as // any node — be given replicas, and so be sent users' prompts — or, in // the other direction, a machine in the path may tell every agent what // to run. Refused unless said here, and then the port listens on // loopback unless agent_listen names an address. For one machine, a // test, or a network nothing untrusted reaches. AgentInsecure bool `koanf:"agent_insecure"` // Monitoring is where the dashboard reads its history from. Monitoring Monitoring `koanf:"monitoring"` // Tracing sends traces to an OpenTelemetry collector: each inference // request, and on a control plane each placement, replica start and // model job. Off unless an endpoint is set here or in // OTEL_EXPORTER_OTLP_ENDPOINT. Tracing obs.Tracing `koanf:"tracing"` // Mirror points the control plane at a model mirror it can administer. // Empty means there is none, and /admin/mirror stays unmounted. Mirror Mirror `koanf:"mirror"` // Offline is a cluster that reaches nothing outside itself: its nodes // install engines, and upgrades fetch releases, from the mirror, out of // a bundle carried in (llmgw bundle create, llmgw bundle import). Offline Offline `koanf:"offline"` // Engines are the engines every node installs, and at which version. The // agents install them themselves — the right build for their hardware, // checked against its publisher's checksum and seen to run — so a node is // never trusted to already have what somebody once put there. // // Unset keeps DefaultEngines. A version replaces the default; an empty // one leaves that engine to be installed by hand. Engines map[string]string `koanf:"engines"` // EngineOverrides give some nodes different engines from the rest: a // vLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at // all on nodes meant only for llama.cpp. Each picks nodes and lays its // engines over the cluster's, in order, later ones last; an empty version // removes that engine from the nodes it picks. EngineOverrides []EngineOverride `koanf:"engine_overrides"` // BackupDir is where `llmgw server backup` writes, and where the newest // archive's age is read for llmgw_backup_last_success_timestamp_seconds. // Empty means /var/backups/llmgw-server. BackupDir string `koanf:"backup_dir"` // Guard is the library of guardrail policies: what requests and answers // are checked for, and what is done. Applied to every request through // guard.default, and to parts of the tenancy tree by attaching them // (llmgw ctl guard attach). Empty checks nothing. Guard guard.Config `koanf:"guard"` // Secrets is where secrets kept outside files are read from, and the key // the store's own secrets are sealed under. Secrets Secrets `koanf:"secrets"` // Notify sends what needs a person — a model parked, a node lost, a // budget spent, new weights published — to webhooks, signed, without a // Prometheus and Alertmanager in between. Empty sends nothing. Notify notify.Config `koanf:"notify"` // Firewall turns llmgw's firewall on every machine on or off. Firewall FirewallSettings `koanf:"firewall"` // FirewallNetworks are networks, besides those each machine reaches the // cluster on, that llmgw's internal ports answer: a monitoring host on // another subnet, nodes behind a router. Written into every machine's // firewall by `server init`, `node add` and `server add`. FirewallNetworks []string `koanf:"firewall_networks"` // BackupRecipients encrypt every archive `llmgw server backup` writes, to // whoever holds one of their keys: age recipients (age1…) or SSH public // keys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds // the certificate authority's key and every secret in the store, and is // copied off the machine; encrypted, the copy is safe wherever it lands. // Empty writes archives unencrypted, readable by root only. BackupRecipients []string `koanf:"backup_recipients"` // BackupCopyTo is where each archive is copied once written, so it // survives this machine: a directory (a mounted share, another disk) or // ssh://user@host[:port]/path, reached with BackupCopyKey and checked // against /etc/llmgw/backup_known_hosts, which learns a host the first // time. The same number of archives is kept there as here. Refused // without backup_recipients: an archive leaves the machine encrypted. BackupCopyTo []string `koanf:"backup_copy_to"` // BackupCopyKey is the SSH private key for ssh:// copies. BackupCopyKey string `koanf:"backup_copy_key"` // Releases is where `llmgw ctl upgrade` fetches release binaries from, // for this machine and — through their agents — every node. Empty means // the project's own releases. Releases string `koanf:"releases"` Router Router `koanf:"router"` Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` // Facility is the power caps of racks, PDUs and other domains, and how // a replica's draw is estimated against them. Where a node is comes from // its llmgw.io/site, room, row, rack and pdu labels; holding a domain for // maintenance is `llmgw ctl facility hold`, not configuration. Facility Facility `koanf:"facility"` // Federation is how this cluster takes part in federation with other // llmgw clusters. The peers themselves are added at runtime, with // `llmgw ctl federation add`, and kept in the store. Federation Federation `koanf:"federation"` // Endpoints is the phase-1 static replica list. It is only consulted when // no model declares a profile, which is what makes a router-only smoke test // against sim-engines possible without a control plane. Endpoints []Endpoint `koanf:"endpoints"` } // Facility configures facility-aware placement: what each rack, PDU or other // domain may draw, and the figures a replica's draw is estimated from when // its profile declares none (power_watts). // // Nothing here is needed for placement to know where nodes are or to spread // replicas across racks; that is the nodes' labels alone. A domain with no // budget is not capped. type Facility struct { // Budgets are the domains with a power cap. Budgets []FacilityBudget `koanf:"budgets"` // Margin is the share of every cap kept free, 0.1 for ten percent. Zero // keeps none, for caps already given derated. Margin float64 `koanf:"margin"` // GPUWatts is a card's draw when it does not report its power limit: // an agent without nvidia-smi's power fields, or a node that has not // sent a heartbeat yet. Zero counts such a card as drawing nothing. GPUWatts float64 `koanf:"gpu_watts"` // CPUCoreWatts is the draw of a core reserved by a replica with no GPU // (cpu_cores). Zero counts such replicas as drawing nothing. CPUCoreWatts float64 `koanf:"cpu_core_watts"` // NodeWatts is what a node draws besides its cards and reserved cores — // idle CPUs, memory, fans, disks — counted for every node in a capped // domain, busy or not. NodeWatts float64 `koanf:"node_watts"` // MeasuredFresh is how old a node's power reading may be and still be // counted against its domains. Zero takes two minutes. MeasuredFresh time.Duration `koanf:"measured_fresh"` } // FacilityBudget is one domain's power cap. type FacilityBudget struct { // Domain is level:name, as the nodes' labels give it: rack:r12-07, // pdu:r12-07-a, row:12. Domain string `koanf:"domain"` // CapWatts is what the domain may draw. For a dual-fed rack it is what // one feed carries alone, derated as the facility derates it (usually // 80%), since that is what the rack has to survive. CapWatts float64 `koanf:"cap_watts"` // OtherWatts is drawn in the domain by what llmgw does not place on: // switches, storage, machines outside the cluster. OtherWatts float64 `koanf:"other_watts"` } // Policy is the configuration as placement reads it. func (f Facility) Policy() (facility.Policy, error) { p := facility.Policy{ Margin: f.Margin, GPUWatts: f.GPUWatts, CPUCoreWatts: f.CPUCoreWatts, NodeWatts: f.NodeWatts, MeasuredFresh: f.MeasuredFresh, } if f.Margin < 0 || f.Margin >= 1 { return p, fmt.Errorf("facility.margin %v is a share of the cap, from 0 to below 1: 0.1 keeps ten percent free", f.Margin) } if f.GPUWatts < 0 || f.CPUCoreWatts < 0 || f.NodeWatts < 0 { return p, fmt.Errorf("facility: gpu_watts, cpu_core_watts and node_watts cannot be negative") } for i, b := range f.Budgets { d, err := facility.ParseDomain(b.Domain) if err != nil { return p, fmt.Errorf("facility.budgets[%d]: %w", i, err) } if b.CapWatts <= 0 { return p, fmt.Errorf("facility.budgets[%d] (%s): cap_watts must be above zero", i, d) } if b.OtherWatts < 0 { return p, fmt.Errorf("facility.budgets[%d] (%s): other_watts cannot be negative", i, d) } if p.Budgets == nil { p.Budgets = map[facility.Domain]facility.Budget{} } if _, dup := p.Budgets[d]; dup { return p, fmt.Errorf("facility.budgets[%d]: %s is given twice", i, d) } p.Budgets[d] = facility.Budget{CapWatts: b.CapWatts, OtherWatts: b.OtherWatts} } return p, nil } // Federation is this cluster's side of federation with other llmgw clusters. type Federation struct { // Name is this cluster as a peer reads it, in the federated_from tag // sent with a request to a peer set to receive tags (`ctl federation add // --send-tags`). Empty is this machine's host name. Name string `koanf:"name"` // Interval is how often each peer is asked whether it is ready, timed, // and what it offers. A peer that does not answer is taken out of // routing until it does. Zero is 10s. Interval time.Duration `koanf:"interval"` } // Monitoring points the dashboard at a Prometheus holding this cluster's // history. // // Read back rather than kept twice. llmgw publishes these metrics already; what // a gauge cannot do is remember, and a ring buffer in this process would be a // rival copy with its own retention, lost on every restart and disagreeing with // the one an operator already has on screen. type Monitoring struct { // Prometheus is the server's address, for example // http://monitoring.internal:9090. Empty leaves the charts off the // dashboard rather than showing an error where one should be. Prometheus string `koanf:"prometheus"` // Window is how far back the charts look. Zero takes an hour. Window time.Duration `koanf:"window"` // Timeout bounds one query. It runs while somebody waits for a page, so a // monitoring system that is slow today is skipped rather than waited for. Timeout time.Duration `koanf:"timeout"` } // Mirror is a model mirror this control plane administers over its API. // // The mirror is a separate machine with a large disk, not part of the gateway, // so what is configured here is an address and how to authenticate to it — // never a directory. A control plane that could write to the mirror's disk // directly would be a way to replace a model with another one that every node // would then fetch without complaint. type Mirror struct { // API is the mirror's management address, scheme included, for example // https://models.internal:8082. Empty means no mirror is configured. // // The management port, not the one nodes fetch bytes from. Those are two // listeners on the mirror precisely so this one can be locked down without // re-certifying every agent's fetcher. API string `koanf:"api"` // TLS authenticates to the mirror's management API, with the same CA the // agents use. Empty leaves the connection in plain HTTP, which is only // acceptable on a network where nothing untrusted can reach that port — // the mirror says as much in its own log when it starts that way. TLS tlsconf.Config `koanf:"tls"` // Registry is the model registry the mirror page browses, empty for the // public Hugging Face. Set it to an internal mirror of the registry where // the control plane cannot reach the internet — or leave the whole mirror // block out, which turns the explorer off rather than giving it a search // box that can only time out. Registry string `koanf:"registry"` // Timeout bounds a management call. It does not bound a download: the // mirror answers a fetch as soon as it has started one, and the bytes keep // arriving long after this deadline has passed. Timeout time.Duration `koanf:"timeout"` // UpdateCheck is how often the registry is asked whether the files of // the models served from it have changed there: a repository can publish // new weights under the same name. Only told, never applied — moving a // model to them is `llmgw ctl model update`. 0 never asks. UpdateCheck time.Duration `koanf:"update_check"` } // Landing configures the public page at "/". // // It exists for one thing the gateway cannot know: who to ask for a key. A page // that explains the API and then leaves the reader with no route to using it is // where onboarding stops. type Landing struct { // Contact is where someone without a key should go — an address, a URL, a // channel name. Rendered as a link when it is one, as text otherwise. Contact string `koanf:"contact"` // Notice is an optional line of your own, for whatever the reader needs to // know that this page cannot guess. Notice string `koanf:"notice"` // Language is what the web portal speaks to someone who has not chosen // a language with the switcher at the foot of every page and whose // browser asks for none it speaks: "en" or "tr". Empty is English. Language string `koanf:"language"` // DocsSignIn serves the docs portal at /docs only to someone signed in. // Off, the portal is public like this page: its client guide is for the // person who has no account yet, and the operator documents describe // llmgw in general — no address, model or key of this installation — // which the project's repository publishes anyway. On, a visitor is sent // to sign in, and without accounts the portal needs an API key. DocsSignIn bool `koanf:"docs_sign_in"` } // Threads configures conversation and memory storage for a gateway without // authentication. // // Everything stored is in one pool with no owner, readable by anyone who can // reach the client port and knows an id. That is acceptable on one machine and // wrong on a shared network, which is why it takes a setting rather than being // the default. type Threads struct { // Store is the SQLite file. Empty means nothing is kept. Store string `koanf:"store"` // RetainHours is how long idle conversations and notes are kept. Required // with Store: a store with no expiry keeps everything anyone typed, by // omission. RetainHours int `koanf:"retain_hours"` } // Auth configures client authentication (plan §7.9). // // Off by default, because a gateway on a trusted network with no keys issued // still has to serve. Turning it on is what makes the client-facing port safe // to expose. type Auth struct { Enabled bool `koanf:"enabled"` // Store is the SQLite file holding keys, tenancy and usage. Store string `koanf:"store"` // CacheTTL bounds how long a revoked key keeps working, traded against a // database read per key per TTL. CacheTTL time.Duration `koanf:"cache_ttl"` // NegativeTTL is how long a rejected key is remembered as rejected, so a // client looping with a bad key cannot become a database read per request. NegativeTTL time.Duration `koanf:"negative_ttl"` // DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to. // // Unlimited used to be the default, and a project created without limits is // the easy path — the "default" project on this cluster still is one. The // first sign of that is the bill or the outage. Zero keeps the old // behaviour for anyone who wants it, but it should be a choice rather than // an omission. A fallback, not a ceiling: an explicit limit at any scope // wins, above or below. DefaultRPM int `koanf:"default_rpm"` DefaultTPM int `koanf:"default_tpm"` // SignupOpen serves the signup form. Accounts it creates are pending until // an operator approves them, so this is not a way in — but turning it off // means accounts are made with `llmgw users create` instead. SignupOpen bool `koanf:"signup_open"` // OIDC is sign-in through an identity provider. OIDC OIDC `koanf:"oidc"` // SCIM is account provisioning by the identity provider. SCIM SCIM `koanf:"scim"` // MaxKeyTTL is how long any API key may live; a key asked for with no // expiry is given this. Organizations, teams and projects can set // shorter ones (llmgw keys ttl). Zero is no limit. MaxKeyTTL time.Duration `koanf:"max_key_ttl"` // Personal is what a person may do on their own, outside any project: make // personal keys and use the chat page personally, within these limits. Personal Personal `koanf:"personal"` // Pricing is what each model costs, for budgets. A model absent from the // table costs whatever DefaultPrice says, which is nothing unless set. Pricing map[string]Price `koanf:"pricing"` DefaultPrice Price `koanf:"default_price"` // FlushInterval and BatchSize decide when usage records are written. FlushInterval time.Duration `koanf:"flush_interval"` BatchSize int `koanf:"batch_size"` // BufferSize bounds the queue. Beyond it records are dropped rather than // blocking a request: losing accounting is bad, turning a stalled database // into an outage is worse. BufferSize int `koanf:"buffer_size"` // UsageRetention is how long the record of each request is kept: who sent // it, which replica answered, how long it queued — what `ctl request` // reads. Zero keeps it for ever. Totals, charts and budgets are read from // five-minute sums kept beside the records, which are not pruned, so this // bounds the table that grows by a row a request without changing any // figure a page shows. At least a day when set. UsageRetention time.Duration `koanf:"usage_retention"` } // Personal configures use that belongs to a person rather than a project. // // On by default: a person need not belong to anything to use the gateway. The // limits are each person's own pool across all their personal keys and their // personal chat, and apply only where nobody wrote a limit for that person. type Personal struct { // Disabled refuses personal keys and personal chat; people then use the // gateway only through projects they are members of. Disabled bool `koanf:"disabled"` RPM int `koanf:"rpm"` TPM int `koanf:"tpm"` // Budget is in whole units per Period ("day" or "month"). Budget float64 `koanf:"budget"` Period string `koanf:"period"` } // Price is a model's cost in micro-units per thousand tokens. // // What a unit means is the operator's choice — self-hosted models have no // per-token price, so it may be GPU-seconds, a notional rate, or 1000 // everywhere, which turns a budget into a token quota (plan §7.9.4). type Price struct { PromptPer1k int64 `koanf:"prompt_per_1k"` CompletionPer1k int64 `koanf:"completion_per_1k"` } // Control configures the control plane. type Control struct { // Interval is how often desired and actual state are compared. Interval time.Duration `koanf:"interval"` // BaseBackoff and MaxBackoff bound how fast a failing model is retried. BaseBackoff time.Duration `koanf:"base_backoff"` MaxBackoff time.Duration `koanf:"max_backoff"` // UnreachableAfter and LostAfter stage node expiry. They are deliberately // far apart: a network blip must not tear down replicas that still serve, // and rebuilding a multi-node replica costs tens of minutes of loading. UnreachableAfter time.Duration `koanf:"unreachable_after"` LostAfter time.Duration `koanf:"lost_after"` // PortRange is the per-node range for engine HTTP and dist-init ports. PortRange []int `koanf:"port_range"` // MgmtInterface is the ethernet interface NCCL bootstraps over. MgmtInterface string `koanf:"mgmt_interface"` // Timezone is where models' scaling schedules are read, an IANA name // like Europe/Istanbul; empty is the control plane's own. Timezone string `koanf:"timezone"` // EvalDir is where quality suites are read from by name, for `llmgw ctl // eval MODEL --suite NAME`, a model's eval gate and the console. The // suites the binary carries (arithmetic, json-format) are found when it // has none of the name; a suite here of the same name replaces one. EvalDir string `koanf:"eval_dir"` // Enabled runs the control plane even when configuration declares no // models. // // Without it, a deployment whose catalog lives entirely in the database // cannot start: control plane mode is inferred from the config file, so an // empty models block means "router only", the process refuses to start for // want of anything to serve, and the models it would have loaded are never // read. That is a cluster that cannot be managed at runtime because it is // managed at runtime. AgentListen cannot carry this signal — it has a // default, so every deployment would become a control plane. Enabled bool `koanf:"enabled"` // LeaseTTL is how long this control plane's claim to be the one placing // work stands without being renewed. // // Only meaningful with a shared store, where a second control plane can // see the claim. Long enough to survive a slow query or a brief network // stall, short enough that a standby takes over before anyone has finished // reading the alert. Zero takes the default. LeaseTTL time.Duration `koanf:"lease_ttl"` } // Router holds routing behaviour. type Router struct { // ActivationWait is how long a request waits for an on-demand model to // load before the client is told to come back. Zero takes the default. // // The one real decision in on-demand loading. Refusing outright makes the // feature useless, since the first request for every model would fail; // waiting without a bound turns a request into a job, and a client that // budgeted thirty seconds ends up holding a connection for ten minutes. ActivationWait time.Duration `koanf:"activation_wait"` ScrapeInterval time.Duration `koanf:"scrape_interval"` ScrapeTimeout time.Duration `koanf:"scrape_timeout"` // StaleAfter is how long an endpoint may go unscraped before being scored // down (plan §7.6). StaleAfter time.Duration `koanf:"stale_after"` // Retries is how many *additional* endpoints may be tried after a failure, // and only while nothing has been written to the client (plan §7.7). Retries int `koanf:"retries"` // Cooldown is how long a failing endpoint leaves the candidate pool. Cooldown time.Duration `koanf:"cooldown"` // FailureThreshold is the consecutive failure count that opens a breaker. // 1 turns the breaker into a pure cooldown. FailureThreshold int `koanf:"failure_threshold"` // DefaultProfile names the routing profile used when a model picks none. DefaultProfile string `koanf:"default_profile"` // Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the // router uses session affinity alone, which is tier 0 and already most of // the win for multi-turn traffic. Prefix Prefix `koanf:"prefix"` // KVIndex tunes the precise KV index (plan §7.5 tier 2). Which models // use it is the catalog's to say, by kv_events on a vLLM profile; this // is only how. KVIndex KVIndex `koanf:"kv_index"` // SessionHeader carries the session id for affinity (plan §7.5 tier 0). SessionHeader string `koanf:"session_header"` // InjectUsage asks engines for usage even when the client did not, so token // accounting works regardless of client behaviour. InjectUsage bool `koanf:"inject_usage"` // TenantLabel is the level of the tree the per-tenant request series are // labelled by — llmgw_tenant_requests_total and its siblings: "org", // "team" or "project", or "none" for no such series. A label per project // on a cluster of thousands is more series than Prometheus wants, and a // team is what an alert is usually about; "team" is the default. TenantLabel string `koanf:"tenant_label"` // TagLabels are the request tags (metadata, x-llmgw-tags) counted in // llmgw_tagged_requests_total and llmgw_tagged_tokens_total, by value. // Only these: a tag is the client's to set, and every value of one is // a series. The first 100 values of each are counted apart, the rest as // "other". Empty counts none; the usage records keep every tag anyway. TagLabels []string `koanf:"tag_labels"` // MaxBodyBytes bounds the request body the router will read. MaxBodyBytes int64 `koanf:"max_body_bytes"` // Batch is the Files and Batch APIs (/v1/files, /v1/batches): a file of // requests served at the lowest priority, its answers written to // another. Needs the store. Batch Batch `koanf:"batch"` // Cache bounds the answers kept for the projects that reuse them // (llmgw keys cache). Which projects do, and for how long, is theirs // to set; this is only how much memory it may take. Cache Cache `koanf:"cache"` // TrustedProxies are the reverse proxies in front of the client port, // as addresses or CIDR prefixes. Their X-Forwarded-For, -Proto and -Host // are believed; anyone else's are not, since they decide the address // written into the setup scripts and whose attempts are counted where. TrustedProxies []string `koanf:"trusted_proxies"` Timeouts Timeouts `koanf:"timeouts"` Scoring Scoring `koanf:"scoring"` Admission Admission `koanf:"admission"` } // Admission configures the flow control layer (plan §7.2). type Admission struct { // Enabled turns the layer on. Off, every request goes straight through, // which is the v0.2.0 behaviour. Enabled bool `koanf:"enabled"` Bands []Band `koanf:"bands"` DefaultBand string `koanf:"default_band"` // BandHeader lets a client declare its band. Until authentication lands in // v1.0.0 this is trusted, so the port must not be exposed to callers who // should not be able to promote themselves. BandHeader string `koanf:"band_header"` QueueTTL time.Duration `koanf:"queue_ttl"` QueueCapacityPerBand int `koanf:"queue_capacity_per_band"` // QueueCapacityTotal bounds everything queued across bands. Reached, a // request is not refused while a lower band has any queued: the newest of // the lowest band is turned away instead, with a 503 and Retry-After. // Zero takes half of all the bands' capacities together. QueueCapacityTotal int `koanf:"queue_capacity_total"` Interval time.Duration `koanf:"interval"` } // Prefix configures approximate prefix routing. type Prefix struct { Enabled bool `koanf:"enabled"` // MaxNodes bounds the tree, MinMatch is the shortest match worth acting on // in bytes, and TTL is how long an entry is believed — an engine evicts on // its own schedule, so an old record may describe a cache that is gone. MaxNodes int `koanf:"max_nodes"` MinMatch int `koanf:"min_match"` TTL time.Duration `koanf:"ttl"` // MatchRatio is the share of the prompt that must match to be worth acting // on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip // before affinity is abandoned. MatchRatio float64 `koanf:"match_ratio"` BalanceAbs float64 `koanf:"balance_abs"` BalanceRel float64 `koanf:"balance_rel"` } // KVIndex tunes routing by the blocks each vLLM replica reports holding. type KVIndex struct { // TokenizeTimeout bounds the call that turns a request into token ids, // made to one of the model's replicas before the request is placed. Past // it the request is routed as tier 1 would. Zero takes 250ms. TokenizeTimeout time.Duration `koanf:"tokenize_timeout"` // MinBlocks is the fewest leading blocks a replica must hold to be // preferred for them; fewer is coincidence — a chat template's opening // tokens — rather than a prefix worth following. Zero takes 4, which is // 64 tokens at vLLM's usual block size of 16. MinBlocks int `koanf:"min_blocks"` // MaxBlocks bounds how many of a prompt's blocks are hashed and looked // up. Beyond it a longer match still helps, but the difference no longer // decides anything. Zero takes 4096. MaxBlocks int `koanf:"max_blocks"` } // Band is one priority class. type Band struct { Name string `koanf:"name"` Priority int `koanf:"priority"` SaturationCeiling float64 `koanf:"saturation_ceiling"` } // Timeouts bounds each phase of an upstream request. type Timeouts struct { Dial time.Duration `koanf:"dial"` // ResponseHeader bounds the wait for headers on a streaming request. It is // a liveness check there, because headers arrive before generation starts. ResponseHeader time.Duration `koanf:"response_header"` // Generation bounds a non-streaming request, where the engine sends no // headers until the completion is done. Setting response_header alone used // to cap both, so 60s read as a liveness guard and behaved as a limit of // about a hundred tokens on a slow model. Generation time.Duration `koanf:"generation"` FirstToken time.Duration `koanf:"first_token"` Idle time.Duration `koanf:"idle"` Total time.Duration `koanf:"total"` } // Scoring holds the thresholds the scorers normalize against. These want to be // measured with ctl bench (plan §10), not guessed. type Scoring struct { TargetWaiting float64 `koanf:"target_waiting"` TargetInflight float64 `koanf:"target_inflight"` KVCeiling float64 `koanf:"kv_ceiling"` SessionLoadFactor float64 `koanf:"session_load_factor"` } // Endpoint is one engine replica. Static in phase 1. type Endpoint struct { ID string `koanf:"id"` URL string `koanf:"url"` // APIKey is sent to the engine as a bearer token, for an engine started // with one. APIKey string `koanf:"api_key"` Models []string `koanf:"models"` // Engine names the inference engine, which decides what its metrics are // called. Empty means vLLM's names. Engine string `koanf:"engine"` Pool string `koanf:"pool"` Weight float64 `koanf:"weight"` Priority int `koanf:"priority"` } // Secrets is where secrets are read from besides files, and how the // store's own are sealed. // // Wherever a setting names a file holding a secret — the store's @address, // api_key_file, client_secret_file, token_file, secret_file — it may name // env:VARIABLE or vault:mount/path#field instead. type Secrets struct { // Vault is how vault: references and vault-transit: are reached. Vault secrets.Vault `koanf:"vault"` // MasterKey seals the secrets the store keeps for itself — what every // engine's key is derived from — so a copy of the store, or of a backup, // does not hold them: vault-transit:KEY seals with Vault's transit engine, // which never lets the key out; otherwise a reference to 32 random bytes // (openssl rand -hex 32), which then must be kept as safely as the store. // Values from before it was set are sealed the first time they are read. MasterKey string `koanf:"master_key"` } // Batch is how the Files and Batch APIs are bounded. type Batch struct { // MaxFileBytes bounds one uploaded file; 100 MB unless set. MaxFileBytes int64 `koanf:"max_file_bytes"` // MaxLines bounds the requests in one batch; 50 000 unless set. MaxLines int `koanf:"max_lines"` // Concurrency is how many of one batch's requests are in flight at // once; 4 unless set. They queue in the lowest admission band, behind // every interactive request. Concurrency int `koanf:"concurrency"` // RetainDays is how long a batch's answer and error files are kept; // 30 unless set. Uploaded files are kept until deleted. RetainDays int `koanf:"retain_days"` } // Cache is how much of this process's memory reused answers may take. The // answers are held here and nowhere else: each gateway process keeps its // own, and a restart begins them again. type Cache struct { // MaxBytes bounds every project's answers together; 128 MB unless set. // Past it, the answers used longest ago go first. MaxBytes int64 `koanf:"max_bytes"` // MaxEntryBytes bounds one answer; a longer one is not kept. 1 MB // unless set. MaxEntryBytes int64 `koanf:"max_entry_bytes"` } // Default returns the configuration used when nothing is specified. func Default() Server { return Server{ Listen: ":8080", AdminListen: "127.0.0.1:9090", AgentListen: ":7443", MaxConnections: 10_000, Mirror: Mirror{UpdateCheck: 6 * time.Hour}, Firewall: FirewallSettings{Enabled: true}, Control: Control{ Interval: 5 * time.Second, BaseBackoff: 10 * time.Second, MaxBackoff: 10 * time.Minute, UnreachableAfter: 15 * time.Second, LostAfter: 60 * time.Second, PortRange: []int{20000, 20999}, MgmtInterface: "eth0", EvalDir: "/etc/llmgw/eval", }, Router: Router{ ScrapeInterval: 2 * time.Second, ScrapeTimeout: time.Second, StaleAfter: 10 * time.Second, Retries: 2, Cooldown: 15 * time.Second, FailureThreshold: 3, DefaultProfile: "chat", TenantLabel: "team", SessionHeader: "x-session-id", InjectUsage: true, MaxBodyBytes: 32 << 20, Timeouts: Timeouts{ Dial: 5 * time.Second, ResponseHeader: 60 * time.Second, FirstToken: 120 * time.Second, Idle: 60 * time.Second, Total: 30 * time.Minute, }, Scoring: Scoring{ TargetWaiting: 4, TargetInflight: 8, KVCeiling: 0.9, SessionLoadFactor: 1.25, }, Admission: Admission{ Enabled: true, BandHeader: "x-llmgw-band", DefaultBand: "gold", Bands: []Band{ {Name: "platinum", Priority: 400, SaturationCeiling: 1.00}, {Name: "gold", Priority: 300, SaturationCeiling: 0.85}, {Name: "silver", Priority: 200, SaturationCeiling: 0.60}, {Name: "bronze", Priority: 100, SaturationCeiling: 0.30}, }, QueueTTL: 30 * time.Second, QueueCapacityPerBand: 2000, Interval: 10 * time.Millisecond, }, }, } } // Load reads configuration from path, overlaying it on the defaults, then // applies LLMGW_-prefixed environment overrides. // // An empty path loads the defaults, which is what makes `llmgw server` with a // couple of --endpoint flags usable for a smoke test. func Load(path string) (Server, error) { var b []byte if path != "" { if _, err := os.Stat(path); err != nil { return Default(), fmt.Errorf("config file: %w", err) } var err error if b, err = os.ReadFile(path); err != nil { //nolint:gosec // the configuration return Default(), fmt.Errorf("config file: %w", err) } } cfg, err := load(b) if err != nil && path != "" && strings.HasPrefix(err.Error(), "parsing settings") { return cfg, fmt.Errorf("parsing %s:%s", path, strings.TrimPrefix(err.Error(), "parsing settings:")) } return cfg, err } // load reads a YAML document of settings over the defaults. func load(b []byte) (Server, error) { cfg := Default() k := koanf.New(".") if len(b) > 0 { if err := k.Load(bytesProvider(b), yaml.Parser()); err != nil { return cfg, fmt.Errorf("parsing settings: %w", err) } } // LLMGW_ROUTER__RETRIES=3 maps onto router.retries. err := k.Load(env.Provider(".", env.Opt{ Prefix: "LLMGW_", TransformFunc: func(key, value string) (string, any) { key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_")) return strings.ReplaceAll(key, "__", "."), value }, }), nil) if err != nil { return cfg, fmt.Errorf("environment overrides: %w", err) } if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil { return cfg, fmt.Errorf("decoding configuration: %w", err) } // So every secret reference in it, from here on, can be read. if err := secrets.Configure(cfg.Secrets.Vault); err != nil { return cfg, err } return cfg, cfg.Validate() } // Validate rejects configurations that would otherwise fail confusingly at // request time. func (s Server) Validate() error { for i, o := range s.EngineOverrides { if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 { return fmt.Errorf("engine_overrides[%d] picks no nodes: give it a gpu_class, nodes or labels", i) } if len(o.Engines) == 0 { return fmt.Errorf("engine_overrides[%d] changes no engine: give it engines, with \"\" for none", i) } } if s.Listen == "" { return fmt.Errorf("listen is empty") } if _, err := s.Facility.Policy(); err != nil { return err } if s.Control.Timezone != "" { if _, err := time.LoadLocation(s.Control.Timezone); err != nil { return fmt.Errorf("control.timezone %q: %w", s.Control.Timezone, err) } } if _, err := ParseBackupRecipients(s.BackupRecipients); err != nil { return err } for i, t := range s.BackupCopyTo { switch { case strings.HasPrefix(t, "ssh://"): if s.BackupCopyKey == "" { return fmt.Errorf("backup_copy_to[%d] is over ssh: give backup_copy_key", i) } case !strings.HasPrefix(t, "/"): return fmt.Errorf("backup_copy_to[%d] %q: an absolute directory or ssh://user@host/path", i, t) } } if len(s.BackupCopyTo) > 0 && len(s.BackupRecipients) == 0 { return fmt.Errorf("backup_copy_to needs backup_recipients: an archive holds the certificate authority's key " + "and every secret in the store, and leaves this machine only encrypted") } if _, err := s.FirewallAllow(); err != nil { return err } if err := s.Notify.Validate(); err != nil { return err } if err := s.Guard.Validate(); err != nil { return err } if err := s.Auth.OIDC.Validate(); err != nil { return err } if err := s.AgentTLS.Validate(); err != nil { return err } if err := s.Offline.Validate(); err != nil { return err } if s.Auth.Enabled && s.Auth.Store == "" { return fmt.Errorf("auth.store is required when auth is enabled") } switch s.Router.TenantLabel { case "", "none", "org", "team", "project": default: return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel) } if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour { return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention) } if s.Threads.Store != "" && s.Threads.RetainHours <= 0 { return fmt.Errorf("threads.retain_hours must be positive when threads.store is set") } seen := make(map[string]struct{}, len(s.Endpoints)) for i, e := range s.Endpoints { switch { case e.ID == "": return fmt.Errorf("endpoints[%d]: id is required", i) case e.URL == "": return fmt.Errorf("endpoint %q: url is required", e.ID) case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"): return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID) case len(e.Models) == 0: return fmt.Errorf("endpoint %q: at least one model is required", e.ID) } if _, dup := seen[e.ID]; dup { return fmt.Errorf("endpoint id %q is used twice", e.ID) } seen[e.ID] = struct{}{} } // The catalog enforces the rest — aliases, fallbacks, parallelism // arithmetic — so there is one place those rules live. if _, err := s.Catalog(); err != nil { return err } return nil } // Catalog builds the model catalog from the configuration. func (s Server) Catalog() (*catalog.Catalog, error) { return catalog.New(s.Models) } // ControlPlaneMode reports whether the server should run a control plane. // // A model with no profiles cannot be placed, so a configuration of bare names // plus a static endpoint list is the router-only mode used for testing against // sim-engines. // // With no static endpoints there is nothing a router-only process could serve, // so a configuration naming neither endpoints nor models is a control plane // with an empty catalog — which is what every new cluster is. It used to be // refused as having nothing to serve, and under systemd that is a unit // restarting every two seconds with nothing on 8080. func (s Server) ControlPlaneMode() bool { if s.Control.Enabled || len(s.Endpoints) == 0 { return true } for _, m := range s.Models { if len(m.Profiles) > 0 { return true } } return false } // DefaultEngines are the engine versions this release was checked against: // every flag llmgw builds for them is one they take. // // vLLM is installed on every node with an NVIDIA GPU it can run on and // skipped elsewhere. SGLang is not here: it is opt-in, by naming a version // under engines. var DefaultEngines = map[string]string{ "llamacpp": "b10991", "vllm": "0.30.0", } // EngineOverride gives the nodes it picks different engines. Every selector // set must match; at least one must be set. type EngineOverride struct { // GPUClass picks nodes with a GPU of this class ("v100-32g"). GPUClass string `koanf:"gpu_class"` // Nodes picks nodes by id. Nodes []string `koanf:"nodes"` // Labels picks nodes carrying all of these. Labels map[string]string `koanf:"labels"` // Engines are laid over the cluster's: a version, or "" for none. Engines map[string]string `koanf:"engines"` } // Picks reports whether the override applies to a node with this id, these // GPU classes and these labels. func (o EngineOverride) Picks(node string, classes []string, labels map[string]string) bool { if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 { return false } if o.GPUClass != "" && !slices.Contains(classes, o.GPUClass) { return false } if len(o.Nodes) > 0 && !slices.Contains(o.Nodes, node) { return false } for k, v := range o.Labels { if labels[k] != v { return false } } return true } // EnginesFor is what one node installs: WantedEngines with every override // that picks it laid over, in order. func (s Server) EnginesFor(node string, classes []string, labels map[string]string) map[string]string { out := s.WantedEngines() for _, o := range s.EngineOverrides { if !o.Picks(node, classes, labels) { continue } for k, v := range o.Engines { if v == "" { delete(out, k) continue } out[k] = v } } return out } // WantedEngines is DefaultEngines with the configuration laid over it. func (s Server) WantedEngines() map[string]string { out := map[string]string{} for k, v := range DefaultEngines { out[k] = v } for k, v := range s.Engines { if v == "" { delete(out, k) continue } out[k] = v } return out } // Offline is a cluster with no way to the internet. // // Everything it would otherwise fetch from outside — llmgw's releases, the // engine builds its nodes install, model files — is carried in as a bundle, // made where there is internet (llmgw bundle create) and imported into the // mirror here (llmgw bundle import). Nodes then fetch from the mirror, and // check what they fetch as strictly as they would online: releases against // the project's signing key, engines against the checksums the control plane // holds from the bundle. type Offline struct { // Mirror is the mirror's file address, as nodes reach it: // http://10.0.0.5:8081 — the port model files are served on, not the // management API. Set, the cluster is offline: nodes install engines // from it and never from their publishers, releases come from it unless // releases names another place, and no registry is asked whether a // model has newer weights. Python engines (vLLM, SGLang) come from PyPI, // which no bundle carries; on an offline node they are installed by hand. Mirror string `koanf:"mirror"` // Checksums is the file of engine checksums this control plane trusts, // kept by llmgw bundle import and llmgw bundle trust; every engine file // a node fetches from the mirror is checked against it. Read again // whenever it changes. Empty is /etc/llmgw/offline.sha256. Checksums string `koanf:"checksums"` } // DefaultOfflineChecksums is where the engine checksums are kept when // offline.checksums does not say. const DefaultOfflineChecksums = "/etc/llmgw/offline.sha256" // Enabled reports whether the cluster is offline. func (o Offline) Enabled() bool { return o.Mirror != "" } // ChecksumsFile is the engine checksums' file. func (o Offline) ChecksumsFile() string { if o.Checksums != "" { return o.Checksums } return DefaultOfflineChecksums } // ReleasesBase is where an offline cluster's releases are on its mirror. func (o Offline) ReleasesBase() string { return strings.TrimSuffix(o.Mirror, "/") + "/_llmgw/releases" } // Validate refuses a mirror address nodes could not fetch from. func (o Offline) Validate() error { if o.Mirror == "" { return nil } if !strings.HasPrefix(o.Mirror, "http://") && !strings.HasPrefix(o.Mirror, "https://") { return fmt.Errorf("offline.mirror %q: give the mirror's file address, http://host:8081", o.Mirror) } if strings.HasSuffix(strings.TrimSuffix(o.Mirror, "/"), ":8082") { return fmt.Errorf("offline.mirror %q is the mirror's management port: give the one nodes fetch files from, :8081", o.Mirror) } return nil } // ReleasesBase is where release binaries are fetched from: releases when // set, the mirror's copy on an offline cluster, and empty — the caller's // default, the project's own releases — otherwise. func (s Server) ReleasesBase() string { if s.Releases != "" { return s.Releases } if s.Offline.Enabled() { return s.Offline.ReleasesBase() } return "" }