llmgw/internal/config/config.go

// Package config loads llmgw's YAML configuration.
//
// Phase 1 only needs the router's half of it: the endpoint list is static here
// and will come from the control plane in phase 3 (plan §6.4). The shape is
// already the one the plan describes, so that later change stays additive.
package config

import (
	"fmt"
	"os"
	"slices"
	"strings"
	"time"

	"github.com/knadh/koanf/parsers/yaml"
	env "github.com/knadh/koanf/providers/env/v2"
	"github.com/knadh/koanf/v2"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
	"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
	"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
	"gitea.mynodes.xyz/ilker/llmgw/internal/secrets"
	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)

// Server is the whole server configuration.
type Server struct {
	// Listen is the client-facing OpenAI API address.
	Listen string `koanf:"listen"`
	// AdminListen serves the dashboard, the management API, health and pprof.
	//
	// Loopback by default, and it should stay there: pprof can dump the process
	// heap, and API keys are in that heap in plaintext. Every comment beside
	// this setting has always said not to expose it, while the default bound
	// every interface — a default that contradicts its own documentation is a
	// footgun, so the default now matches the advice.
	AdminListen string `koanf:"admin_listen"`

	// MetricsListen serves Prometheus metrics and nothing else.
	//
	// Separate from AdminListen because the reason that one is loopback is
	// pprof, not metrics — and metrics are the one thing on it that something
	// else genuinely needs to reach. Sharing a listener meant a fleet could
	// either be monitored or keep its heap private, which is not a choice
	// anybody should have to make.
	//
	// Empty means off, and metrics remain available on AdminListen either way,
	// so nothing that scrapes it today has to change.
	MetricsListen string `koanf:"metrics_listen"`

	// AgentListen is where node agents connect. Separate from the client port
	// so the fleet-facing surface can be firewalled independently.
	AgentListen string `koanf:"agent_listen"`

	// Auth turns on API keys on the client-facing port.
	Auth Auth `koanf:"auth"`

	// Threads keeps conversations and agent memory without API keys, for a
	// gateway on one trusted machine. With auth on they live in the auth store,
	// per project, and this block is ignored.
	Threads Threads `koanf:"threads"`

	// Landing configures the page served at "/".
	Landing Landing `koanf:"landing"`

	// AgentTLS authenticates the fleet-facing port. It deliberately does not
	// cover the client-facing one: that surface is fronted by a publicly
	// trusted certificate, while this one is mutual and private.
	AgentTLS tlsconf.Config `koanf:"agent_tls"`
	// MaxConnections bounds the client port's open connections; past it new
	// ones wait in the kernel's backlog. 0 is no bound.
	MaxConnections int `koanf:"max_connections"`
	// AdminTokenFile holds the token the admin API's /api and /debug paths
	// demand. Empty is admin-token in the systemd unit's state directory
	// (/var/lib/llmgw), or secrets/admin-token beside this file for a server
	// run by hand. Made, 0600, when the server starts and it does not exist;
	// `sudo llmgw ctl` and the console read it from there.
	AdminTokenFile string `koanf:"admin_token_file"`
	// AdminRemote allows admin_listen on an address other than loopback. The
	// token still guards the API, but it then crosses the network in clear.
	AdminRemote bool `koanf:"admin_remote"`
	// DebugPprof serves Go's profiler on the admin listener, behind the token.
	// Off by default: a heap profile holds the API keys requests presented.
	DebugPprof bool `koanf:"debug_pprof"`
	// AgentInsecure lets a control plane run its agent port without
	// agent_tls: plaintext, and any machine that reaches it may register as
	// any node — be given replicas, and so be sent users' prompts — or, in
	// the other direction, a machine in the path may tell every agent what
	// to run. Refused unless said here, and then the port listens on
	// loopback unless agent_listen names an address. For one machine, a
	// test, or a network nothing untrusted reaches.
	AgentInsecure bool `koanf:"agent_insecure"`

	// Monitoring is where the dashboard reads its history from.
	Monitoring Monitoring `koanf:"monitoring"`
	// Tracing sends traces to an OpenTelemetry collector: each inference
	// request, and on a control plane each placement, replica start and
	// model job. Off unless an endpoint is set here or in
	// OTEL_EXPORTER_OTLP_ENDPOINT.
	Tracing obs.Tracing `koanf:"tracing"`

	// Mirror points the control plane at a model mirror it can administer.
	// Empty means there is none, and /admin/mirror stays unmounted.
	Mirror Mirror `koanf:"mirror"`

	// Offline is a cluster that reaches nothing outside itself: its nodes
	// install engines, and upgrades fetch releases, from the mirror, out of
	// a bundle carried in (llmgw bundle create, llmgw bundle import).
	Offline Offline `koanf:"offline"`

	// Engines are the engines every node installs, and at which version. The
	// agents install them themselves — the right build for their hardware,
	// checked against its publisher's checksum and seen to run — so a node is
	// never trusted to already have what somebody once put there.
	//
	// Unset keeps DefaultEngines. A version replaces the default; an empty
	// one leaves that engine to be installed by hand.
	Engines map[string]string `koanf:"engines"`

	// EngineOverrides give some nodes different engines from the rest: a
	// vLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at
	// all on nodes meant only for llama.cpp. Each picks nodes and lays its
	// engines over the cluster's, in order, later ones last; an empty version
	// removes that engine from the nodes it picks.
	EngineOverrides []EngineOverride `koanf:"engine_overrides"`

	// BackupDir is where `llmgw server backup` writes, and where the newest
	// archive's age is read for llmgw_backup_last_success_timestamp_seconds.
	// Empty means /var/backups/llmgw-server.
	BackupDir string `koanf:"backup_dir"`

	// Guard is the library of guardrail policies: what requests and answers
	// are checked for, and what is done. Applied to every request through
	// guard.default, and to parts of the tenancy tree by attaching them
	// (llmgw ctl guard attach). Empty checks nothing.
	Guard guard.Config `koanf:"guard"`

	// Secrets is where secrets kept outside files are read from, and the key
	// the store's own secrets are sealed under.
	Secrets Secrets `koanf:"secrets"`

	// Notify sends what needs a person — a model parked, a node lost, a
	// budget spent, new weights published — to webhooks, signed, without a
	// Prometheus and Alertmanager in between. Empty sends nothing.
	Notify notify.Config `koanf:"notify"`

	// Firewall turns llmgw's firewall on every machine on or off.
	Firewall FirewallSettings `koanf:"firewall"`

	// FirewallNetworks are networks, besides those each machine reaches the
	// cluster on, that llmgw's internal ports answer: a monitoring host on
	// another subnet, nodes behind a router. Written into every machine's
	// firewall by `server init`, `node add` and `server add`.
	FirewallNetworks []string `koanf:"firewall_networks"`

	// BackupRecipients encrypt every archive `llmgw server backup` writes, to
	// whoever holds one of their keys: age recipients (age1…) or SSH public
	// keys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds
	// the certificate authority's key and every secret in the store, and is
	// copied off the machine; encrypted, the copy is safe wherever it lands.
	// Empty writes archives unencrypted, readable by root only.
	BackupRecipients []string `koanf:"backup_recipients"`

	// BackupCopyTo is where each archive is copied once written, so it
	// survives this machine: a directory (a mounted share, another disk) or
	// ssh://user@host[:port]/path, reached with BackupCopyKey and checked
	// against /etc/llmgw/backup_known_hosts, which learns a host the first
	// time. The same number of archives is kept there as here. Refused
	// without backup_recipients: an archive leaves the machine encrypted.
	BackupCopyTo []string `koanf:"backup_copy_to"`
	// BackupCopyKey is the SSH private key for ssh:// copies.
	BackupCopyKey string `koanf:"backup_copy_key"`

	// Releases is where `llmgw ctl upgrade` fetches release binaries from,
	// for this machine and — through their agents — every node. Empty means
	// the project's own releases.
	Releases string `koanf:"releases"`

	Router  Router          `koanf:"router"`
	Control Control         `koanf:"control"`
	Models  []catalog.Model `koanf:"models"`

	// Facility is the power caps of racks, PDUs and other domains, and how
	// a replica's draw is estimated against them. Where a node is comes from
	// its llmgw.io/site, room, row, rack and pdu labels; holding a domain for
	// maintenance is `llmgw ctl facility hold`, not configuration.
	Facility Facility `koanf:"facility"`

	// Federation is how this cluster takes part in federation with other
	// llmgw clusters. The peers themselves are added at runtime, with
	// `llmgw ctl federation add`, and kept in the store.
	Federation Federation `koanf:"federation"`

	// Endpoints is the phase-1 static replica list. It is only consulted when
	// no model declares a profile, which is what makes a router-only smoke test
	// against sim-engines possible without a control plane.
	Endpoints []Endpoint `koanf:"endpoints"`
}

// Facility configures facility-aware placement: what each rack, PDU or other
// domain may draw, and the figures a replica's draw is estimated from when
// its profile declares none (power_watts).
//
// Nothing here is needed for placement to know where nodes are or to spread
// replicas across racks; that is the nodes' labels alone. A domain with no
// budget is not capped.
type Facility struct {
	// Budgets are the domains with a power cap.
	Budgets []FacilityBudget `koanf:"budgets"`
	// Margin is the share of every cap kept free, 0.1 for ten percent. Zero
	// keeps none, for caps already given derated.
	Margin float64 `koanf:"margin"`
	// GPUWatts is a card's draw when it does not report its power limit:
	// an agent without nvidia-smi's power fields, or a node that has not
	// sent a heartbeat yet. Zero counts such a card as drawing nothing.
	GPUWatts float64 `koanf:"gpu_watts"`
	// CPUCoreWatts is the draw of a core reserved by a replica with no GPU
	// (cpu_cores). Zero counts such replicas as drawing nothing.
	CPUCoreWatts float64 `koanf:"cpu_core_watts"`
	// NodeWatts is what a node draws besides its cards and reserved cores —
	// idle CPUs, memory, fans, disks — counted for every node in a capped
	// domain, busy or not.
	NodeWatts float64 `koanf:"node_watts"`
	// MeasuredFresh is how old a node's power reading may be and still be
	// counted against its domains. Zero takes two minutes.
	MeasuredFresh time.Duration `koanf:"measured_fresh"`
}

// FacilityBudget is one domain's power cap.
type FacilityBudget struct {
	// Domain is level:name, as the nodes' labels give it: rack:r12-07,
	// pdu:r12-07-a, row:12.
	Domain string `koanf:"domain"`
	// CapWatts is what the domain may draw. For a dual-fed rack it is what
	// one feed carries alone, derated as the facility derates it (usually
	// 80%), since that is what the rack has to survive.
	CapWatts float64 `koanf:"cap_watts"`
	// OtherWatts is drawn in the domain by what llmgw does not place on:
	// switches, storage, machines outside the cluster.
	OtherWatts float64 `koanf:"other_watts"`
}

// Policy is the configuration as placement reads it.
func (f Facility) Policy() (facility.Policy, error) {
	p := facility.Policy{
		Margin: f.Margin, GPUWatts: f.GPUWatts, CPUCoreWatts: f.CPUCoreWatts,
		NodeWatts: f.NodeWatts, MeasuredFresh: f.MeasuredFresh,
	}
	if f.Margin < 0 || f.Margin >= 1 {
		return p, fmt.Errorf("facility.margin %v is a share of the cap, from 0 to below 1: 0.1 keeps ten percent free", f.Margin)
	}
	if f.GPUWatts < 0 || f.CPUCoreWatts < 0 || f.NodeWatts < 0 {
		return p, fmt.Errorf("facility: gpu_watts, cpu_core_watts and node_watts cannot be negative")
	}
	for i, b := range f.Budgets {
		d, err := facility.ParseDomain(b.Domain)
		if err != nil {
			return p, fmt.Errorf("facility.budgets[%d]: %w", i, err)
		}
		if b.CapWatts <= 0 {
			return p, fmt.Errorf("facility.budgets[%d] (%s): cap_watts must be above zero", i, d)
		}
		if b.OtherWatts < 0 {
			return p, fmt.Errorf("facility.budgets[%d] (%s): other_watts cannot be negative", i, d)
		}
		if p.Budgets == nil {
			p.Budgets = map[facility.Domain]facility.Budget{}
		}
		if _, dup := p.Budgets[d]; dup {
			return p, fmt.Errorf("facility.budgets[%d]: %s is given twice", i, d)
		}
		p.Budgets[d] = facility.Budget{CapWatts: b.CapWatts, OtherWatts: b.OtherWatts}
	}
	return p, nil
}

// Federation is this cluster's side of federation with other llmgw clusters.
type Federation struct {
	// Name is this cluster as a peer reads it, in the federated_from tag
	// sent with a request to a peer set to receive tags (`ctl federation add
	// --send-tags`). Empty is this machine's host name.
	Name string `koanf:"name"`
	// Interval is how often each peer is asked whether it is ready, timed,
	// and what it offers. A peer that does not answer is taken out of
	// routing until it does. Zero is 10s.
	Interval time.Duration `koanf:"interval"`
}

// Monitoring points the dashboard at a Prometheus holding this cluster's
// history.
//
// Read back rather than kept twice. llmgw publishes these metrics already; what
// a gauge cannot do is remember, and a ring buffer in this process would be a
// rival copy with its own retention, lost on every restart and disagreeing with
// the one an operator already has on screen.
type Monitoring struct {
	// Prometheus is the server's address, for example
	// http://monitoring.internal:9090. Empty leaves the charts off the
	// dashboard rather than showing an error where one should be.
	Prometheus string `koanf:"prometheus"`
	// Window is how far back the charts look. Zero takes an hour.
	Window time.Duration `koanf:"window"`
	// Timeout bounds one query. It runs while somebody waits for a page, so a
	// monitoring system that is slow today is skipped rather than waited for.
	Timeout time.Duration `koanf:"timeout"`
}

// Mirror is a model mirror this control plane administers over its API.
//
// The mirror is a separate machine with a large disk, not part of the gateway,
// so what is configured here is an address and how to authenticate to it —
// never a directory. A control plane that could write to the mirror's disk
// directly would be a way to replace a model with another one that every node
// would then fetch without complaint.
type Mirror struct {
	// API is the mirror's management address, scheme included, for example
	// https://models.internal:8082. Empty means no mirror is configured.
	//
	// The management port, not the one nodes fetch bytes from. Those are two
	// listeners on the mirror precisely so this one can be locked down without
	// re-certifying every agent's fetcher.
	API string `koanf:"api"`
	// TLS authenticates to the mirror's management API, with the same CA the
	// agents use. Empty leaves the connection in plain HTTP, which is only
	// acceptable on a network where nothing untrusted can reach that port —
	// the mirror says as much in its own log when it starts that way.
	TLS tlsconf.Config `koanf:"tls"`
	// Registry is the model registry the mirror page browses, empty for the
	// public Hugging Face. Set it to an internal mirror of the registry where
	// the control plane cannot reach the internet — or leave the whole mirror
	// block out, which turns the explorer off rather than giving it a search
	// box that can only time out.
	Registry string `koanf:"registry"`
	// Timeout bounds a management call. It does not bound a download: the
	// mirror answers a fetch as soon as it has started one, and the bytes keep
	// arriving long after this deadline has passed.
	Timeout time.Duration `koanf:"timeout"`
	// UpdateCheck is how often the registry is asked whether the files of
	// the models served from it have changed there: a repository can publish
	// new weights under the same name. Only told, never applied — moving a
	// model to them is `llmgw ctl model update`. 0 never asks.
	UpdateCheck time.Duration `koanf:"update_check"`
}

// Landing configures the public page at "/".
//
// It exists for one thing the gateway cannot know: who to ask for a key. A page
// that explains the API and then leaves the reader with no route to using it is
// where onboarding stops.
type Landing struct {
	// Contact is where someone without a key should go — an address, a URL, a
	// channel name. Rendered as a link when it is one, as text otherwise.
	Contact string `koanf:"contact"`
	// Notice is an optional line of your own, for whatever the reader needs to
	// know that this page cannot guess.
	Notice string `koanf:"notice"`
	// Language is what the web portal speaks to someone who has not chosen
	// a language with the switcher at the foot of every page and whose
	// browser asks for none it speaks: "en" or "tr". Empty is English.
	Language string `koanf:"language"`
	// DocsSignIn serves the docs portal at /docs only to someone signed in.
	// Off, the portal is public like this page: its client guide is for the
	// person who has no account yet, and the operator documents describe
	// llmgw in general — no address, model or key of this installation —
	// which the project's repository publishes anyway. On, a visitor is sent
	// to sign in, and without accounts the portal needs an API key.
	DocsSignIn bool `koanf:"docs_sign_in"`
}

// Threads configures conversation and memory storage for a gateway without
// authentication.
//
// Everything stored is in one pool with no owner, readable by anyone who can
// reach the client port and knows an id. That is acceptable on one machine and
// wrong on a shared network, which is why it takes a setting rather than being
// the default.
type Threads struct {
	// Store is the SQLite file. Empty means nothing is kept.
	Store string `koanf:"store"`
	// RetainHours is how long idle conversations and notes are kept. Required
	// with Store: a store with no expiry keeps everything anyone typed, by
	// omission.
	RetainHours int `koanf:"retain_hours"`
}

// Auth configures client authentication (plan §7.9).
//
// Off by default, because a gateway on a trusted network with no keys issued
// still has to serve. Turning it on is what makes the client-facing port safe
// to expose.
type Auth struct {
	Enabled bool `koanf:"enabled"`
	// Store is the SQLite file holding keys, tenancy and usage.
	Store string `koanf:"store"`
	// CacheTTL bounds how long a revoked key keeps working, traded against a
	// database read per key per TTL.
	CacheTTL time.Duration `koanf:"cache_ttl"`
	// NegativeTTL is how long a rejected key is remembered as rejected, so a
	// client looping with a bad key cannot become a database read per request.
	NegativeTTL time.Duration `koanf:"negative_ttl"`

	// DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to.
	//
	// Unlimited used to be the default, and a project created without limits is
	// the easy path — the "default" project on this cluster still is one. The
	// first sign of that is the bill or the outage. Zero keeps the old
	// behaviour for anyone who wants it, but it should be a choice rather than
	// an omission. A fallback, not a ceiling: an explicit limit at any scope
	// wins, above or below.
	DefaultRPM int `koanf:"default_rpm"`
	DefaultTPM int `koanf:"default_tpm"`

	// SignupOpen serves the signup form. Accounts it creates are pending until
	// an operator approves them, so this is not a way in — but turning it off
	// means accounts are made with `llmgw users create` instead.
	SignupOpen bool `koanf:"signup_open"`

	// OIDC is sign-in through an identity provider.
	OIDC OIDC `koanf:"oidc"`
	// SCIM is account provisioning by the identity provider.
	SCIM SCIM `koanf:"scim"`

	// MaxKeyTTL is how long any API key may live; a key asked for with no
	// expiry is given this. Organizations, teams and projects can set
	// shorter ones (llmgw keys ttl). Zero is no limit.
	MaxKeyTTL time.Duration `koanf:"max_key_ttl"`

	// Personal is what a person may do on their own, outside any project: make
	// personal keys and use the chat page personally, within these limits.
	Personal Personal `koanf:"personal"`

	// Pricing is what each model costs, for budgets. A model absent from the
	// table costs whatever DefaultPrice says, which is nothing unless set.
	Pricing      map[string]Price `koanf:"pricing"`
	DefaultPrice Price            `koanf:"default_price"`

	// FlushInterval and BatchSize decide when usage records are written.
	FlushInterval time.Duration `koanf:"flush_interval"`
	BatchSize     int           `koanf:"batch_size"`
	// BufferSize bounds the queue. Beyond it records are dropped rather than
	// blocking a request: losing accounting is bad, turning a stalled database
	// into an outage is worse.
	BufferSize int `koanf:"buffer_size"`

	// UsageRetention is how long the record of each request is kept: who sent
	// it, which replica answered, how long it queued — what `ctl request`
	// reads. Zero keeps it for ever. Totals, charts and budgets are read from
	// five-minute sums kept beside the records, which are not pruned, so this
	// bounds the table that grows by a row a request without changing any
	// figure a page shows. At least a day when set.
	UsageRetention time.Duration `koanf:"usage_retention"`
}

// Personal configures use that belongs to a person rather than a project.
//
// On by default: a person need not belong to anything to use the gateway. The
// limits are each person's own pool across all their personal keys and their
// personal chat, and apply only where nobody wrote a limit for that person.
type Personal struct {
	// Disabled refuses personal keys and personal chat; people then use the
	// gateway only through projects they are members of.
	Disabled bool `koanf:"disabled"`
	RPM      int  `koanf:"rpm"`
	TPM      int  `koanf:"tpm"`
	// Budget is in whole units per Period ("day" or "month").
	Budget float64 `koanf:"budget"`
	Period string  `koanf:"period"`
}

// Price is a model's cost in micro-units per thousand tokens.
//
// What a unit means is the operator's choice — self-hosted models have no
// per-token price, so it may be GPU-seconds, a notional rate, or 1000
// everywhere, which turns a budget into a token quota (plan §7.9.4).
type Price struct {
	PromptPer1k     int64 `koanf:"prompt_per_1k"`
	CompletionPer1k int64 `koanf:"completion_per_1k"`
}

// Control configures the control plane.
type Control struct {
	// Interval is how often desired and actual state are compared.
	Interval time.Duration `koanf:"interval"`
	// BaseBackoff and MaxBackoff bound how fast a failing model is retried.
	BaseBackoff time.Duration `koanf:"base_backoff"`
	MaxBackoff  time.Duration `koanf:"max_backoff"`
	// UnreachableAfter and LostAfter stage node expiry. They are deliberately
	// far apart: a network blip must not tear down replicas that still serve,
	// and rebuilding a multi-node replica costs tens of minutes of loading.
	UnreachableAfter time.Duration `koanf:"unreachable_after"`
	LostAfter        time.Duration `koanf:"lost_after"`
	// PortRange is the per-node range for engine HTTP and dist-init ports.
	PortRange []int `koanf:"port_range"`
	// MgmtInterface is the ethernet interface NCCL bootstraps over.
	MgmtInterface string `koanf:"mgmt_interface"`
	// Timezone is where models' scaling schedules are read, an IANA name
	// like Europe/Istanbul; empty is the control plane's own.
	Timezone string `koanf:"timezone"`
	// EvalDir is where quality suites are read from by name, for `llmgw ctl
	// eval MODEL --suite NAME`, a model's eval gate and the console. The
	// suites the binary carries (arithmetic, json-format) are found when it
	// has none of the name; a suite here of the same name replaces one.
	EvalDir string `koanf:"eval_dir"`

	// Enabled runs the control plane even when configuration declares no
	// models.
	//
	// Without it, a deployment whose catalog lives entirely in the database
	// cannot start: control plane mode is inferred from the config file, so an
	// empty models block means "router only", the process refuses to start for
	// want of anything to serve, and the models it would have loaded are never
	// read. That is a cluster that cannot be managed at runtime because it is
	// managed at runtime. AgentListen cannot carry this signal — it has a
	// default, so every deployment would become a control plane.
	Enabled bool `koanf:"enabled"`

	// LeaseTTL is how long this control plane's claim to be the one placing
	// work stands without being renewed.
	//
	// Only meaningful with a shared store, where a second control plane can
	// see the claim. Long enough to survive a slow query or a brief network
	// stall, short enough that a standby takes over before anyone has finished
	// reading the alert. Zero takes the default.
	LeaseTTL time.Duration `koanf:"lease_ttl"`
}

// Router holds routing behaviour.
type Router struct {
	// ActivationWait is how long a request waits for an on-demand model to
	// load before the client is told to come back. Zero takes the default.
	//
	// The one real decision in on-demand loading. Refusing outright makes the
	// feature useless, since the first request for every model would fail;
	// waiting without a bound turns a request into a job, and a client that
	// budgeted thirty seconds ends up holding a connection for ten minutes.
	ActivationWait time.Duration `koanf:"activation_wait"`

	ScrapeInterval time.Duration `koanf:"scrape_interval"`
	ScrapeTimeout  time.Duration `koanf:"scrape_timeout"`
	// StaleAfter is how long an endpoint may go unscraped before being scored
	// down (plan §7.6).
	StaleAfter time.Duration `koanf:"stale_after"`

	// Retries is how many *additional* endpoints may be tried after a failure,
	// and only while nothing has been written to the client (plan §7.7).
	Retries int `koanf:"retries"`
	// Cooldown is how long a failing endpoint leaves the candidate pool.
	Cooldown time.Duration `koanf:"cooldown"`
	// FailureThreshold is the consecutive failure count that opens a breaker.
	// 1 turns the breaker into a pure cooldown.
	FailureThreshold int `koanf:"failure_threshold"`

	// DefaultProfile names the routing profile used when a model picks none.
	DefaultProfile string `koanf:"default_profile"`
	// Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the
	// router uses session affinity alone, which is tier 0 and already most of
	// the win for multi-turn traffic.
	Prefix Prefix `koanf:"prefix"`
	// KVIndex tunes the precise KV index (plan §7.5 tier 2). Which models
	// use it is the catalog's to say, by kv_events on a vLLM profile; this
	// is only how.
	KVIndex KVIndex `koanf:"kv_index"`

	// SessionHeader carries the session id for affinity (plan §7.5 tier 0).
	SessionHeader string `koanf:"session_header"`
	// InjectUsage asks engines for usage even when the client did not, so token
	// accounting works regardless of client behaviour.
	InjectUsage bool `koanf:"inject_usage"`

	// TenantLabel is the level of the tree the per-tenant request series are
	// labelled by — llmgw_tenant_requests_total and its siblings: "org",
	// "team" or "project", or "none" for no such series. A label per project
	// on a cluster of thousands is more series than Prometheus wants, and a
	// team is what an alert is usually about; "team" is the default.
	TenantLabel string `koanf:"tenant_label"`

	// TagLabels are the request tags (metadata, x-llmgw-tags) counted in
	// llmgw_tagged_requests_total and llmgw_tagged_tokens_total, by value.
	// Only these: a tag is the client's to set, and every value of one is
	// a series. The first 100 values of each are counted apart, the rest as
	// "other". Empty counts none; the usage records keep every tag anyway.
	TagLabels []string `koanf:"tag_labels"`

	// MaxBodyBytes bounds the request body the router will read.
	MaxBodyBytes int64 `koanf:"max_body_bytes"`

	// Batch is the Files and Batch APIs (/v1/files, /v1/batches): a file of
	// requests served at the lowest priority, its answers written to
	// another. Needs the store.
	Batch Batch `koanf:"batch"`

	// Cache bounds the answers kept for the projects that reuse them
	// (llmgw keys cache). Which projects do, and for how long, is theirs
	// to set; this is only how much memory it may take.
	Cache Cache `koanf:"cache"`

	// TrustedProxies are the reverse proxies in front of the client port,
	// as addresses or CIDR prefixes. Their X-Forwarded-For, -Proto and -Host
	// are believed; anyone else's are not, since they decide the address
	// written into the setup scripts and whose attempts are counted where.
	TrustedProxies []string `koanf:"trusted_proxies"`

	Timeouts  Timeouts  `koanf:"timeouts"`
	Scoring   Scoring   `koanf:"scoring"`
	Admission Admission `koanf:"admission"`
}

// Admission configures the flow control layer (plan §7.2).
type Admission struct {
	// Enabled turns the layer on. Off, every request goes straight through,
	// which is the v0.2.0 behaviour.
	Enabled bool `koanf:"enabled"`

	Bands       []Band `koanf:"bands"`
	DefaultBand string `koanf:"default_band"`
	// BandHeader lets a client declare its band. Until authentication lands in
	// v1.0.0 this is trusted, so the port must not be exposed to callers who
	// should not be able to promote themselves.
	BandHeader string `koanf:"band_header"`

	QueueTTL             time.Duration `koanf:"queue_ttl"`
	QueueCapacityPerBand int           `koanf:"queue_capacity_per_band"`
	// QueueCapacityTotal bounds everything queued across bands. Reached, a
	// request is not refused while a lower band has any queued: the newest of
	// the lowest band is turned away instead, with a 503 and Retry-After.
	// Zero takes half of all the bands' capacities together.
	QueueCapacityTotal int           `koanf:"queue_capacity_total"`
	Interval           time.Duration `koanf:"interval"`
}

// Prefix configures approximate prefix routing.
type Prefix struct {
	Enabled bool `koanf:"enabled"`
	// MaxNodes bounds the tree, MinMatch is the shortest match worth acting on
	// in bytes, and TTL is how long an entry is believed — an engine evicts on
	// its own schedule, so an old record may describe a cache that is gone.
	MaxNodes int           `koanf:"max_nodes"`
	MinMatch int           `koanf:"min_match"`
	TTL      time.Duration `koanf:"ttl"`

	// MatchRatio is the share of the prompt that must match to be worth acting
	// on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip
	// before affinity is abandoned.
	MatchRatio float64 `koanf:"match_ratio"`
	BalanceAbs float64 `koanf:"balance_abs"`
	BalanceRel float64 `koanf:"balance_rel"`
}

// KVIndex tunes routing by the blocks each vLLM replica reports holding.
type KVIndex struct {
	// TokenizeTimeout bounds the call that turns a request into token ids,
	// made to one of the model's replicas before the request is placed. Past
	// it the request is routed as tier 1 would. Zero takes 250ms.
	TokenizeTimeout time.Duration `koanf:"tokenize_timeout"`
	// MinBlocks is the fewest leading blocks a replica must hold to be
	// preferred for them; fewer is coincidence — a chat template's opening
	// tokens — rather than a prefix worth following. Zero takes 4, which is
	// 64 tokens at vLLM's usual block size of 16.
	MinBlocks int `koanf:"min_blocks"`
	// MaxBlocks bounds how many of a prompt's blocks are hashed and looked
	// up. Beyond it a longer match still helps, but the difference no longer
	// decides anything. Zero takes 4096.
	MaxBlocks int `koanf:"max_blocks"`
}

// Band is one priority class.
type Band struct {
	Name              string  `koanf:"name"`
	Priority          int     `koanf:"priority"`
	SaturationCeiling float64 `koanf:"saturation_ceiling"`
}

// Timeouts bounds each phase of an upstream request.
type Timeouts struct {
	Dial time.Duration `koanf:"dial"`
	// ResponseHeader bounds the wait for headers on a streaming request. It is
	// a liveness check there, because headers arrive before generation starts.
	ResponseHeader time.Duration `koanf:"response_header"`
	// Generation bounds a non-streaming request, where the engine sends no
	// headers until the completion is done. Setting response_header alone used
	// to cap both, so 60s read as a liveness guard and behaved as a limit of
	// about a hundred tokens on a slow model.
	Generation time.Duration `koanf:"generation"`
	FirstToken time.Duration `koanf:"first_token"`
	Idle       time.Duration `koanf:"idle"`
	Total      time.Duration `koanf:"total"`
}

// Scoring holds the thresholds the scorers normalize against. These want to be
// measured with ctl bench (plan §10), not guessed.
type Scoring struct {
	TargetWaiting     float64 `koanf:"target_waiting"`
	TargetInflight    float64 `koanf:"target_inflight"`
	KVCeiling         float64 `koanf:"kv_ceiling"`
	SessionLoadFactor float64 `koanf:"session_load_factor"`
}

// Endpoint is one engine replica. Static in phase 1.
type Endpoint struct {
	ID  string `koanf:"id"`
	URL string `koanf:"url"`
	// APIKey is sent to the engine as a bearer token, for an engine started
	// with one.
	APIKey string   `koanf:"api_key"`
	Models []string `koanf:"models"`
	// Engine names the inference engine, which decides what its metrics are
	// called. Empty means vLLM's names.
	Engine   string  `koanf:"engine"`
	Pool     string  `koanf:"pool"`
	Weight   float64 `koanf:"weight"`
	Priority int     `koanf:"priority"`
}

// Secrets is where secrets are read from besides files, and how the
// store's own are sealed.
//
// Wherever a setting names a file holding a secret — the store's @address,
// api_key_file, client_secret_file, token_file, secret_file — it may name
// env:VARIABLE or vault:mount/path#field instead.
type Secrets struct {
	// Vault is how vault: references and vault-transit: are reached.
	Vault secrets.Vault `koanf:"vault"`
	// MasterKey seals the secrets the store keeps for itself — what every
	// engine's key is derived from — so a copy of the store, or of a backup,
	// does not hold them: vault-transit:KEY seals with Vault's transit engine,
	// which never lets the key out; otherwise a reference to 32 random bytes
	// (openssl rand -hex 32), which then must be kept as safely as the store.
	// Values from before it was set are sealed the first time they are read.
	MasterKey string `koanf:"master_key"`
}

// Batch is how the Files and Batch APIs are bounded.
type Batch struct {
	// MaxFileBytes bounds one uploaded file; 100 MB unless set.
	MaxFileBytes int64 `koanf:"max_file_bytes"`
	// MaxLines bounds the requests in one batch; 50 000 unless set.
	MaxLines int `koanf:"max_lines"`
	// Concurrency is how many of one batch's requests are in flight at
	// once; 4 unless set. They queue in the lowest admission band, behind
	// every interactive request.
	Concurrency int `koanf:"concurrency"`
	// RetainDays is how long a batch's answer and error files are kept;
	// 30 unless set. Uploaded files are kept until deleted.
	RetainDays int `koanf:"retain_days"`
}

// Cache is how much of this process's memory reused answers may take. The
// answers are held here and nowhere else: each gateway process keeps its
// own, and a restart begins them again.
type Cache struct {
	// MaxBytes bounds every project's answers together; 128 MB unless set.
	// Past it, the answers used longest ago go first.
	MaxBytes int64 `koanf:"max_bytes"`
	// MaxEntryBytes bounds one answer; a longer one is not kept. 1 MB
	// unless set.
	MaxEntryBytes int64 `koanf:"max_entry_bytes"`
}

// Default returns the configuration used when nothing is specified.
func Default() Server {
	return Server{
		Listen:         ":8080",
		AdminListen:    "127.0.0.1:9090",
		AgentListen:    ":7443",
		MaxConnections: 10_000,
		Mirror:         Mirror{UpdateCheck: 6 * time.Hour},
		Firewall:       FirewallSettings{Enabled: true},
		Control: Control{
			Interval:         5 * time.Second,
			BaseBackoff:      10 * time.Second,
			MaxBackoff:       10 * time.Minute,
			UnreachableAfter: 15 * time.Second,
			LostAfter:        60 * time.Second,
			PortRange:        []int{20000, 20999},
			MgmtInterface:    "eth0",
			EvalDir:          "/etc/llmgw/eval",
		},
		Router: Router{
			ScrapeInterval:   2 * time.Second,
			ScrapeTimeout:    time.Second,
			StaleAfter:       10 * time.Second,
			Retries:          2,
			Cooldown:         15 * time.Second,
			FailureThreshold: 3,
			DefaultProfile:   "chat",
			TenantLabel:      "team",
			SessionHeader:    "x-session-id",
			InjectUsage:      true,
			MaxBodyBytes:     32 << 20,
			Timeouts: Timeouts{
				Dial:           5 * time.Second,
				ResponseHeader: 60 * time.Second,
				FirstToken:     120 * time.Second,
				Idle:           60 * time.Second,
				Total:          30 * time.Minute,
			},
			Scoring: Scoring{
				TargetWaiting:     4,
				TargetInflight:    8,
				KVCeiling:         0.9,
				SessionLoadFactor: 1.25,
			},
			Admission: Admission{
				Enabled:     true,
				BandHeader:  "x-llmgw-band",
				DefaultBand: "gold",
				Bands: []Band{
					{Name: "platinum", Priority: 400, SaturationCeiling: 1.00},
					{Name: "gold", Priority: 300, SaturationCeiling: 0.85},
					{Name: "silver", Priority: 200, SaturationCeiling: 0.60},
					{Name: "bronze", Priority: 100, SaturationCeiling: 0.30},
				},
				QueueTTL:             30 * time.Second,
				QueueCapacityPerBand: 2000,
				Interval:             10 * time.Millisecond,
			},
		},
	}
}

// Load reads configuration from path, overlaying it on the defaults, then
// applies LLMGW_-prefixed environment overrides.
//
// An empty path loads the defaults, which is what makes `llmgw server` with a
// couple of --endpoint flags usable for a smoke test.
func Load(path string) (Server, error) {
	var b []byte
	if path != "" {
		if _, err := os.Stat(path); err != nil {
			return Default(), fmt.Errorf("config file: %w", err)
		}
		var err error
		if b, err = os.ReadFile(path); err != nil { //nolint:gosec // the configuration
			return Default(), fmt.Errorf("config file: %w", err)
		}
	}
	cfg, err := load(b)
	if err != nil && path != "" && strings.HasPrefix(err.Error(), "parsing settings") {
		return cfg, fmt.Errorf("parsing %s:%s", path, strings.TrimPrefix(err.Error(), "parsing settings:"))
	}
	return cfg, err
}

// load reads a YAML document of settings over the defaults.
func load(b []byte) (Server, error) {
	cfg := Default()
	k := koanf.New(".")
	if len(b) > 0 {
		if err := k.Load(bytesProvider(b), yaml.Parser()); err != nil {
			return cfg, fmt.Errorf("parsing settings: %w", err)
		}
	}

	// LLMGW_ROUTER__RETRIES=3 maps onto router.retries.
	err := k.Load(env.Provider(".", env.Opt{
		Prefix: "LLMGW_",
		TransformFunc: func(key, value string) (string, any) {
			key = strings.ToLower(strings.TrimPrefix(key, "LLMGW_"))
			return strings.ReplaceAll(key, "__", "."), value
		},
	}), nil)
	if err != nil {
		return cfg, fmt.Errorf("environment overrides: %w", err)
	}

	if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
		return cfg, fmt.Errorf("decoding configuration: %w", err)
	}
	// So every secret reference in it, from here on, can be read.
	if err := secrets.Configure(cfg.Secrets.Vault); err != nil {
		return cfg, err
	}
	return cfg, cfg.Validate()
}

// Validate rejects configurations that would otherwise fail confusingly at
// request time.
func (s Server) Validate() error {
	for i, o := range s.EngineOverrides {
		if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 {
			return fmt.Errorf("engine_overrides[%d] picks no nodes: give it a gpu_class, nodes or labels", i)
		}
		if len(o.Engines) == 0 {
			return fmt.Errorf("engine_overrides[%d] changes no engine: give it engines, with \"\" for none", i)
		}
	}
	if s.Listen == "" {
		return fmt.Errorf("listen is empty")
	}
	if _, err := s.Facility.Policy(); err != nil {
		return err
	}
	if s.Control.Timezone != "" {
		if _, err := time.LoadLocation(s.Control.Timezone); err != nil {
			return fmt.Errorf("control.timezone %q: %w", s.Control.Timezone, err)
		}
	}
	if _, err := ParseBackupRecipients(s.BackupRecipients); err != nil {
		return err
	}
	for i, t := range s.BackupCopyTo {
		switch {
		case strings.HasPrefix(t, "ssh://"):
			if s.BackupCopyKey == "" {
				return fmt.Errorf("backup_copy_to[%d] is over ssh: give backup_copy_key", i)
			}
		case !strings.HasPrefix(t, "/"):
			return fmt.Errorf("backup_copy_to[%d] %q: an absolute directory or ssh://user@host/path", i, t)
		}
	}
	if len(s.BackupCopyTo) > 0 && len(s.BackupRecipients) == 0 {
		return fmt.Errorf("backup_copy_to needs backup_recipients: an archive holds the certificate authority's key " +
			"and every secret in the store, and leaves this machine only encrypted")
	}
	if _, err := s.FirewallAllow(); err != nil {
		return err
	}
	if err := s.Notify.Validate(); err != nil {
		return err
	}
	if err := s.Guard.Validate(); err != nil {
		return err
	}
	if err := s.Auth.OIDC.Validate(); err != nil {
		return err
	}
	if err := s.AgentTLS.Validate(); err != nil {
		return err
	}
	if err := s.Offline.Validate(); err != nil {
		return err
	}

	if s.Auth.Enabled && s.Auth.Store == "" {
		return fmt.Errorf("auth.store is required when auth is enabled")
	}
	switch s.Router.TenantLabel {
	case "", "none", "org", "team", "project":
	default:
		return fmt.Errorf("router.tenant_label is %q: org, team, project or none", s.Router.TenantLabel)
	}
	if s.Auth.UsageRetention != 0 && s.Auth.UsageRetention < 24*time.Hour {
		return fmt.Errorf("auth.usage_retention is %s: at least 24h, or 0 to keep every request", s.Auth.UsageRetention)
	}
	if s.Threads.Store != "" && s.Threads.RetainHours <= 0 {
		return fmt.Errorf("threads.retain_hours must be positive when threads.store is set")
	}

	seen := make(map[string]struct{}, len(s.Endpoints))
	for i, e := range s.Endpoints {
		switch {
		case e.ID == "":
			return fmt.Errorf("endpoints[%d]: id is required", i)
		case e.URL == "":
			return fmt.Errorf("endpoint %q: url is required", e.ID)
		case !strings.HasPrefix(e.URL, "http://") && !strings.HasPrefix(e.URL, "https://"):
			return fmt.Errorf("endpoint %q: url must start with http:// or https://", e.ID)
		case len(e.Models) == 0:
			return fmt.Errorf("endpoint %q: at least one model is required", e.ID)
		}
		if _, dup := seen[e.ID]; dup {
			return fmt.Errorf("endpoint id %q is used twice", e.ID)
		}
		seen[e.ID] = struct{}{}
	}

	// The catalog enforces the rest — aliases, fallbacks, parallelism
	// arithmetic — so there is one place those rules live.
	if _, err := s.Catalog(); err != nil {
		return err
	}
	return nil
}

// Catalog builds the model catalog from the configuration.
func (s Server) Catalog() (*catalog.Catalog, error) {
	return catalog.New(s.Models)
}

// ControlPlaneMode reports whether the server should run a control plane.
//
// A model with no profiles cannot be placed, so a configuration of bare names
// plus a static endpoint list is the router-only mode used for testing against
// sim-engines.
//
// With no static endpoints there is nothing a router-only process could serve,
// so a configuration naming neither endpoints nor models is a control plane
// with an empty catalog — which is what every new cluster is. It used to be
// refused as having nothing to serve, and under systemd that is a unit
// restarting every two seconds with nothing on 8080.
func (s Server) ControlPlaneMode() bool {
	if s.Control.Enabled || len(s.Endpoints) == 0 {
		return true
	}
	for _, m := range s.Models {
		if len(m.Profiles) > 0 {
			return true
		}
	}
	return false
}

// DefaultEngines are the engine versions this release was checked against:
// every flag llmgw builds for them is one they take.
//
// vLLM is installed on every node with an NVIDIA GPU it can run on and
// skipped elsewhere. SGLang is not here: it is opt-in, by naming a version
// under engines.
var DefaultEngines = map[string]string{
	"llamacpp": "b10991",
	"vllm":     "0.30.0",
}

// EngineOverride gives the nodes it picks different engines. Every selector
// set must match; at least one must be set.
type EngineOverride struct {
	// GPUClass picks nodes with a GPU of this class ("v100-32g").
	GPUClass string `koanf:"gpu_class"`
	// Nodes picks nodes by id.
	Nodes []string `koanf:"nodes"`
	// Labels picks nodes carrying all of these.
	Labels map[string]string `koanf:"labels"`
	// Engines are laid over the cluster's: a version, or "" for none.
	Engines map[string]string `koanf:"engines"`
}

// Picks reports whether the override applies to a node with this id, these
// GPU classes and these labels.
func (o EngineOverride) Picks(node string, classes []string, labels map[string]string) bool {
	if o.GPUClass == "" && len(o.Nodes) == 0 && len(o.Labels) == 0 {
		return false
	}
	if o.GPUClass != "" && !slices.Contains(classes, o.GPUClass) {
		return false
	}
	if len(o.Nodes) > 0 && !slices.Contains(o.Nodes, node) {
		return false
	}
	for k, v := range o.Labels {
		if labels[k] != v {
			return false
		}
	}
	return true
}

// EnginesFor is what one node installs: WantedEngines with every override
// that picks it laid over, in order.
func (s Server) EnginesFor(node string, classes []string, labels map[string]string) map[string]string {
	out := s.WantedEngines()
	for _, o := range s.EngineOverrides {
		if !o.Picks(node, classes, labels) {
			continue
		}
		for k, v := range o.Engines {
			if v == "" {
				delete(out, k)
				continue
			}
			out[k] = v
		}
	}
	return out
}

// WantedEngines is DefaultEngines with the configuration laid over it.
func (s Server) WantedEngines() map[string]string {
	out := map[string]string{}
	for k, v := range DefaultEngines {
		out[k] = v
	}
	for k, v := range s.Engines {
		if v == "" {
			delete(out, k)
			continue
		}
		out[k] = v
	}
	return out
}

// Offline is a cluster with no way to the internet.
//
// Everything it would otherwise fetch from outside — llmgw's releases, the
// engine builds its nodes install, model files — is carried in as a bundle,
// made where there is internet (llmgw bundle create) and imported into the
// mirror here (llmgw bundle import). Nodes then fetch from the mirror, and
// check what they fetch as strictly as they would online: releases against
// the project's signing key, engines against the checksums the control plane
// holds from the bundle.
type Offline struct {
	// Mirror is the mirror's file address, as nodes reach it:
	// http://10.0.0.5:8081 — the port model files are served on, not the
	// management API. Set, the cluster is offline: nodes install engines
	// from it and never from their publishers, releases come from it unless
	// releases names another place, and no registry is asked whether a
	// model has newer weights. Python engines (vLLM, SGLang) come from PyPI,
	// which no bundle carries; on an offline node they are installed by hand.
	Mirror string `koanf:"mirror"`
	// Checksums is the file of engine checksums this control plane trusts,
	// kept by llmgw bundle import and llmgw bundle trust; every engine file
	// a node fetches from the mirror is checked against it. Read again
	// whenever it changes. Empty is /etc/llmgw/offline.sha256.
	Checksums string `koanf:"checksums"`
}

// DefaultOfflineChecksums is where the engine checksums are kept when
// offline.checksums does not say.
const DefaultOfflineChecksums = "/etc/llmgw/offline.sha256"

// Enabled reports whether the cluster is offline.
func (o Offline) Enabled() bool { return o.Mirror != "" }

// ChecksumsFile is the engine checksums' file.
func (o Offline) ChecksumsFile() string {
	if o.Checksums != "" {
		return o.Checksums
	}
	return DefaultOfflineChecksums
}

// ReleasesBase is where an offline cluster's releases are on its mirror.
func (o Offline) ReleasesBase() string {
	return strings.TrimSuffix(o.Mirror, "/") + "/_llmgw/releases"
}

// Validate refuses a mirror address nodes could not fetch from.
func (o Offline) Validate() error {
	if o.Mirror == "" {
		return nil
	}
	if !strings.HasPrefix(o.Mirror, "http://") && !strings.HasPrefix(o.Mirror, "https://") {
		return fmt.Errorf("offline.mirror %q: give the mirror's file address, http://host:8081", o.Mirror)
	}
	if strings.HasSuffix(strings.TrimSuffix(o.Mirror, "/"), ":8082") {
		return fmt.Errorf("offline.mirror %q is the mirror's management port: give the one nodes fetch files from, :8081", o.Mirror)
	}
	return nil
}

// ReleasesBase is where release binaries are fetched from: releases when
// set, the mirror's copy on an offline cluster, and empty — the caller's
// default, the project's own releases — otherwise.
func (s Server) ReleasesBase() string {
	if s.Releases != "" {
		return s.Releases
	}
	if s.Offline.Enabled() {
		return s.Offline.ReleasesBase()
	}
	return ""
}