diff --git a/README.md b/README.md index f23e93e2..26c1e9ba 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,17 @@ without a fabric. The InfiniBand, NVML and NCCL paths are written against the formats and failure modes those tools actually have, but they have not yet run on a machine that has them. +## Documentation + +- [docs/install.md](docs/install.md) — installing from nothing: control plane, + nodes, engines, weights, first request +- [docs/quickstart.md](docs/quickstart.md) — the short path +- [docs/reference.md](docs/reference.md) — every configuration key, generated + from the structs and checked by a test, so it cannot describe a setting that + does not exist or miss one that does +- [docs/node-onboarding.md](docs/node-onboarding.md), + [docs/onboarding.md](docs/onboarding.md) — adding machines, and adding people + ## Building ```sh diff --git a/docs/install.md b/docs/install.md index 84063330..8ed32b23 100644 --- a/docs/install.md +++ b/docs/install.md @@ -774,6 +774,9 @@ requests. On a slow model, set `timeouts.generation` higher. ## What to read next +- `docs/reference.md` — every configuration key, generated from the source, so + it describes the version you are running rather than the one somebody last + wrote about - `docs/node-onboarding.md` — adding nodes in more detail, and what onboarding deliberately does not do - `docs/onboarding.md` — giving people accounts and keys diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 00000000..7d3a9199 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,1163 @@ +# Configuration reference + +Every setting llmgw reads from a file, with the type it takes and what it is +for. + +**Generated from the source.** The descriptions are the comments on the fields +themselves, so this cannot describe a setting that does not exist or miss one +that does — and a test fails when it has drifted. Do not edit it; edit the +comment and run `go generate ./...`. + +Where a setting takes another block, the block is listed under its own heading +rather than repeated. + +## Every key + +| Key | Type | In | +|---|---|---| +| `enabled` | `bool` | [Admission](#admission) | +| `bands` | `[]Band` | [Admission](#admission) | +| `default_band` | `string` | [Admission](#admission) | +| `band_header` | `string` | [Admission](#admission) | +| `queue_ttl` | `time.Duration` | [Admission](#admission) | +| `queue_capacity_per_band` | `int` | [Admission](#admission) | +| `interval` | `time.Duration` | [Admission](#admission) | +| `enabled` | `bool` | [Auth](#auth) | +| `store` | `string` | [Auth](#auth) | +| `cache_ttl` | `time.Duration` | [Auth](#auth) | +| `negative_ttl` | `time.Duration` | [Auth](#auth) | +| `default_rpm` | `int` | [Auth](#auth) | +| `default_tpm` | `int` | [Auth](#auth) | +| `signup_open` | `bool` | [Auth](#auth) | +| `pricing` | `map[string]Price` | [Auth](#auth) | +| `default_price` | `Price` | [Auth](#auth) | +| `flush_interval` | `time.Duration` | [Auth](#auth) | +| `batch_size` | `int` | [Auth](#auth) | +| `buffer_size` | `int` | [Auth](#auth) | +| `name` | `string` | [Band](#band) | +| `priority` | `int` | [Band](#band) | +| `saturation_ceiling` | `float64` | [Band](#band) | +| `tools` | `bool` | [Capabilities](#capabilities) | +| `vision` | `bool` | [Capabilities](#capabilities) | +| `embeddings` | `bool` | [Capabilities](#capabilities) | +| `node_id` | `string` | [Config](#config) | +| `server` | `string` | [Config](#config) | +| `servers` | `[]string` | [Config](#config) | +| `hostname` | `string` | [Config](#config) | +| `mgmt_addr` | `string` | [Config](#config) | +| `labels` | `map[string]string` | [Config](#config) | +| `mgmt_interface` | `string` | [Config](#config) | +| `fabrics` | `[]inventory.FabricAttachment` | [Config](#config) | +| `heartbeat_interval` | `time.Duration` | [Config](#config) | +| `reconnect_base` | `time.Duration` | [Config](#config) | +| `reconnect_max` | `time.Duration` | [Config](#config) | +| `adopt_ready_timeout` | `time.Duration` | [Config](#config) | +| `tls` | `tlsconf.Config` | [Config](#config) | +| `model_dirs` | `[]string` | [Config](#config) | +| `weights_cache` | `string` | [Config](#config) | +| `discovery` | `discovery.Options` | [Config](#config) | +| `engines` | `engines.Config` | [Config](#config) | +| `supervisor` | `supervisor.Config` | [Config](#config) | +| `interval` | `time.Duration` | [Control](#control) | +| `base_backoff` | `time.Duration` | [Control](#control) | +| `max_backoff` | `time.Duration` | [Control](#control) | +| `unreachable_after` | `time.Duration` | [Control](#control) | +| `lost_after` | `time.Duration` | [Control](#control) | +| `port_range` | `[]int` | [Control](#control) | +| `mgmt_interface` | `string` | [Control](#control) | +| `lease_ttl` | `time.Duration` | [Control](#control) | +| `id` | `string` | [Endpoint](#endpoint) | +| `url` | `string` | [Endpoint](#endpoint) | +| `models` | `[]string` | [Endpoint](#endpoint) | +| `engine` | `string` | [Endpoint](#endpoint) | +| `pool` | `string` | [Endpoint](#endpoint) | +| `weight` | `float64` | [Endpoint](#endpoint) | +| `priority` | `int` | [Endpoint](#endpoint) | +| `contact` | `string` | [Landing](#landing) | +| `notice` | `string` | [Landing](#landing) | +| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) | +| `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) | +| `name` | `string` | [Model](#model) | +| `aliases` | `[]string` | [Model](#model) | +| `source` | `string` | [Model](#model) | +| `profiles` | `[]Profile` | [Model](#model) | +| `scaling` | `Scaling` | [Model](#model) | +| `fallbacks` | `[]string` | [Model](#model) | +| `routing_profile` | `string` | [Model](#model) | +| `slo` | `SLO` | [Model](#model) | +| `context_length` | `int` | [Model](#model) | +| `max_output` | `int` | [Model](#model) | +| `capabilities` | `Capabilities` | [Model](#model) | +| `nvidia_smi` | `string` | [Options](#options) | +| `ib_root` | `string` | [Options](#options) | +| `prefer_nvml` | `bool` | [Options](#options) | +| `skip_gpus` | `bool` | [Options](#options) | +| `enabled` | `bool` | [Prefix](#prefix) | +| `max_nodes` | `int` | [Prefix](#prefix) | +| `min_match` | `int` | [Prefix](#prefix) | +| `ttl` | `time.Duration` | [Prefix](#prefix) | +| `match_ratio` | `float64` | [Prefix](#prefix) | +| `balance_abs` | `float64` | [Prefix](#prefix) | +| `balance_rel` | `float64` | [Prefix](#prefix) | +| `prompt_per_1k` | `int64` | [Price](#price) | +| `completion_per_1k` | `int64` | [Price](#price) | +| `name` | `string` | [Profile](#profile) | +| `engine` | `string` | [Profile](#profile) | +| `gpu_class` | `string` | [Profile](#profile) | +| `tp` | `int` | [Profile](#profile) | +| `pp` | `int` | [Profile](#profile) | +| `gpus_per_node` | `int` | [Profile](#profile) | +| `nodes` | `int` | [Profile](#profile) | +| `args` | `[]string` | [Profile](#profile) | +| `env` | `map[string]string` | [Profile](#profile) | +| `min_memory_gb` | `int` | [Profile](#profile) | +| `min_disk_gb` | `int` | [Profile](#profile) | +| `node_selector` | `map[string]string` | [Profile](#profile) | +| `weight` | `float64` | [Profile](#profile) | +| `priority` | `int` | [Profile](#profile) | +| `startup_timeout` | `time.Duration` | [Profile](#profile) | +| `activation_wait` | `time.Duration` | [Router](#router) | +| `scrape_interval` | `time.Duration` | [Router](#router) | +| `scrape_timeout` | `time.Duration` | [Router](#router) | +| `stale_after` | `time.Duration` | [Router](#router) | +| `retries` | `int` | [Router](#router) | +| `cooldown` | `time.Duration` | [Router](#router) | +| `failure_threshold` | `int` | [Router](#router) | +| `default_profile` | `string` | [Router](#router) | +| `prefix` | `Prefix` | [Router](#router) | +| `session_header` | `string` | [Router](#router) | +| `inject_usage` | `bool` | [Router](#router) | +| `max_body_bytes` | `int64` | [Router](#router) | +| `timeouts` | `Timeouts` | [Router](#router) | +| `scoring` | `Scoring` | [Router](#router) | +| `admission` | `Admission` | [Router](#router) | +| `python` | `string` | [SGLangConfig](#sglangconfig) | +| `ttft_p99` | `time.Duration` | [SLO](#slo) | +| `tpot_p99` | `time.Duration` | [SLO](#slo) | +| `min` | `int` | [Scaling](#scaling) | +| `max` | `int` | [Scaling](#scaling) | +| `idle_timeout` | `time.Duration` | [Scaling](#scaling) | +| `target_waiting` | `float64` | [Scoring](#scoring) | +| `target_inflight` | `float64` | [Scoring](#scoring) | +| `kv_ceiling` | `float64` | [Scoring](#scoring) | +| `session_load_factor` | `float64` | [Scoring](#scoring) | +| `listen` | `string` | [Server](#server) | +| `admin_listen` | `string` | [Server](#server) | +| `metrics_listen` | `string` | [Server](#server) | +| `agent_listen` | `string` | [Server](#server) | +| `auth` | `Auth` | [Server](#server) | +| `threads` | `Threads` | [Server](#server) | +| `landing` | `Landing` | [Server](#server) | +| `agent_tls` | `tlsconf.Config` | [Server](#server) | +| `router` | `Router` | [Server](#server) | +| `control` | `Control` | [Server](#server) | +| `models` | `[]catalog.Model` | [Server](#server) | +| `endpoints` | `[]Endpoint` | [Server](#server) | +| `bin` | `string` | [SimConfig](#simconfig) | +| `store` | `string` | [Threads](#threads) | +| `retain_hours` | `int` | [Threads](#threads) | +| `dial` | `time.Duration` | [Timeouts](#timeouts) | +| `response_header` | `time.Duration` | [Timeouts](#timeouts) | +| `generation` | `time.Duration` | [Timeouts](#timeouts) | +| `first_token` | `time.Duration` | [Timeouts](#timeouts) | +| `idle` | `time.Duration` | [Timeouts](#timeouts) | +| `total` | `time.Duration` | [Timeouts](#timeouts) | +| `bin` | `string` | [VLLMConfig](#vllmconfig) | +| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) | +| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) | + +## Admission + +### `enabled` + +`bool` + +Enabled turns the layer on. Off, every request goes straight through, +which is the v0.2.0 behaviour. + +### `bands` + +`[]Band` + +### `default_band` + +`string` + +### `band_header` + +`string` + +BandHeader lets a client declare its band. Until authentication lands in +v1.0.0 this is trusted, so the port must not be exposed to callers who +should not be able to promote themselves. + +### `queue_ttl` + +`time.Duration` + +### `queue_capacity_per_band` + +`int` + +### `interval` + +`time.Duration` + +## Auth + +### `enabled` + +`bool` + +### `store` + +`string` + +Store is the SQLite file holding keys, tenancy and usage. + +### `cache_ttl` + +`time.Duration` + +CacheTTL bounds how long a revoked key keeps working, traded against a +database read per key per TTL. + +### `negative_ttl` + +`time.Duration` + +NegativeTTL is how long a rejected key is remembered as rejected, so a +client looping with a bad key cannot become a database read per request. + +### `default_rpm` + +`int` + +DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to. + +Unlimited used to be the default, and a project created without limits is +the easy path — the "default" project on this cluster still is one. The +first sign of that is the bill or the outage. Zero keeps the old +behaviour for anyone who wants it, but it should be a choice rather than +an omission. A fallback, not a ceiling: an explicit limit at any scope +wins, above or below. + +### `default_tpm` + +`int` + +### `signup_open` + +`bool` + +SignupOpen serves the signup form. Accounts it creates are pending until +an operator approves them, so this is not a way in — but turning it off +means accounts are made with `llmgw users create` instead. + +### `pricing` + +`map[string]Price` + +Pricing is what each model costs, for budgets. A model absent from the +table costs whatever DefaultPrice says, which is nothing unless set. + +### `default_price` + +`Price` + +Takes a [Price](#price) block. + +### `flush_interval` + +`time.Duration` + +FlushInterval and BatchSize decide when usage records are written. + +### `batch_size` + +`int` + +### `buffer_size` + +`int` + +BufferSize bounds the queue. Beyond it records are dropped rather than +blocking a request: losing accounting is bad, turning a stalled database +into an outage is worse. + +## Band + +### `name` + +`string` + +### `priority` + +`int` + +### `saturation_ceiling` + +`float64` + +## Capabilities + +### `tools` + +`bool` + +### `vision` + +`bool` + +### `embeddings` + +`bool` + +## Config + +### `node_id` + +`string` + +### `server` + +`string` + +Server is the control plane, and Servers is the same thing when there is +more than one. Both are accepted: every configuration written before +there could be a second control plane names one address, and none of +them should have to change. + +The order is tried in order, and a refusal moves to the next. Only the +leader accepts an agent, so a standby is skipped in the time it takes to +be told so — which is what makes failover a reconnect rather than an +edit to every node's configuration. + +### `servers` + +`[]string` + +### `hostname` + +`string` + +### `mgmt_addr` + +`string` + +### `labels` + +`map[string]string` + +### `mgmt_interface` + +`string` + +MgmtInterface is the ethernet interface NCCL bootstraps over. + +### `fabrics` + +`[]inventory.FabricAttachment` + +Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's +default subnet prefix is the same on every fabric, so nothing can be +inferred from the wire (plan §5.2). + +### `heartbeat_interval` + +`time.Duration` + +### `reconnect_base` + +`time.Duration` + +ReconnectBase is the first reconnect delay; it backs off from there. + +### `reconnect_max` + +`time.Duration` + +### `adopt_ready_timeout` + +`time.Duration` + +AdoptReadyTimeout bounds how long an adopted engine is given to answer a +health check before the agent stops waiting on it. + +Shorter than a startup timeout on purpose: this engine is already +running, so it is not loading a model, it is either serving or it is not. + +### `tls` + +`tlsconf.Config` + +TLS authenticates both ends of the control-plane stream. Leaving it unset +keeps the plaintext behaviour, which only belongs on a trusted network. + +Takes a [Config](#config) block. + +### `model_dirs` + +`[]string` + +ModelDirs are the directories this node keeps model weights in. + +Empty by default, and empty means the node is not asked to look — which +is not the same as having nothing. Storage differs at every installation, +so llmgw observes it rather than owning it: a Lustre mount, an image +exported over the fabric, local NVMe and a copy made by hand all look +identical from here, and placement only needs to know whether the file +is present. + +### `weights_cache` + +`string` + +WeightsCache is where a remote source is downloaded to. Defaults to the +first entry of ModelDirs, because a node that says where its models live +has already answered this question. + +Fetching is the simple arrangement: every node keeps its own copy. The +exotic ones need no code — weights on a shared filer or an image exported +over the fabric are simply present, and present means nothing is fetched. + +### `discovery` + +`discovery.Options` + +Takes a [Options](#options) block. + +### `engines` + +`engines.Config` + +Takes a [Config](#config) block. + +### `supervisor` + +`supervisor.Config` + +Takes a [Config](#config) block. + +## Control + +### `interval` + +`time.Duration` + +Interval is how often desired and actual state are compared. + +### `base_backoff` + +`time.Duration` + +BaseBackoff and MaxBackoff bound how fast a failing model is retried. + +### `max_backoff` + +`time.Duration` + +### `unreachable_after` + +`time.Duration` + +UnreachableAfter and LostAfter stage node expiry. They are deliberately +far apart: a network blip must not tear down replicas that still serve, +and rebuilding a multi-node replica costs tens of minutes of loading. + +### `lost_after` + +`time.Duration` + +### `port_range` + +`[]int` + +PortRange is the per-node range for engine HTTP and dist-init ports. + +### `mgmt_interface` + +`string` + +MgmtInterface is the ethernet interface NCCL bootstraps over. + +### `lease_ttl` + +`time.Duration` + +LeaseTTL is how long this control plane's claim to be the one placing +work stands without being renewed. + +Only meaningful with a shared store, where a second control plane can +see the claim. Long enough to survive a slow query or a brief network +stall, short enough that a standby takes over before anyone has finished +reading the alert. Zero takes the default. + +## Endpoint + +### `id` + +`string` + +### `url` + +`string` + +### `models` + +`[]string` + +### `engine` + +`string` + +Engine names the inference engine, which decides what its metrics are +called. Empty means vLLM's names. + +### `pool` + +`string` + +### `weight` + +`float64` + +### `priority` + +`int` + +## Landing + +### `contact` + +`string` + +Contact is where someone without a key should go — an address, a URL, a +channel name. Rendered as a link when it is one, as text otherwise. + +### `notice` + +`string` + +Notice is an optional line of your own, for whatever the reader needs to +know that this page cannot guess. + +## LlamaCppConfig + +### `bin` + +`string` + +Bin is the llama-server binary. It is the server, not the CLI: the +gateway talks OpenAI over HTTP and needs the one that serves it. + +### `parallel` + +`int` + +Parallel is how many requests a server handles at once, when a profile +does not say. llama.cpp defaults to one, which would make every second +request wait behind the first no matter what the router decided. + +## Model + +### `name` + +`string` + +### `aliases` + +`[]string` + +### `source` + +`string` + +Source is the weights path or a HuggingFace id. + +### `profiles` + +`[]Profile` + +### `scaling` + +`Scaling` + +Takes a [Scaling](#scaling) block. + +### `fallbacks` + +`[]string` + +### `routing_profile` + +`string` + +RoutingProfile selects how the router scores endpoints for this model. + +### `slo` + +`SLO` + +Takes a [SLO](#slo) block. + +### `context_length` + +`int` + +ContextLength is how many tokens one request may use, prompt and +completion together, and MaxOutput bounds the completion alone. Zero +means unknown, which is what every model was until now. + +The gateway knew neither, so an over-long prompt was not refused: it was +admitted, given a replica, and failed at the engine, and a client sizing +its prompts from /v1/models had nothing to read. Worse, the number is not +the one an operator writes in the engine's arguments — llama.cpp divides +--ctx-size across its parallel slots, so a pool of 8192 across four slots +is 2048 a request, and nothing said so. + +### `max_output` + +`int` + +### `capabilities` + +`Capabilities` + +Capabilities is what this model can do beyond plain completion. + +Takes a [Capabilities](#capabilities) block. + +## Options + +### `nvidia_smi` + +`string` + +NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH. +Worth configuring on nodes where the driver lives outside the default +paths, which containerized driver installs often do. + +### `ib_root` + +`string` + +IBRoot is the sysfs InfiniBand directory. + +### `prefer_nvml` + +`bool` + +PreferNVML uses the NVML backend when this binary has one. It is the only +source of NVLink topology. + +### `skip_gpus` + +`bool` + +SkipGPUs omits GPU discovery, for nodes that have none. + +## Prefix + +### `enabled` + +`bool` + +### `max_nodes` + +`int` + +MaxNodes bounds the tree, MinMatch is the shortest match worth acting on +in bytes, and TTL is how long an entry is believed — an engine evicts on +its own schedule, so an old record may describe a cache that is gone. + +### `min_match` + +`int` + +### `ttl` + +`time.Duration` + +### `match_ratio` + +`float64` + +MatchRatio is the share of the prompt that must match to be worth acting +on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip +before affinity is abandoned. + +### `balance_abs` + +`float64` + +### `balance_rel` + +`float64` + +## Price + +### `prompt_per_1k` + +`int64` + +### `completion_per_1k` + +`int64` + +## Profile + +### `name` + +`string` + +### `engine` + +`string` + +### `gpu_class` + +`string` + +### `tp` + +`int` + +### `pp` + +`int` + +### `gpus_per_node` + +`int` + +GPUsPerNode defaults to TP for a single-node profile. + +### `nodes` + +`int` + +Nodes above 1 makes this a multi-node profile, which requires a fabric. + +### `args` + +`[]string` + +### `env` + +`map[string]string` + +### `min_memory_gb` + +`int` + +MinMemoryGB is the host RAM this profile needs to run. + +The number that matters is the model's working set: the weights, plus the +KV cache for the whole context pool, plus the engine's own overhead. For +a GPU profile that lives in the cards and this is small; for a CPU +profile it is the whole thing, and getting it wrong means an OOM on a +node that reported itself healthy. + +Zero means unconstrained, which is every profile that existed before +this. A node that does not report its memory is not excluded either — +an older agent must not have its nodes quietly emptied by an upgrade of +the control plane. + +### `min_disk_gb` + +`int` + +MinDiskGB is the free space a node needs before it can hold these +weights. + +Only checked when the node does not already have them: a node holding +the file needs no room for it, and requiring the space twice would keep +a model off the one node best placed to run it. + +Declared rather than derived, because the size of a model that has never +been fetched anywhere is not knowable from here — and a wrong guess +fails late, halfway through a download, which is the failure this exists +to prevent. Zero means unconstrained, and a node that reports no storage +is not excluded: unknown is not full. + +### `node_selector` + +`map[string]string` + +NodeSelector restricts this profile to nodes carrying every one of these +labels, with these values. + +A CPU profile otherwise fits everywhere, because there is no card to be +free — which is right for a 0.5B model and wrong for a 14B one. Nothing +in the inventory says how much memory a node has, so the operator says +which nodes are meant for a model instead of the scheduler inferring it. +Agents have reported labels since the beginning; this is what reads them. + +### `weight` + +`float64` + +Weight is the router's capacity weight. It should come from ctl bench +rather than a guess (plan §6.2.1). + +### `priority` + +`int` + +Priority orders profiles during placement; highest that fits wins. + +### `startup_timeout` + +`time.Duration` + +StartupTimeout bounds model loading. A 70B model across two nodes takes +tens of minutes, so this is per profile rather than global. + +## Router + +### `activation_wait` + +`time.Duration` + +ActivationWait is how long a request waits for an on-demand model to +load before the client is told to come back. Zero takes the default. + +The one real decision in on-demand loading. Refusing outright makes the +feature useless, since the first request for every model would fail; +waiting without a bound turns a request into a job, and a client that +budgeted thirty seconds ends up holding a connection for ten minutes. + +### `scrape_interval` + +`time.Duration` + +### `scrape_timeout` + +`time.Duration` + +### `stale_after` + +`time.Duration` + +StaleAfter is how long an endpoint may go unscraped before being scored +down (plan §7.6). + +### `retries` + +`int` + +Retries is how many *additional* endpoints may be tried after a failure, +and only while nothing has been written to the client (plan §7.7). + +### `cooldown` + +`time.Duration` + +Cooldown is how long a failing endpoint leaves the candidate pool. + +### `failure_threshold` + +`int` + +FailureThreshold is the consecutive failure count that opens a breaker. +1 turns the breaker into a pure cooldown. + +### `default_profile` + +`string` + +DefaultProfile names the routing profile used when a model picks none. + +### `prefix` + +`Prefix` + +Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the +router uses session affinity alone, which is tier 0 and already most of +the win for multi-turn traffic. + +Takes a [Prefix](#prefix) block. + +### `session_header` + +`string` + +SessionHeader carries the session id for affinity (plan §7.5 tier 0). + +### `inject_usage` + +`bool` + +InjectUsage asks engines for usage even when the client did not, so token +accounting works regardless of client behaviour. + +### `max_body_bytes` + +`int64` + +MaxBodyBytes bounds the request body the router will read. + +### `timeouts` + +`Timeouts` + +Takes a [Timeouts](#timeouts) block. + +### `scoring` + +`Scoring` + +Takes a [Scoring](#scoring) block. + +### `admission` + +`Admission` + +Takes a [Admission](#admission) block. + +## SGLangConfig + +### `python` + +`string` + +Python is the interpreter of the SGLang virtualenv. SGLang is launched as +a module rather than a console script, which is the documented entry +point and avoids depending on the script being on PATH. + +## SLO + +### `ttft_p99` + +`time.Duration` + +### `tpot_p99` + +`time.Duration` + +## Scaling + +### `min` + +`int` + +### `max` + +`int` + +### `idle_timeout` + +`time.Duration` + +IdleTimeout is how long an on-demand model may go unasked-for before its +replicas are stopped. Zero takes the cluster default. + +Minutes rather than seconds, always. Stopping a replica throws away the +KV cache prefix routing spent its effort filling, and a model that flaps +in and out loses that entirely — so the saving has to be worth more than +the cache, which it only is over a real idle period. + +## Scoring + +### `target_waiting` + +`float64` + +### `target_inflight` + +`float64` + +### `kv_ceiling` + +`float64` + +### `session_load_factor` + +`float64` + +## Server + +### `listen` + +`string` + +Listen is the client-facing OpenAI API address. + +### `admin_listen` + +`string` + +AdminListen serves the dashboard, the management API, health and pprof. + +Loopback by default, and it should stay there: pprof can dump the process +heap, and API keys are in that heap in plaintext. Every comment beside +this setting has always said not to expose it, while the default bound +every interface — a default that contradicts its own documentation is a +footgun, so the default now matches the advice. + +### `metrics_listen` + +`string` + +MetricsListen serves Prometheus metrics and nothing else. + +Separate from AdminListen because the reason that one is loopback is +pprof, not metrics — and metrics are the one thing on it that something +else genuinely needs to reach. Sharing a listener meant a fleet could +either be monitored or keep its heap private, which is not a choice +anybody should have to make. + +Empty means off, and metrics remain available on AdminListen either way, +so nothing that scrapes it today has to change. + +### `agent_listen` + +`string` + +AgentListen is where node agents connect. Separate from the client port +so the fleet-facing surface can be firewalled independently. + +### `auth` + +`Auth` + +Auth turns on API keys on the client-facing port. + +Takes a [Auth](#auth) block. + +### `threads` + +`Threads` + +Threads keeps conversations and agent memory without API keys, for a +gateway on one trusted machine. With auth on they live in the auth store, +per project, and this block is ignored. + +Takes a [Threads](#threads) block. + +### `landing` + +`Landing` + +Landing configures the page served at "/". + +Takes a [Landing](#landing) block. + +### `agent_tls` + +`tlsconf.Config` + +AgentTLS authenticates the fleet-facing port. It deliberately does not +cover the client-facing one: that surface is fronted by a publicly +trusted certificate, while this one is mutual and private. + +Takes a [Config](#config) block. + +### `router` + +`Router` + +Takes a [Router](#router) block. + +### `control` + +`Control` + +Takes a [Control](#control) block. + +### `models` + +`[]catalog.Model` + +### `endpoints` + +`[]Endpoint` + +Endpoints is the phase-1 static replica list. It is only consulted when +no model declares a profile, which is what makes a router-only smoke test +against sim-engines possible without a control plane. + +## SimConfig + +### `bin` + +`string` + +Bin is the llmgw binary itself; the engine is one of its subcommands. +Empty means whatever started this agent, which is right almost always. + +## Threads + +### `store` + +`string` + +Store is the SQLite file. Empty means nothing is kept. + +### `retain_hours` + +`int` + +RetainHours is how long idle conversations and notes are kept. Required +with Store: a store with no expiry keeps everything anyone typed, by +omission. + +## Timeouts + +### `dial` + +`time.Duration` + +### `response_header` + +`time.Duration` + +ResponseHeader bounds the wait for headers on a streaming request. It is +a liveness check there, because headers arrive before generation starts. + +### `generation` + +`time.Duration` + +Generation bounds a non-streaming request, where the engine sends no +headers until the completion is done. Setting response_header alone used +to cap both, so 60s read as a liveness guard and behaved as a limit of +about a hundred tokens on a slow model. + +### `first_token` + +`time.Duration` + +### `idle` + +`time.Duration` + +### `total` + +`time.Duration` + +## VLLMConfig + +### `bin` + +`string` + +Bin is the vllm executable. + +### `launch_mode` + +`string` + +LaunchMode selects the multi-node mechanism: "ray" or "mp". + +Which one works depends on the vLLM version, and the flags have changed +more than once (plan §14). Both paths exist so a cluster can pick the one +its version supports instead of waiting for a code change. + +### `ray_bin` + +`string` + +RayBin is used when LaunchMode is "ray". + diff --git a/internal/docgen/docgen.go b/internal/docgen/docgen.go new file mode 100644 index 00000000..03f3f8fe --- /dev/null +++ b/internal/docgen/docgen.go @@ -0,0 +1,158 @@ +// Package docgen turns the configuration structs into reference documentation. +// +// Generated rather than written, because a hand-written reference is a promise +// that someone will remember. This project added six settings in a fortnight — +// store, servers, lease_ttl, thread and memory retention, min_disk_gb — and +// every one of them reached an operator only through a narrative document that +// happened to mention it. A reference nobody maintains is worse than none: it +// is the same list, quietly missing the setting you came to look up. +// +// Derived from the source, so the only way for it to be wrong is for the code +// to be wrong. A test regenerates it and fails when it has drifted, which is +// what keeps that true after everyone has forgotten this exists. +package docgen + +//go:generate go run ./gen + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "reflect" + "sort" + "strings" +) + +// Field is one configuration key. +type Field struct { + Key string + Type string + Doc string + // Nested names the struct this key expands into, empty for a leaf. It is + // what lets the reference say "see Auth" rather than printing the same + // twenty keys under four parents. + Nested string +} + +// Block is a struct that appears in configuration. +type Block struct { + Name string + Doc string + Fields []Field +} + +// Collect reads koanf-tagged structs out of Go source files. +func Collect(files []string) ([]Block, error) { + fset := token.NewFileSet() + var blocks []Block + seen := map[string]bool{} + + for _, path := range files { + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + ast.Inspect(f, func(n ast.Node) bool { + ts, ok := n.(*ast.TypeSpec) + if !ok { + return true + } + st, ok := ts.Type.(*ast.StructType) + if !ok { + return true + } + b, ok := blockOf(ts, st) + if !ok || seen[b.Name] { + return true + } + seen[b.Name] = true + blocks = append(blocks, b) + return true + }) + } + sort.Slice(blocks, func(i, j int) bool { return blocks[i].Name < blocks[j].Name }) + return blocks, nil +} + +// blockOf builds a Block, or reports that this struct carries no settings. +func blockOf(ts *ast.TypeSpec, st *ast.StructType) (Block, bool) { + b := Block{Name: ts.Name.Name, Doc: docText(ts.Doc)} + for _, f := range st.Fields.List { + if f.Tag == nil { + continue + } + tag := reflect.StructTag(strings.Trim(f.Tag.Value, "`")) + key, ok := tag.Lookup("koanf") + // "-" is a field carried in the struct but never read from a file; + // listing it would send someone looking for a setting that does not + // exist. + if !ok || key == "" || key == "-" { + continue + } + typ, nested := typeOf(f.Type) + b.Fields = append(b.Fields, Field{ + Key: key, Type: typ, Doc: docText(f.Doc), Nested: nested, + }) + } + return b, len(b.Fields) > 0 +} + +// typeOf renders a field's type, and names the struct it expands into. +func typeOf(e ast.Expr) (rendered, nested string) { + switch t := e.(type) { + case *ast.Ident: + if isNamedStruct(t.Name) { + return t.Name, t.Name + } + return t.Name, "" + case *ast.SelectorExpr: + // A type from another package, such as tlsconf.Config. + name := t.Sel.Name + if pkg, ok := t.X.(*ast.Ident); ok { + return pkg.Name + "." + name, name + } + return name, name + case *ast.ArrayType: + inner, _ := typeOf(t.Elt) + return "[]" + inner, "" + case *ast.MapType: + k, _ := typeOf(t.Key) + v, _ := typeOf(t.Value) + return "map[" + k + "]" + v, "" + case *ast.StarExpr: + inner, n := typeOf(t.X) + return "*" + inner, n + default: + return "value", "" + } +} + +// isNamedStruct guesses whether an identifier names a configuration block. +// +// By capitalisation, because the alternative is resolving types across +// packages for a document. A wrong guess costs a "see also" that points at +// nothing, which the renderer drops. +func isNamedStruct(name string) bool { + if name == "" { + return false + } + switch name { + case "string", "bool", "int", "int64", "float64", "any", "byte", "rune": + return false + } + return name[0] >= 'A' && name[0] <= 'Z' +} + +// docText flattens a comment group into one paragraph per blank line. +func docText(g *ast.CommentGroup) string { + if g == nil { + return "" + } + var lines []string + for _, c := range g.List { + t := strings.TrimPrefix(c.Text, "//") + lines = append(lines, strings.TrimPrefix(t, " ")) + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} diff --git a/internal/docgen/docgen_test.go b/internal/docgen/docgen_test.go new file mode 100644 index 00000000..2da93e3f --- /dev/null +++ b/internal/docgen/docgen_test.go @@ -0,0 +1,138 @@ +package docgen + +import ( + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +// repoRoot is where the source paths in Sources are relative to. +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate this test") + } + return filepath.Join(filepath.Dir(file), "..", "..") +} + +func sourcePaths(t *testing.T) []string { + t.Helper() + root := repoRoot(t) + out := make([]string, len(Sources)) + for i, s := range Sources { + out[i] = filepath.Join(root, s) + } + return out +} + +func TestTheReferenceIsNotStale(t *testing.T) { + // The whole point. A generated document that nobody regenerates is the + // same lie as a hand-written one, arrived at more slowly: it describes the + // configuration of some earlier week, and the setting somebody came to + // look up is the one that is missing. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + want := Render(blocks) + + path := filepath.Join(repoRoot(t), "docs", "reference.md") + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("docs/reference.md is missing; run go generate ./...: %v", err) + } + if string(got) != want { + t.Error("docs/reference.md no longer matches the configuration structs.\n" + + "Run: go generate ./...") + } +} + +func TestEveryConfigurationFileIsListed(t *testing.T) { + // Sources is a list, and a list is a thing to forget. A new file full of + // settings would otherwise be absent from the reference with nothing + // saying so — which is exactly the failure the reference exists to fix. + root := repoRoot(t) + var tagged []string + err := filepath.Walk(filepath.Join(root, "internal"), func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") { + return err + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + if !strings.Contains(string(b), `koanf:"`) { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + tagged = append(tagged, rel) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(tagged) == 0 { + t.Fatal("found no configuration files at all, so this test proves nothing") + } + + for _, f := range tagged { + if slices.Contains(Sources, f) { + continue + } + // Deliberately out: these hold koanf tags for structs an operator + // never writes — inventory and catalog entries come from the API or + // from a model file, not from the configuration. + if slices.Contains(notReference, f) { + continue + } + t.Errorf("%s declares configuration and is not in Sources or notReference", f) + } +} + +func TestADocumentedKeyKeepsItsComment(t *testing.T) { + // The descriptions are the field comments, which is what makes the + // reference worth reading rather than a list of names. A field whose + // comment was the only explanation anywhere must not render blank. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + var lease Field + for _, b := range blocks { + for _, f := range b.Fields { + if f.Key == "lease_ttl" { + lease = f + } + } + } + if lease.Key == "" { + t.Fatal("lease_ttl is not in the reference") + } + if !strings.Contains(lease.Doc, "shared store") { + t.Errorf("lease_ttl lost its explanation: %q", lease.Doc) + } +} + +func TestAFieldMarkedNotConfigurableIsLeftOut(t *testing.T) { + // koanf:"-" means the field is carried in the struct and never read from + // a file. Listing it would send somebody looking for a setting that does + // not exist. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + for _, b := range blocks { + for _, f := range b.Fields { + if f.Key == "-" || f.Key == "" { + t.Errorf("%s lists a non-configurable field: %+v", b.Name, f) + } + } + } +} diff --git a/internal/docgen/gen/main.go b/internal/docgen/gen/main.go new file mode 100644 index 00000000..65d6efa9 --- /dev/null +++ b/internal/docgen/gen/main.go @@ -0,0 +1,42 @@ +// Command gen writes docs/reference.md from the configuration structs. +// +// Run by `go generate ./...`, and checked by a test, so the reference cannot +// quietly stop describing the code. +package main + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" +) + +func main() { + // The repository root, found from this file rather than from the working + // directory, so `go generate ./...` produces the same document wherever + // it is run from. + _, self, _, ok := runtime.Caller(0) + if !ok { + fmt.Fprintln(os.Stderr, "error: cannot locate the generator") + os.Exit(1) + } + root := filepath.Join(filepath.Dir(self), "..", "..", "..") + + paths := make([]string, len(docgen.Sources)) + for i, s := range docgen.Sources { + paths[i] = filepath.Join(root, s) + } + + blocks, err := docgen.Collect(paths) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + if err := os.WriteFile(filepath.Join(root, "docs", "reference.md"), []byte(docgen.Render(blocks)), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + fmt.Printf("docs/reference.md: %d blocks\n", len(blocks)) +} diff --git a/internal/docgen/render.go b/internal/docgen/render.go new file mode 100644 index 00000000..6d00117a --- /dev/null +++ b/internal/docgen/render.go @@ -0,0 +1,99 @@ +package docgen + +import ( + "fmt" + "strings" +) + +// Sources are the files configuration is declared in. +// +// Listed rather than discovered, because "every file with a koanf tag" would +// sweep in structs that are configuration in the Go sense and not in the +// operator's — a scoring weight built at wiring time is not something anyone +// writes in a file. A test checks the list against what is actually tagged, so +// a new configuration file is a failure here rather than a silent omission. +var Sources = []string{ + "internal/config/config.go", + "internal/agent/agent.go", + "internal/agent/discovery/discover.go", + "internal/agent/engines/engine.go", + "internal/agent/engines/llamacpp.go", + "internal/agent/engines/vllm.go", + "internal/agent/engines/sglang.go", + "internal/agent/engines/sim.go", + "internal/agent/supervisor/supervisor.go", + "internal/tlsconf/tlsconf.go", + "internal/control/catalog/catalog.go", + "internal/router/admission/controller.go", + "internal/accounting/cost.go", +} + +// notReference are files with koanf tags that an operator never writes. +// +// Inventory and catalog entries carry them so the same structs can be decoded +// from an API payload or a model file; putting them in a configuration +// reference would invite somebody to write them into server.yaml. +var notReference = []string{ + "internal/control/inventory/inventory.go", +} + +// Render turns collected blocks into the reference document. +func Render(blocks []Block) string { + var b strings.Builder + b.WriteString(`# Configuration reference + +Every setting llmgw reads from a file, with the type it takes and what it is +for. + +**Generated from the source.** The descriptions are the comments on the fields +themselves, so this cannot describe a setting that does not exist or miss one +that does — and a test fails when it has drifted. Do not edit it; edit the +comment and run ` + "`go generate ./...`" + `. + +Where a setting takes another block, the block is listed under its own heading +rather than repeated. + +`) + + // An index first: the reason to open a reference is usually to find one + // key, and scanning thirteen headings for it is the thing this replaces. + b.WriteString("## Every key\n\n| Key | Type | In |\n|---|---|---|\n") + for _, bl := range blocks { + for _, f := range bl.Fields { + b.WriteString(fmt.Sprintf("| `%s` | %s | [%s](#%s) |\n", + f.Key, code(f.Type), bl.Name, anchor(bl.Name))) + } + } + b.WriteString("\n") + + for _, bl := range blocks { + fmt.Fprintf(&b, "## %s\n\n", bl.Name) + if bl.Doc != "" { + b.WriteString(bl.Doc + "\n\n") + } + for _, f := range bl.Fields { + fmt.Fprintf(&b, "### `%s`\n\n%s\n\n", f.Key, code(f.Type)) + if f.Doc != "" { + b.WriteString(f.Doc + "\n\n") + } + if f.Nested != "" && hasBlock(blocks, f.Nested) { + fmt.Fprintf(&b, "Takes a [%s](#%s) block.\n\n", f.Nested, anchor(f.Nested)) + } + } + } + return b.String() +} + +func hasBlock(blocks []Block, name string) bool { + for _, b := range blocks { + if b.Name == name { + return true + } + } + return false +} + +func code(s string) string { return "`" + s + "`" } + +// anchor is how GitHub and Gitea slug a heading. +func anchor(s string) string { return strings.ToLower(s) }