From 100f10ac170502ade699bd8a53ef53e4e815a078 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 07:43:30 +0200 Subject: [PATCH] Generate the configuration reference, and fail when it goes stale MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was no reference. Four narrative documents explain how to install and operate llmgw, and the only way to learn that `lease_ttl` exists was to read the one that happened to mention it — or the struct. Written by hand it would have been wrong within a fortnight. Six settings arrived in the last two weeks alone: store, servers, lease_ttl, the two retention windows, min_disk_gb. A reference nobody maintains is worse than none, because it is the same list with the setting you came to look up quietly missing from it. So it is derived from the structs, with the field comments as the descriptions — which means the only way for it to be wrong is for the code to be wrong, and it makes the comments worth writing twice over. 1163 lines, 24 blocks, 87 keys. Two tests keep that true after everyone has forgotten this exists. One regenerates and compares, so adding a setting without regenerating is a failure rather than an omission — checked by adding one and watching it fail. The other walks the tree for koanf tags and fails on a configuration file that is in neither the source list nor the short list of structs an operator never writes, because a list is a thing to forget. The generator finds the repository from its own path rather than the working directory, so `go generate ./...` produces the same document from anywhere. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 11 + docs/install.md | 3 + docs/reference.md | 1163 ++++++++++++++++++++++++++++++++ internal/docgen/docgen.go | 158 +++++ internal/docgen/docgen_test.go | 138 ++++ internal/docgen/gen/main.go | 42 ++ internal/docgen/render.go | 99 +++ 7 files changed, 1614 insertions(+) create mode 100644 docs/reference.md create mode 100644 internal/docgen/docgen.go create mode 100644 internal/docgen/docgen_test.go create mode 100644 internal/docgen/gen/main.go create mode 100644 internal/docgen/render.go diff --git a/README.md b/README.md index f23e93e2..26c1e9ba 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,17 @@ without a fabric. The InfiniBand, NVML and NCCL paths are written against the formats and failure modes those tools actually have, but they have not yet run on a machine that has them. +## Documentation + +- [docs/install.md](docs/install.md) — installing from nothing: control plane, + nodes, engines, weights, first request +- [docs/quickstart.md](docs/quickstart.md) — the short path +- [docs/reference.md](docs/reference.md) — every configuration key, generated + from the structs and checked by a test, so it cannot describe a setting that + does not exist or miss one that does +- [docs/node-onboarding.md](docs/node-onboarding.md), + [docs/onboarding.md](docs/onboarding.md) — adding machines, and adding people + ## Building ```sh diff --git a/docs/install.md b/docs/install.md index 84063330..8ed32b23 100644 --- a/docs/install.md +++ b/docs/install.md @@ -774,6 +774,9 @@ requests. On a slow model, set `timeouts.generation` higher. ## What to read next +- `docs/reference.md` — every configuration key, generated from the source, so + it describes the version you are running rather than the one somebody last + wrote about - `docs/node-onboarding.md` — adding nodes in more detail, and what onboarding deliberately does not do - `docs/onboarding.md` — giving people accounts and keys diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 00000000..7d3a9199 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,1163 @@ +# Configuration reference + +Every setting llmgw reads from a file, with the type it takes and what it is +for. + +**Generated from the source.** The descriptions are the comments on the fields +themselves, so this cannot describe a setting that does not exist or miss one +that does — and a test fails when it has drifted. Do not edit it; edit the +comment and run `go generate ./...`. + +Where a setting takes another block, the block is listed under its own heading +rather than repeated. + +## Every key + +| Key | Type | In | +|---|---|---| +| `enabled` | `bool` | [Admission](#admission) | +| `bands` | `[]Band` | [Admission](#admission) | +| `default_band` | `string` | [Admission](#admission) | +| `band_header` | `string` | [Admission](#admission) | +| `queue_ttl` | `time.Duration` | [Admission](#admission) | +| `queue_capacity_per_band` | `int` | [Admission](#admission) | +| `interval` | `time.Duration` | [Admission](#admission) | +| `enabled` | `bool` | [Auth](#auth) | +| `store` | `string` | [Auth](#auth) | +| `cache_ttl` | `time.Duration` | [Auth](#auth) | +| `negative_ttl` | `time.Duration` | [Auth](#auth) | +| `default_rpm` | `int` | [Auth](#auth) | +| `default_tpm` | `int` | [Auth](#auth) | +| `signup_open` | `bool` | [Auth](#auth) | +| `pricing` | `map[string]Price` | [Auth](#auth) | +| `default_price` | `Price` | [Auth](#auth) | +| `flush_interval` | `time.Duration` | [Auth](#auth) | +| `batch_size` | `int` | [Auth](#auth) | +| `buffer_size` | `int` | [Auth](#auth) | +| `name` | `string` | [Band](#band) | +| `priority` | `int` | [Band](#band) | +| `saturation_ceiling` | `float64` | [Band](#band) | +| `tools` | `bool` | [Capabilities](#capabilities) | +| `vision` | `bool` | [Capabilities](#capabilities) | +| `embeddings` | `bool` | [Capabilities](#capabilities) | +| `node_id` | `string` | [Config](#config) | +| `server` | `string` | [Config](#config) | +| `servers` | `[]string` | [Config](#config) | +| `hostname` | `string` | [Config](#config) | +| `mgmt_addr` | `string` | [Config](#config) | +| `labels` | `map[string]string` | [Config](#config) | +| `mgmt_interface` | `string` | [Config](#config) | +| `fabrics` | `[]inventory.FabricAttachment` | [Config](#config) | +| `heartbeat_interval` | `time.Duration` | [Config](#config) | +| `reconnect_base` | `time.Duration` | [Config](#config) | +| `reconnect_max` | `time.Duration` | [Config](#config) | +| `adopt_ready_timeout` | `time.Duration` | [Config](#config) | +| `tls` | `tlsconf.Config` | [Config](#config) | +| `model_dirs` | `[]string` | [Config](#config) | +| `weights_cache` | `string` | [Config](#config) | +| `discovery` | `discovery.Options` | [Config](#config) | +| `engines` | `engines.Config` | [Config](#config) | +| `supervisor` | `supervisor.Config` | [Config](#config) | +| `interval` | `time.Duration` | [Control](#control) | +| `base_backoff` | `time.Duration` | [Control](#control) | +| `max_backoff` | `time.Duration` | [Control](#control) | +| `unreachable_after` | `time.Duration` | [Control](#control) | +| `lost_after` | `time.Duration` | [Control](#control) | +| `port_range` | `[]int` | [Control](#control) | +| `mgmt_interface` | `string` | [Control](#control) | +| `lease_ttl` | `time.Duration` | [Control](#control) | +| `id` | `string` | [Endpoint](#endpoint) | +| `url` | `string` | [Endpoint](#endpoint) | +| `models` | `[]string` | [Endpoint](#endpoint) | +| `engine` | `string` | [Endpoint](#endpoint) | +| `pool` | `string` | [Endpoint](#endpoint) | +| `weight` | `float64` | [Endpoint](#endpoint) | +| `priority` | `int` | [Endpoint](#endpoint) | +| `contact` | `string` | [Landing](#landing) | +| `notice` | `string` | [Landing](#landing) | +| `bin` | `string` | [LlamaCppConfig](#llamacppconfig) | +| `parallel` | `int` | [LlamaCppConfig](#llamacppconfig) | +| `name` | `string` | [Model](#model) | +| `aliases` | `[]string` | [Model](#model) | +| `source` | `string` | [Model](#model) | +| `profiles` | `[]Profile` | [Model](#model) | +| `scaling` | `Scaling` | [Model](#model) | +| `fallbacks` | `[]string` | [Model](#model) | +| `routing_profile` | `string` | [Model](#model) | +| `slo` | `SLO` | [Model](#model) | +| `context_length` | `int` | [Model](#model) | +| `max_output` | `int` | [Model](#model) | +| `capabilities` | `Capabilities` | [Model](#model) | +| `nvidia_smi` | `string` | [Options](#options) | +| `ib_root` | `string` | [Options](#options) | +| `prefer_nvml` | `bool` | [Options](#options) | +| `skip_gpus` | `bool` | [Options](#options) | +| `enabled` | `bool` | [Prefix](#prefix) | +| `max_nodes` | `int` | [Prefix](#prefix) | +| `min_match` | `int` | [Prefix](#prefix) | +| `ttl` | `time.Duration` | [Prefix](#prefix) | +| `match_ratio` | `float64` | [Prefix](#prefix) | +| `balance_abs` | `float64` | [Prefix](#prefix) | +| `balance_rel` | `float64` | [Prefix](#prefix) | +| `prompt_per_1k` | `int64` | [Price](#price) | +| `completion_per_1k` | `int64` | [Price](#price) | +| `name` | `string` | [Profile](#profile) | +| `engine` | `string` | [Profile](#profile) | +| `gpu_class` | `string` | [Profile](#profile) | +| `tp` | `int` | [Profile](#profile) | +| `pp` | `int` | [Profile](#profile) | +| `gpus_per_node` | `int` | [Profile](#profile) | +| `nodes` | `int` | [Profile](#profile) | +| `args` | `[]string` | [Profile](#profile) | +| `env` | `map[string]string` | [Profile](#profile) | +| `min_memory_gb` | `int` | [Profile](#profile) | +| `min_disk_gb` | `int` | [Profile](#profile) | +| `node_selector` | `map[string]string` | [Profile](#profile) | +| `weight` | `float64` | [Profile](#profile) | +| `priority` | `int` | [Profile](#profile) | +| `startup_timeout` | `time.Duration` | [Profile](#profile) | +| `activation_wait` | `time.Duration` | [Router](#router) | +| `scrape_interval` | `time.Duration` | [Router](#router) | +| `scrape_timeout` | `time.Duration` | [Router](#router) | +| `stale_after` | `time.Duration` | [Router](#router) | +| `retries` | `int` | [Router](#router) | +| `cooldown` | `time.Duration` | [Router](#router) | +| `failure_threshold` | `int` | [Router](#router) | +| `default_profile` | `string` | [Router](#router) | +| `prefix` | `Prefix` | [Router](#router) | +| `session_header` | `string` | [Router](#router) | +| `inject_usage` | `bool` | [Router](#router) | +| `max_body_bytes` | `int64` | [Router](#router) | +| `timeouts` | `Timeouts` | [Router](#router) | +| `scoring` | `Scoring` | [Router](#router) | +| `admission` | `Admission` | [Router](#router) | +| `python` | `string` | [SGLangConfig](#sglangconfig) | +| `ttft_p99` | `time.Duration` | [SLO](#slo) | +| `tpot_p99` | `time.Duration` | [SLO](#slo) | +| `min` | `int` | [Scaling](#scaling) | +| `max` | `int` | [Scaling](#scaling) | +| `idle_timeout` | `time.Duration` | [Scaling](#scaling) | +| `target_waiting` | `float64` | [Scoring](#scoring) | +| `target_inflight` | `float64` | [Scoring](#scoring) | +| `kv_ceiling` | `float64` | [Scoring](#scoring) | +| `session_load_factor` | `float64` | [Scoring](#scoring) | +| `listen` | `string` | [Server](#server) | +| `admin_listen` | `string` | [Server](#server) | +| `metrics_listen` | `string` | [Server](#server) | +| `agent_listen` | `string` | [Server](#server) | +| `auth` | `Auth` | [Server](#server) | +| `threads` | `Threads` | [Server](#server) | +| `landing` | `Landing` | [Server](#server) | +| `agent_tls` | `tlsconf.Config` | [Server](#server) | +| `router` | `Router` | [Server](#server) | +| `control` | `Control` | [Server](#server) | +| `models` | `[]catalog.Model` | [Server](#server) | +| `endpoints` | `[]Endpoint` | [Server](#server) | +| `bin` | `string` | [SimConfig](#simconfig) | +| `store` | `string` | [Threads](#threads) | +| `retain_hours` | `int` | [Threads](#threads) | +| `dial` | `time.Duration` | [Timeouts](#timeouts) | +| `response_header` | `time.Duration` | [Timeouts](#timeouts) | +| `generation` | `time.Duration` | [Timeouts](#timeouts) | +| `first_token` | `time.Duration` | [Timeouts](#timeouts) | +| `idle` | `time.Duration` | [Timeouts](#timeouts) | +| `total` | `time.Duration` | [Timeouts](#timeouts) | +| `bin` | `string` | [VLLMConfig](#vllmconfig) | +| `launch_mode` | `string` | [VLLMConfig](#vllmconfig) | +| `ray_bin` | `string` | [VLLMConfig](#vllmconfig) | + +## Admission + +### `enabled` + +`bool` + +Enabled turns the layer on. Off, every request goes straight through, +which is the v0.2.0 behaviour. + +### `bands` + +`[]Band` + +### `default_band` + +`string` + +### `band_header` + +`string` + +BandHeader lets a client declare its band. Until authentication lands in +v1.0.0 this is trusted, so the port must not be exposed to callers who +should not be able to promote themselves. + +### `queue_ttl` + +`time.Duration` + +### `queue_capacity_per_band` + +`int` + +### `interval` + +`time.Duration` + +## Auth + +### `enabled` + +`bool` + +### `store` + +`string` + +Store is the SQLite file holding keys, tenancy and usage. + +### `cache_ttl` + +`time.Duration` + +CacheTTL bounds how long a revoked key keeps working, traded against a +database read per key per TTL. + +### `negative_ttl` + +`time.Duration` + +NegativeTTL is how long a rejected key is remembered as rejected, so a +client looping with a bad key cannot become a database read per request. + +### `default_rpm` + +`int` + +DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to. + +Unlimited used to be the default, and a project created without limits is +the easy path — the "default" project on this cluster still is one. The +first sign of that is the bill or the outage. Zero keeps the old +behaviour for anyone who wants it, but it should be a choice rather than +an omission. A fallback, not a ceiling: an explicit limit at any scope +wins, above or below. + +### `default_tpm` + +`int` + +### `signup_open` + +`bool` + +SignupOpen serves the signup form. Accounts it creates are pending until +an operator approves them, so this is not a way in — but turning it off +means accounts are made with `llmgw users create` instead. + +### `pricing` + +`map[string]Price` + +Pricing is what each model costs, for budgets. A model absent from the +table costs whatever DefaultPrice says, which is nothing unless set. + +### `default_price` + +`Price` + +Takes a [Price](#price) block. + +### `flush_interval` + +`time.Duration` + +FlushInterval and BatchSize decide when usage records are written. + +### `batch_size` + +`int` + +### `buffer_size` + +`int` + +BufferSize bounds the queue. Beyond it records are dropped rather than +blocking a request: losing accounting is bad, turning a stalled database +into an outage is worse. + +## Band + +### `name` + +`string` + +### `priority` + +`int` + +### `saturation_ceiling` + +`float64` + +## Capabilities + +### `tools` + +`bool` + +### `vision` + +`bool` + +### `embeddings` + +`bool` + +## Config + +### `node_id` + +`string` + +### `server` + +`string` + +Server is the control plane, and Servers is the same thing when there is +more than one. Both are accepted: every configuration written before +there could be a second control plane names one address, and none of +them should have to change. + +The order is tried in order, and a refusal moves to the next. Only the +leader accepts an agent, so a standby is skipped in the time it takes to +be told so — which is what makes failover a reconnect rather than an +edit to every node's configuration. + +### `servers` + +`[]string` + +### `hostname` + +`string` + +### `mgmt_addr` + +`string` + +### `labels` + +`map[string]string` + +### `mgmt_interface` + +`string` + +MgmtInterface is the ethernet interface NCCL bootstraps over. + +### `fabrics` + +`[]inventory.FabricAttachment` + +Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's +default subnet prefix is the same on every fabric, so nothing can be +inferred from the wire (plan §5.2). + +### `heartbeat_interval` + +`time.Duration` + +### `reconnect_base` + +`time.Duration` + +ReconnectBase is the first reconnect delay; it backs off from there. + +### `reconnect_max` + +`time.Duration` + +### `adopt_ready_timeout` + +`time.Duration` + +AdoptReadyTimeout bounds how long an adopted engine is given to answer a +health check before the agent stops waiting on it. + +Shorter than a startup timeout on purpose: this engine is already +running, so it is not loading a model, it is either serving or it is not. + +### `tls` + +`tlsconf.Config` + +TLS authenticates both ends of the control-plane stream. Leaving it unset +keeps the plaintext behaviour, which only belongs on a trusted network. + +Takes a [Config](#config) block. + +### `model_dirs` + +`[]string` + +ModelDirs are the directories this node keeps model weights in. + +Empty by default, and empty means the node is not asked to look — which +is not the same as having nothing. Storage differs at every installation, +so llmgw observes it rather than owning it: a Lustre mount, an image +exported over the fabric, local NVMe and a copy made by hand all look +identical from here, and placement only needs to know whether the file +is present. + +### `weights_cache` + +`string` + +WeightsCache is where a remote source is downloaded to. Defaults to the +first entry of ModelDirs, because a node that says where its models live +has already answered this question. + +Fetching is the simple arrangement: every node keeps its own copy. The +exotic ones need no code — weights on a shared filer or an image exported +over the fabric are simply present, and present means nothing is fetched. + +### `discovery` + +`discovery.Options` + +Takes a [Options](#options) block. + +### `engines` + +`engines.Config` + +Takes a [Config](#config) block. + +### `supervisor` + +`supervisor.Config` + +Takes a [Config](#config) block. + +## Control + +### `interval` + +`time.Duration` + +Interval is how often desired and actual state are compared. + +### `base_backoff` + +`time.Duration` + +BaseBackoff and MaxBackoff bound how fast a failing model is retried. + +### `max_backoff` + +`time.Duration` + +### `unreachable_after` + +`time.Duration` + +UnreachableAfter and LostAfter stage node expiry. They are deliberately +far apart: a network blip must not tear down replicas that still serve, +and rebuilding a multi-node replica costs tens of minutes of loading. + +### `lost_after` + +`time.Duration` + +### `port_range` + +`[]int` + +PortRange is the per-node range for engine HTTP and dist-init ports. + +### `mgmt_interface` + +`string` + +MgmtInterface is the ethernet interface NCCL bootstraps over. + +### `lease_ttl` + +`time.Duration` + +LeaseTTL is how long this control plane's claim to be the one placing +work stands without being renewed. + +Only meaningful with a shared store, where a second control plane can +see the claim. Long enough to survive a slow query or a brief network +stall, short enough that a standby takes over before anyone has finished +reading the alert. Zero takes the default. + +## Endpoint + +### `id` + +`string` + +### `url` + +`string` + +### `models` + +`[]string` + +### `engine` + +`string` + +Engine names the inference engine, which decides what its metrics are +called. Empty means vLLM's names. + +### `pool` + +`string` + +### `weight` + +`float64` + +### `priority` + +`int` + +## Landing + +### `contact` + +`string` + +Contact is where someone without a key should go — an address, a URL, a +channel name. Rendered as a link when it is one, as text otherwise. + +### `notice` + +`string` + +Notice is an optional line of your own, for whatever the reader needs to +know that this page cannot guess. + +## LlamaCppConfig + +### `bin` + +`string` + +Bin is the llama-server binary. It is the server, not the CLI: the +gateway talks OpenAI over HTTP and needs the one that serves it. + +### `parallel` + +`int` + +Parallel is how many requests a server handles at once, when a profile +does not say. llama.cpp defaults to one, which would make every second +request wait behind the first no matter what the router decided. + +## Model + +### `name` + +`string` + +### `aliases` + +`[]string` + +### `source` + +`string` + +Source is the weights path or a HuggingFace id. + +### `profiles` + +`[]Profile` + +### `scaling` + +`Scaling` + +Takes a [Scaling](#scaling) block. + +### `fallbacks` + +`[]string` + +### `routing_profile` + +`string` + +RoutingProfile selects how the router scores endpoints for this model. + +### `slo` + +`SLO` + +Takes a [SLO](#slo) block. + +### `context_length` + +`int` + +ContextLength is how many tokens one request may use, prompt and +completion together, and MaxOutput bounds the completion alone. Zero +means unknown, which is what every model was until now. + +The gateway knew neither, so an over-long prompt was not refused: it was +admitted, given a replica, and failed at the engine, and a client sizing +its prompts from /v1/models had nothing to read. Worse, the number is not +the one an operator writes in the engine's arguments — llama.cpp divides +--ctx-size across its parallel slots, so a pool of 8192 across four slots +is 2048 a request, and nothing said so. + +### `max_output` + +`int` + +### `capabilities` + +`Capabilities` + +Capabilities is what this model can do beyond plain completion. + +Takes a [Capabilities](#capabilities) block. + +## Options + +### `nvidia_smi` + +`string` + +NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH. +Worth configuring on nodes where the driver lives outside the default +paths, which containerized driver installs often do. + +### `ib_root` + +`string` + +IBRoot is the sysfs InfiniBand directory. + +### `prefer_nvml` + +`bool` + +PreferNVML uses the NVML backend when this binary has one. It is the only +source of NVLink topology. + +### `skip_gpus` + +`bool` + +SkipGPUs omits GPU discovery, for nodes that have none. + +## Prefix + +### `enabled` + +`bool` + +### `max_nodes` + +`int` + +MaxNodes bounds the tree, MinMatch is the shortest match worth acting on +in bytes, and TTL is how long an entry is believed — an engine evicts on +its own schedule, so an old record may describe a cache that is gone. + +### `min_match` + +`int` + +### `ttl` + +`time.Duration` + +### `match_ratio` + +`float64` + +MatchRatio is the share of the prompt that must match to be worth acting +on. BalanceAbs and BalanceRel are the imbalance guard, and both must trip +before affinity is abandoned. + +### `balance_abs` + +`float64` + +### `balance_rel` + +`float64` + +## Price + +### `prompt_per_1k` + +`int64` + +### `completion_per_1k` + +`int64` + +## Profile + +### `name` + +`string` + +### `engine` + +`string` + +### `gpu_class` + +`string` + +### `tp` + +`int` + +### `pp` + +`int` + +### `gpus_per_node` + +`int` + +GPUsPerNode defaults to TP for a single-node profile. + +### `nodes` + +`int` + +Nodes above 1 makes this a multi-node profile, which requires a fabric. + +### `args` + +`[]string` + +### `env` + +`map[string]string` + +### `min_memory_gb` + +`int` + +MinMemoryGB is the host RAM this profile needs to run. + +The number that matters is the model's working set: the weights, plus the +KV cache for the whole context pool, plus the engine's own overhead. For +a GPU profile that lives in the cards and this is small; for a CPU +profile it is the whole thing, and getting it wrong means an OOM on a +node that reported itself healthy. + +Zero means unconstrained, which is every profile that existed before +this. A node that does not report its memory is not excluded either — +an older agent must not have its nodes quietly emptied by an upgrade of +the control plane. + +### `min_disk_gb` + +`int` + +MinDiskGB is the free space a node needs before it can hold these +weights. + +Only checked when the node does not already have them: a node holding +the file needs no room for it, and requiring the space twice would keep +a model off the one node best placed to run it. + +Declared rather than derived, because the size of a model that has never +been fetched anywhere is not knowable from here — and a wrong guess +fails late, halfway through a download, which is the failure this exists +to prevent. Zero means unconstrained, and a node that reports no storage +is not excluded: unknown is not full. + +### `node_selector` + +`map[string]string` + +NodeSelector restricts this profile to nodes carrying every one of these +labels, with these values. + +A CPU profile otherwise fits everywhere, because there is no card to be +free — which is right for a 0.5B model and wrong for a 14B one. Nothing +in the inventory says how much memory a node has, so the operator says +which nodes are meant for a model instead of the scheduler inferring it. +Agents have reported labels since the beginning; this is what reads them. + +### `weight` + +`float64` + +Weight is the router's capacity weight. It should come from ctl bench +rather than a guess (plan §6.2.1). + +### `priority` + +`int` + +Priority orders profiles during placement; highest that fits wins. + +### `startup_timeout` + +`time.Duration` + +StartupTimeout bounds model loading. A 70B model across two nodes takes +tens of minutes, so this is per profile rather than global. + +## Router + +### `activation_wait` + +`time.Duration` + +ActivationWait is how long a request waits for an on-demand model to +load before the client is told to come back. Zero takes the default. + +The one real decision in on-demand loading. Refusing outright makes the +feature useless, since the first request for every model would fail; +waiting without a bound turns a request into a job, and a client that +budgeted thirty seconds ends up holding a connection for ten minutes. + +### `scrape_interval` + +`time.Duration` + +### `scrape_timeout` + +`time.Duration` + +### `stale_after` + +`time.Duration` + +StaleAfter is how long an endpoint may go unscraped before being scored +down (plan §7.6). + +### `retries` + +`int` + +Retries is how many *additional* endpoints may be tried after a failure, +and only while nothing has been written to the client (plan §7.7). + +### `cooldown` + +`time.Duration` + +Cooldown is how long a failing endpoint leaves the candidate pool. + +### `failure_threshold` + +`int` + +FailureThreshold is the consecutive failure count that opens a breaker. +1 turns the breaker into a pure cooldown. + +### `default_profile` + +`string` + +DefaultProfile names the routing profile used when a model picks none. + +### `prefix` + +`Prefix` + +Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the +router uses session affinity alone, which is tier 0 and already most of +the win for multi-turn traffic. + +Takes a [Prefix](#prefix) block. + +### `session_header` + +`string` + +SessionHeader carries the session id for affinity (plan §7.5 tier 0). + +### `inject_usage` + +`bool` + +InjectUsage asks engines for usage even when the client did not, so token +accounting works regardless of client behaviour. + +### `max_body_bytes` + +`int64` + +MaxBodyBytes bounds the request body the router will read. + +### `timeouts` + +`Timeouts` + +Takes a [Timeouts](#timeouts) block. + +### `scoring` + +`Scoring` + +Takes a [Scoring](#scoring) block. + +### `admission` + +`Admission` + +Takes a [Admission](#admission) block. + +## SGLangConfig + +### `python` + +`string` + +Python is the interpreter of the SGLang virtualenv. SGLang is launched as +a module rather than a console script, which is the documented entry +point and avoids depending on the script being on PATH. + +## SLO + +### `ttft_p99` + +`time.Duration` + +### `tpot_p99` + +`time.Duration` + +## Scaling + +### `min` + +`int` + +### `max` + +`int` + +### `idle_timeout` + +`time.Duration` + +IdleTimeout is how long an on-demand model may go unasked-for before its +replicas are stopped. Zero takes the cluster default. + +Minutes rather than seconds, always. Stopping a replica throws away the +KV cache prefix routing spent its effort filling, and a model that flaps +in and out loses that entirely — so the saving has to be worth more than +the cache, which it only is over a real idle period. + +## Scoring + +### `target_waiting` + +`float64` + +### `target_inflight` + +`float64` + +### `kv_ceiling` + +`float64` + +### `session_load_factor` + +`float64` + +## Server + +### `listen` + +`string` + +Listen is the client-facing OpenAI API address. + +### `admin_listen` + +`string` + +AdminListen serves the dashboard, the management API, health and pprof. + +Loopback by default, and it should stay there: pprof can dump the process +heap, and API keys are in that heap in plaintext. Every comment beside +this setting has always said not to expose it, while the default bound +every interface — a default that contradicts its own documentation is a +footgun, so the default now matches the advice. + +### `metrics_listen` + +`string` + +MetricsListen serves Prometheus metrics and nothing else. + +Separate from AdminListen because the reason that one is loopback is +pprof, not metrics — and metrics are the one thing on it that something +else genuinely needs to reach. Sharing a listener meant a fleet could +either be monitored or keep its heap private, which is not a choice +anybody should have to make. + +Empty means off, and metrics remain available on AdminListen either way, +so nothing that scrapes it today has to change. + +### `agent_listen` + +`string` + +AgentListen is where node agents connect. Separate from the client port +so the fleet-facing surface can be firewalled independently. + +### `auth` + +`Auth` + +Auth turns on API keys on the client-facing port. + +Takes a [Auth](#auth) block. + +### `threads` + +`Threads` + +Threads keeps conversations and agent memory without API keys, for a +gateway on one trusted machine. With auth on they live in the auth store, +per project, and this block is ignored. + +Takes a [Threads](#threads) block. + +### `landing` + +`Landing` + +Landing configures the page served at "/". + +Takes a [Landing](#landing) block. + +### `agent_tls` + +`tlsconf.Config` + +AgentTLS authenticates the fleet-facing port. It deliberately does not +cover the client-facing one: that surface is fronted by a publicly +trusted certificate, while this one is mutual and private. + +Takes a [Config](#config) block. + +### `router` + +`Router` + +Takes a [Router](#router) block. + +### `control` + +`Control` + +Takes a [Control](#control) block. + +### `models` + +`[]catalog.Model` + +### `endpoints` + +`[]Endpoint` + +Endpoints is the phase-1 static replica list. It is only consulted when +no model declares a profile, which is what makes a router-only smoke test +against sim-engines possible without a control plane. + +## SimConfig + +### `bin` + +`string` + +Bin is the llmgw binary itself; the engine is one of its subcommands. +Empty means whatever started this agent, which is right almost always. + +## Threads + +### `store` + +`string` + +Store is the SQLite file. Empty means nothing is kept. + +### `retain_hours` + +`int` + +RetainHours is how long idle conversations and notes are kept. Required +with Store: a store with no expiry keeps everything anyone typed, by +omission. + +## Timeouts + +### `dial` + +`time.Duration` + +### `response_header` + +`time.Duration` + +ResponseHeader bounds the wait for headers on a streaming request. It is +a liveness check there, because headers arrive before generation starts. + +### `generation` + +`time.Duration` + +Generation bounds a non-streaming request, where the engine sends no +headers until the completion is done. Setting response_header alone used +to cap both, so 60s read as a liveness guard and behaved as a limit of +about a hundred tokens on a slow model. + +### `first_token` + +`time.Duration` + +### `idle` + +`time.Duration` + +### `total` + +`time.Duration` + +## VLLMConfig + +### `bin` + +`string` + +Bin is the vllm executable. + +### `launch_mode` + +`string` + +LaunchMode selects the multi-node mechanism: "ray" or "mp". + +Which one works depends on the vLLM version, and the flags have changed +more than once (plan §14). Both paths exist so a cluster can pick the one +its version supports instead of waiting for a code change. + +### `ray_bin` + +`string` + +RayBin is used when LaunchMode is "ray". + diff --git a/internal/docgen/docgen.go b/internal/docgen/docgen.go new file mode 100644 index 00000000..03f3f8fe --- /dev/null +++ b/internal/docgen/docgen.go @@ -0,0 +1,158 @@ +// Package docgen turns the configuration structs into reference documentation. +// +// Generated rather than written, because a hand-written reference is a promise +// that someone will remember. This project added six settings in a fortnight — +// store, servers, lease_ttl, thread and memory retention, min_disk_gb — and +// every one of them reached an operator only through a narrative document that +// happened to mention it. A reference nobody maintains is worse than none: it +// is the same list, quietly missing the setting you came to look up. +// +// Derived from the source, so the only way for it to be wrong is for the code +// to be wrong. A test regenerates it and fails when it has drifted, which is +// what keeps that true after everyone has forgotten this exists. +package docgen + +//go:generate go run ./gen + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "reflect" + "sort" + "strings" +) + +// Field is one configuration key. +type Field struct { + Key string + Type string + Doc string + // Nested names the struct this key expands into, empty for a leaf. It is + // what lets the reference say "see Auth" rather than printing the same + // twenty keys under four parents. + Nested string +} + +// Block is a struct that appears in configuration. +type Block struct { + Name string + Doc string + Fields []Field +} + +// Collect reads koanf-tagged structs out of Go source files. +func Collect(files []string) ([]Block, error) { + fset := token.NewFileSet() + var blocks []Block + seen := map[string]bool{} + + for _, path := range files { + f, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + ast.Inspect(f, func(n ast.Node) bool { + ts, ok := n.(*ast.TypeSpec) + if !ok { + return true + } + st, ok := ts.Type.(*ast.StructType) + if !ok { + return true + } + b, ok := blockOf(ts, st) + if !ok || seen[b.Name] { + return true + } + seen[b.Name] = true + blocks = append(blocks, b) + return true + }) + } + sort.Slice(blocks, func(i, j int) bool { return blocks[i].Name < blocks[j].Name }) + return blocks, nil +} + +// blockOf builds a Block, or reports that this struct carries no settings. +func blockOf(ts *ast.TypeSpec, st *ast.StructType) (Block, bool) { + b := Block{Name: ts.Name.Name, Doc: docText(ts.Doc)} + for _, f := range st.Fields.List { + if f.Tag == nil { + continue + } + tag := reflect.StructTag(strings.Trim(f.Tag.Value, "`")) + key, ok := tag.Lookup("koanf") + // "-" is a field carried in the struct but never read from a file; + // listing it would send someone looking for a setting that does not + // exist. + if !ok || key == "" || key == "-" { + continue + } + typ, nested := typeOf(f.Type) + b.Fields = append(b.Fields, Field{ + Key: key, Type: typ, Doc: docText(f.Doc), Nested: nested, + }) + } + return b, len(b.Fields) > 0 +} + +// typeOf renders a field's type, and names the struct it expands into. +func typeOf(e ast.Expr) (rendered, nested string) { + switch t := e.(type) { + case *ast.Ident: + if isNamedStruct(t.Name) { + return t.Name, t.Name + } + return t.Name, "" + case *ast.SelectorExpr: + // A type from another package, such as tlsconf.Config. + name := t.Sel.Name + if pkg, ok := t.X.(*ast.Ident); ok { + return pkg.Name + "." + name, name + } + return name, name + case *ast.ArrayType: + inner, _ := typeOf(t.Elt) + return "[]" + inner, "" + case *ast.MapType: + k, _ := typeOf(t.Key) + v, _ := typeOf(t.Value) + return "map[" + k + "]" + v, "" + case *ast.StarExpr: + inner, n := typeOf(t.X) + return "*" + inner, n + default: + return "value", "" + } +} + +// isNamedStruct guesses whether an identifier names a configuration block. +// +// By capitalisation, because the alternative is resolving types across +// packages for a document. A wrong guess costs a "see also" that points at +// nothing, which the renderer drops. +func isNamedStruct(name string) bool { + if name == "" { + return false + } + switch name { + case "string", "bool", "int", "int64", "float64", "any", "byte", "rune": + return false + } + return name[0] >= 'A' && name[0] <= 'Z' +} + +// docText flattens a comment group into one paragraph per blank line. +func docText(g *ast.CommentGroup) string { + if g == nil { + return "" + } + var lines []string + for _, c := range g.List { + t := strings.TrimPrefix(c.Text, "//") + lines = append(lines, strings.TrimPrefix(t, " ")) + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} diff --git a/internal/docgen/docgen_test.go b/internal/docgen/docgen_test.go new file mode 100644 index 00000000..2da93e3f --- /dev/null +++ b/internal/docgen/docgen_test.go @@ -0,0 +1,138 @@ +package docgen + +import ( + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" +) + +// repoRoot is where the source paths in Sources are relative to. +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate this test") + } + return filepath.Join(filepath.Dir(file), "..", "..") +} + +func sourcePaths(t *testing.T) []string { + t.Helper() + root := repoRoot(t) + out := make([]string, len(Sources)) + for i, s := range Sources { + out[i] = filepath.Join(root, s) + } + return out +} + +func TestTheReferenceIsNotStale(t *testing.T) { + // The whole point. A generated document that nobody regenerates is the + // same lie as a hand-written one, arrived at more slowly: it describes the + // configuration of some earlier week, and the setting somebody came to + // look up is the one that is missing. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + want := Render(blocks) + + path := filepath.Join(repoRoot(t), "docs", "reference.md") + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("docs/reference.md is missing; run go generate ./...: %v", err) + } + if string(got) != want { + t.Error("docs/reference.md no longer matches the configuration structs.\n" + + "Run: go generate ./...") + } +} + +func TestEveryConfigurationFileIsListed(t *testing.T) { + // Sources is a list, and a list is a thing to forget. A new file full of + // settings would otherwise be absent from the reference with nothing + // saying so — which is exactly the failure the reference exists to fix. + root := repoRoot(t) + var tagged []string + err := filepath.Walk(filepath.Join(root, "internal"), func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") { + return err + } + b, err := os.ReadFile(p) + if err != nil { + return err + } + if !strings.Contains(string(b), `koanf:"`) { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + tagged = append(tagged, rel) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(tagged) == 0 { + t.Fatal("found no configuration files at all, so this test proves nothing") + } + + for _, f := range tagged { + if slices.Contains(Sources, f) { + continue + } + // Deliberately out: these hold koanf tags for structs an operator + // never writes — inventory and catalog entries come from the API or + // from a model file, not from the configuration. + if slices.Contains(notReference, f) { + continue + } + t.Errorf("%s declares configuration and is not in Sources or notReference", f) + } +} + +func TestADocumentedKeyKeepsItsComment(t *testing.T) { + // The descriptions are the field comments, which is what makes the + // reference worth reading rather than a list of names. A field whose + // comment was the only explanation anywhere must not render blank. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + var lease Field + for _, b := range blocks { + for _, f := range b.Fields { + if f.Key == "lease_ttl" { + lease = f + } + } + } + if lease.Key == "" { + t.Fatal("lease_ttl is not in the reference") + } + if !strings.Contains(lease.Doc, "shared store") { + t.Errorf("lease_ttl lost its explanation: %q", lease.Doc) + } +} + +func TestAFieldMarkedNotConfigurableIsLeftOut(t *testing.T) { + // koanf:"-" means the field is carried in the struct and never read from + // a file. Listing it would send somebody looking for a setting that does + // not exist. + blocks, err := Collect(sourcePaths(t)) + if err != nil { + t.Fatal(err) + } + for _, b := range blocks { + for _, f := range b.Fields { + if f.Key == "-" || f.Key == "" { + t.Errorf("%s lists a non-configurable field: %+v", b.Name, f) + } + } + } +} diff --git a/internal/docgen/gen/main.go b/internal/docgen/gen/main.go new file mode 100644 index 00000000..65d6efa9 --- /dev/null +++ b/internal/docgen/gen/main.go @@ -0,0 +1,42 @@ +// Command gen writes docs/reference.md from the configuration structs. +// +// Run by `go generate ./...`, and checked by a test, so the reference cannot +// quietly stop describing the code. +package main + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" +) + +func main() { + // The repository root, found from this file rather than from the working + // directory, so `go generate ./...` produces the same document wherever + // it is run from. + _, self, _, ok := runtime.Caller(0) + if !ok { + fmt.Fprintln(os.Stderr, "error: cannot locate the generator") + os.Exit(1) + } + root := filepath.Join(filepath.Dir(self), "..", "..", "..") + + paths := make([]string, len(docgen.Sources)) + for i, s := range docgen.Sources { + paths[i] = filepath.Join(root, s) + } + + blocks, err := docgen.Collect(paths) + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + if err := os.WriteFile(filepath.Join(root, "docs", "reference.md"), []byte(docgen.Render(blocks)), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + fmt.Printf("docs/reference.md: %d blocks\n", len(blocks)) +} diff --git a/internal/docgen/render.go b/internal/docgen/render.go new file mode 100644 index 00000000..6d00117a --- /dev/null +++ b/internal/docgen/render.go @@ -0,0 +1,99 @@ +package docgen + +import ( + "fmt" + "strings" +) + +// Sources are the files configuration is declared in. +// +// Listed rather than discovered, because "every file with a koanf tag" would +// sweep in structs that are configuration in the Go sense and not in the +// operator's — a scoring weight built at wiring time is not something anyone +// writes in a file. A test checks the list against what is actually tagged, so +// a new configuration file is a failure here rather than a silent omission. +var Sources = []string{ + "internal/config/config.go", + "internal/agent/agent.go", + "internal/agent/discovery/discover.go", + "internal/agent/engines/engine.go", + "internal/agent/engines/llamacpp.go", + "internal/agent/engines/vllm.go", + "internal/agent/engines/sglang.go", + "internal/agent/engines/sim.go", + "internal/agent/supervisor/supervisor.go", + "internal/tlsconf/tlsconf.go", + "internal/control/catalog/catalog.go", + "internal/router/admission/controller.go", + "internal/accounting/cost.go", +} + +// notReference are files with koanf tags that an operator never writes. +// +// Inventory and catalog entries carry them so the same structs can be decoded +// from an API payload or a model file; putting them in a configuration +// reference would invite somebody to write them into server.yaml. +var notReference = []string{ + "internal/control/inventory/inventory.go", +} + +// Render turns collected blocks into the reference document. +func Render(blocks []Block) string { + var b strings.Builder + b.WriteString(`# Configuration reference + +Every setting llmgw reads from a file, with the type it takes and what it is +for. + +**Generated from the source.** The descriptions are the comments on the fields +themselves, so this cannot describe a setting that does not exist or miss one +that does — and a test fails when it has drifted. Do not edit it; edit the +comment and run ` + "`go generate ./...`" + `. + +Where a setting takes another block, the block is listed under its own heading +rather than repeated. + +`) + + // An index first: the reason to open a reference is usually to find one + // key, and scanning thirteen headings for it is the thing this replaces. + b.WriteString("## Every key\n\n| Key | Type | In |\n|---|---|---|\n") + for _, bl := range blocks { + for _, f := range bl.Fields { + b.WriteString(fmt.Sprintf("| `%s` | %s | [%s](#%s) |\n", + f.Key, code(f.Type), bl.Name, anchor(bl.Name))) + } + } + b.WriteString("\n") + + for _, bl := range blocks { + fmt.Fprintf(&b, "## %s\n\n", bl.Name) + if bl.Doc != "" { + b.WriteString(bl.Doc + "\n\n") + } + for _, f := range bl.Fields { + fmt.Fprintf(&b, "### `%s`\n\n%s\n\n", f.Key, code(f.Type)) + if f.Doc != "" { + b.WriteString(f.Doc + "\n\n") + } + if f.Nested != "" && hasBlock(blocks, f.Nested) { + fmt.Fprintf(&b, "Takes a [%s](#%s) block.\n\n", f.Nested, anchor(f.Nested)) + } + } + } + return b.String() +} + +func hasBlock(blocks []Block, name string) bool { + for _, b := range blocks { + if b.Name == name { + return true + } + } + return false +} + +func code(s string) string { return "`" + s + "`" } + +// anchor is how GitHub and Gitea slug a heading. +func anchor(s string) string { return strings.ToLower(s) }