Files
llmgw/internal/docgen/render.go
T
Ilker ManapandClaude Opus 5.5 f4b34dbf11
CI / test (push) Successful in 9m37s
FreeToken, an engine for models larger than the card
FreeToken (FlashML-org/FreeToken) serves mixture-of-experts models several
times a card's size from one card, experts in RAM, from Hugging Face
safetensors checkpoints. Installed like vLLM, from PyPI by uv, on every node
it runs on: compute capability 8.0+, driver 580+, x86-64, and the CUDA 13
toolkit it compiles kernels with. Older cards are passed over: on cuda01's
Quadro RTX 6000 (7.5) 0.2.1 had no kernel image for the first prefill, and
in fp16 hit an illegal memory access on its first request.

ft serve gets the model, address, card and served name; the key goes in
FREETOKEN_API_KEY. Flags that read files or fetch are refused, as are
FREETOKEN_ variables naming paths or the agent's own settings. ctl model
context sets --max-seq-len-override.

Readiness now also waits for an engine whose /health answers 200 while its
body says "loading", as FreeToken's does from the moment its HTTP server is up.

FreeToken is followed release by release: a weekly workflow fails when PyPI
has a version newer than DefaultEngines installs (scripts/engine_releases.py).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-31 22:10:05 +01:00

140 lines
4.8 KiB
Go

package docgen
import (
"fmt"
"strings"
)
// Sources are the files configuration is declared in.
//
// Listed rather than discovered, because "every file with a koanf tag" would
// sweep in structs that are configuration in the Go sense and not in the
// operator's — a scoring weight built at wiring time is not something anyone
// writes in a file. A test checks the list against what is actually tagged, so
// a new configuration file is a failure here rather than a silent omission.
var Sources = []string{
"internal/config/config.go",
"internal/agent/config.go",
"internal/agent/discovery/discover.go",
"internal/agent/engines/engine.go",
"internal/agent/engines/llamacpp.go",
"internal/agent/engines/vllm.go",
"internal/agent/engines/sglang.go",
"internal/agent/engines/trtllm.go",
"internal/agent/engines/sim.go",
"internal/agent/engines/comfyui.go",
"internal/agent/engines/strata.go",
"internal/agent/engines/freetoken.go",
"internal/agent/supervisor/supervisor.go",
"internal/tlsconf/tlsconf.go",
"internal/control/catalog/catalog.go",
"internal/control/catalog/workflow.go",
"internal/control/catalog/hardware.go",
"internal/router/admission/controller.go",
"internal/accounting/cost.go",
"internal/guard/policy.go",
"internal/guard/service.go",
"internal/guard/judge.go",
"internal/config/oidc.go",
"internal/config/ldap.go",
"internal/config/firewall.go",
"internal/config/serving.go",
"internal/config/presets.go",
"internal/config/ondemand.go",
"internal/config/agents.go",
"internal/mail/mail.go",
"internal/notify/notify.go",
"internal/secrets/secrets.go",
"internal/obs/trace.go",
}
// notReference are files with koanf tags that an operator never writes.
//
// Inventory and catalog entries carry them so the same structs can be decoded
// from an API payload or a model file; putting them in a configuration
// reference would invite somebody to write them into server.yaml.
var notReference = []string{
"internal/control/inventory/inventory.go",
}
// Render turns collected blocks into the reference document.
func Render(blocks []Block) string {
var b strings.Builder
b.WriteString(`# Configuration reference
Every setting llmgw reads from a file, with the type it takes and what it is
for.
**Generated from the source.** The descriptions are the comments on the fields
themselves, so this cannot describe a setting that does not exist or miss one
that does — and a test fails when it has drifted. Do not edit it; edit the
comment and run ` + "`go generate ./...`" + `.
Where a setting takes another block, the block is listed under its own heading
rather than repeated.
`)
// An index first: the reason to open a reference is usually to find one
// key, and scanning thirteen headings for it is the thing this replaces.
b.WriteString("## Every key\n\n| Key | Type | In |\n|---|---|---|\n")
for _, bl := range blocks {
for _, f := range bl.Fields {
b.WriteString(fmt.Sprintf("| `%s` | %s | [%s](#%s) |\n",
f.Key, code(f.Type), bl.Title, anchor(bl.Title)))
}
}
b.WriteString("\n")
for _, bl := range blocks {
fmt.Fprintf(&b, "## %s\n\n", bl.Title)
if bl.Doc != "" {
b.WriteString(bl.Doc + "\n\n")
}
for _, f := range bl.Fields {
fmt.Fprintf(&b, "### `%s`\n\n%s\n\n", f.Key, code(f.Type))
if f.Doc != "" {
b.WriteString(f.Doc + "\n\n")
}
if nb, ok := Find(blocks, f.Nested); ok {
fmt.Fprintf(&b, "Takes a [%s](#%s) block.\n\n", nb.Title, anchor(nb.Title))
}
}
}
return b.String()
}
// Find is the block with an ID, as a Field's Nested names it.
func Find(blocks []Block, id string) (Block, bool) {
for _, b := range blocks {
if id != "" && b.ID() == id {
return b, true
}
}
return Block{}, false
}
func code(s string) string { return "`" + s + "`" }
// anchor is how GitHub and Gitea slug a heading: lower case, and punctuation
// such as the dot in tlsconf.Config dropped.
func anchor(s string) string { return strings.ToLower(strings.ReplaceAll(s, ".", "")) }
// RenderGo writes the blocks as Go, for the settings screens: every key, its
// type and what it does, from the same source as the reference, so a setting
// added to the code is on the screens without anyone listing it.
func RenderGo(blocks []Block) string {
var b strings.Builder
b.WriteString("// Code generated by go run ./internal/docgen/gen; DO NOT EDIT.\n\npackage docgen\n\n")
b.WriteString("// Reference is every configuration block, as Collect reads them.\nvar Reference = []Block{\n")
for _, bl := range blocks {
fmt.Fprintf(&b, "\t{Package: %q, Name: %q, Title: %q, Doc: %q, Fields: []Field{\n", bl.Package, bl.Name, bl.Title, bl.Doc)
for _, f := range bl.Fields {
fmt.Fprintf(&b, "\t\t{Key: %q, Type: %q, Doc: %q, Nested: %q},\n", f.Key, f.Type, f.Doc, f.Nested)
}
b.WriteString("\t}},\n")
}
b.WriteString("}\n")
return b.String()
}