Generate the configuration reference, and fail when it goes stale
There was no reference. Four narrative documents explain how to install and operate llmgw, and the only way to learn that `lease_ttl` exists was to read the one that happened to mention it — or the struct. Written by hand it would have been wrong within a fortnight. Six settings arrived in the last two weeks alone: store, servers, lease_ttl, the two retention windows, min_disk_gb. A reference nobody maintains is worse than none, because it is the same list with the setting you came to look up quietly missing from it. So it is derived from the structs, with the field comments as the descriptions — which means the only way for it to be wrong is for the code to be wrong, and it makes the comments worth writing twice over. 1163 lines, 24 blocks, 87 keys. Two tests keep that true after everyone has forgotten this exists. One regenerates and compares, so adding a setting without regenerating is a failure rather than an omission — checked by adding one and watching it fail. The other walks the tree for koanf tags and fails on a configuration file that is in neither the source list nor the short list of structs an operator never writes, because a list is a thing to forget. The generator finds the repository from its own path rather than the working directory, so `go generate ./...` produces the same document from anywhere. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed
1 parent
8e57171c4f
commit
100f10ac17
7 files changed
+1614
No files matched your search
@@ -52,6 +52,17 @@ without a fabric. The InfiniBand, NVML and NCCL paths are written against the
|
||||
formats and failure modes those tools actually have, but they have not yet run
|
||||
on a machine that has them.
|
||||
|
||||
## Documentation
|
||||
|
||||
- [docs/install.md](docs/install.md) — installing from nothing: control plane,
|
||||
nodes, engines, weights, first request
|
||||
- [docs/quickstart.md](docs/quickstart.md) — the short path
|
||||
- [docs/reference.md](docs/reference.md) — every configuration key, generated
|
||||
from the structs and checked by a test, so it cannot describe a setting that
|
||||
does not exist or miss one that does
|
||||
- [docs/node-onboarding.md](docs/node-onboarding.md),
|
||||
[docs/onboarding.md](docs/onboarding.md) — adding machines, and adding people
|
||||
|
||||
## Building
|
||||
|
||||
```sh
|
||||
|
||||
@@ -774,6 +774,9 @@ requests. On a slow model, set `timeouts.generation` higher.
|
||||
|
||||
## What to read next
|
||||
|
||||
- `docs/reference.md` — every configuration key, generated from the source, so
|
||||
it describes the version you are running rather than the one somebody last
|
||||
wrote about
|
||||
- `docs/node-onboarding.md` — adding nodes in more detail, and what onboarding
|
||||
deliberately does not do
|
||||
- `docs/onboarding.md` — giving people accounts and keys
|
||||
|
||||
+1163
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,158 @@
|
||||
// Package docgen turns the configuration structs into reference documentation.
|
||||
//
|
||||
// Generated rather than written, because a hand-written reference is a promise
|
||||
// that someone will remember. This project added six settings in a fortnight —
|
||||
// store, servers, lease_ttl, thread and memory retention, min_disk_gb — and
|
||||
// every one of them reached an operator only through a narrative document that
|
||||
// happened to mention it. A reference nobody maintains is worse than none: it
|
||||
// is the same list, quietly missing the setting you came to look up.
|
||||
//
|
||||
// Derived from the source, so the only way for it to be wrong is for the code
|
||||
// to be wrong. A test regenerates it and fails when it has drifted, which is
|
||||
// what keeps that true after everyone has forgotten this exists.
|
||||
package docgen
|
||||
|
||||
//go:generate go run ./gen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Field is one configuration key.
|
||||
type Field struct {
|
||||
Key string
|
||||
Type string
|
||||
Doc string
|
||||
// Nested names the struct this key expands into, empty for a leaf. It is
|
||||
// what lets the reference say "see Auth" rather than printing the same
|
||||
// twenty keys under four parents.
|
||||
Nested string
|
||||
}
|
||||
|
||||
// Block is a struct that appears in configuration.
|
||||
type Block struct {
|
||||
Name string
|
||||
Doc string
|
||||
Fields []Field
|
||||
}
|
||||
|
||||
// Collect reads koanf-tagged structs out of Go source files.
|
||||
func Collect(files []string) ([]Block, error) {
|
||||
fset := token.NewFileSet()
|
||||
var blocks []Block
|
||||
seen := map[string]bool{}
|
||||
|
||||
for _, path := range files {
|
||||
f, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing %s: %w", path, err)
|
||||
}
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
ts, ok := n.(*ast.TypeSpec)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
st, ok := ts.Type.(*ast.StructType)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
b, ok := blockOf(ts, st)
|
||||
if !ok || seen[b.Name] {
|
||||
return true
|
||||
}
|
||||
seen[b.Name] = true
|
||||
blocks = append(blocks, b)
|
||||
return true
|
||||
})
|
||||
}
|
||||
sort.Slice(blocks, func(i, j int) bool { return blocks[i].Name < blocks[j].Name })
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// blockOf builds a Block, or reports that this struct carries no settings.
|
||||
func blockOf(ts *ast.TypeSpec, st *ast.StructType) (Block, bool) {
|
||||
b := Block{Name: ts.Name.Name, Doc: docText(ts.Doc)}
|
||||
for _, f := range st.Fields.List {
|
||||
if f.Tag == nil {
|
||||
continue
|
||||
}
|
||||
tag := reflect.StructTag(strings.Trim(f.Tag.Value, "`"))
|
||||
key, ok := tag.Lookup("koanf")
|
||||
// "-" is a field carried in the struct but never read from a file;
|
||||
// listing it would send someone looking for a setting that does not
|
||||
// exist.
|
||||
if !ok || key == "" || key == "-" {
|
||||
continue
|
||||
}
|
||||
typ, nested := typeOf(f.Type)
|
||||
b.Fields = append(b.Fields, Field{
|
||||
Key: key, Type: typ, Doc: docText(f.Doc), Nested: nested,
|
||||
})
|
||||
}
|
||||
return b, len(b.Fields) > 0
|
||||
}
|
||||
|
||||
// typeOf renders a field's type, and names the struct it expands into.
|
||||
func typeOf(e ast.Expr) (rendered, nested string) {
|
||||
switch t := e.(type) {
|
||||
case *ast.Ident:
|
||||
if isNamedStruct(t.Name) {
|
||||
return t.Name, t.Name
|
||||
}
|
||||
return t.Name, ""
|
||||
case *ast.SelectorExpr:
|
||||
// A type from another package, such as tlsconf.Config.
|
||||
name := t.Sel.Name
|
||||
if pkg, ok := t.X.(*ast.Ident); ok {
|
||||
return pkg.Name + "." + name, name
|
||||
}
|
||||
return name, name
|
||||
case *ast.ArrayType:
|
||||
inner, _ := typeOf(t.Elt)
|
||||
return "[]" + inner, ""
|
||||
case *ast.MapType:
|
||||
k, _ := typeOf(t.Key)
|
||||
v, _ := typeOf(t.Value)
|
||||
return "map[" + k + "]" + v, ""
|
||||
case *ast.StarExpr:
|
||||
inner, n := typeOf(t.X)
|
||||
return "*" + inner, n
|
||||
default:
|
||||
return "value", ""
|
||||
}
|
||||
}
|
||||
|
||||
// isNamedStruct guesses whether an identifier names a configuration block.
|
||||
//
|
||||
// By capitalisation, because the alternative is resolving types across
|
||||
// packages for a document. A wrong guess costs a "see also" that points at
|
||||
// nothing, which the renderer drops.
|
||||
func isNamedStruct(name string) bool {
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
switch name {
|
||||
case "string", "bool", "int", "int64", "float64", "any", "byte", "rune":
|
||||
return false
|
||||
}
|
||||
return name[0] >= 'A' && name[0] <= 'Z'
|
||||
}
|
||||
|
||||
// docText flattens a comment group into one paragraph per blank line.
|
||||
func docText(g *ast.CommentGroup) string {
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
var lines []string
|
||||
for _, c := range g.List {
|
||||
t := strings.TrimPrefix(c.Text, "//")
|
||||
lines = append(lines, strings.TrimPrefix(t, " "))
|
||||
}
|
||||
return strings.TrimSpace(strings.Join(lines, "\n"))
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package docgen
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// repoRoot is where the source paths in Sources are relative to.
|
||||
func repoRoot(t *testing.T) string {
|
||||
t.Helper()
|
||||
_, file, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
t.Fatal("cannot locate this test")
|
||||
}
|
||||
return filepath.Join(filepath.Dir(file), "..", "..")
|
||||
}
|
||||
|
||||
func sourcePaths(t *testing.T) []string {
|
||||
t.Helper()
|
||||
root := repoRoot(t)
|
||||
out := make([]string, len(Sources))
|
||||
for i, s := range Sources {
|
||||
out[i] = filepath.Join(root, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTheReferenceIsNotStale(t *testing.T) {
|
||||
// The whole point. A generated document that nobody regenerates is the
|
||||
// same lie as a hand-written one, arrived at more slowly: it describes the
|
||||
// configuration of some earlier week, and the setting somebody came to
|
||||
// look up is the one that is missing.
|
||||
blocks, err := Collect(sourcePaths(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := Render(blocks)
|
||||
|
||||
path := filepath.Join(repoRoot(t), "docs", "reference.md")
|
||||
got, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("docs/reference.md is missing; run go generate ./...: %v", err)
|
||||
}
|
||||
if string(got) != want {
|
||||
t.Error("docs/reference.md no longer matches the configuration structs.\n" +
|
||||
"Run: go generate ./...")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryConfigurationFileIsListed(t *testing.T) {
|
||||
// Sources is a list, and a list is a thing to forget. A new file full of
|
||||
// settings would otherwise be absent from the reference with nothing
|
||||
// saying so — which is exactly the failure the reference exists to fix.
|
||||
root := repoRoot(t)
|
||||
var tagged []string
|
||||
err := filepath.Walk(filepath.Join(root, "internal"), func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() || !strings.HasSuffix(p, ".go") || strings.HasSuffix(p, "_test.go") {
|
||||
return err
|
||||
}
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !strings.Contains(string(b), `koanf:"`) {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tagged = append(tagged, rel)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(tagged) == 0 {
|
||||
t.Fatal("found no configuration files at all, so this test proves nothing")
|
||||
}
|
||||
|
||||
for _, f := range tagged {
|
||||
if slices.Contains(Sources, f) {
|
||||
continue
|
||||
}
|
||||
// Deliberately out: these hold koanf tags for structs an operator
|
||||
// never writes — inventory and catalog entries come from the API or
|
||||
// from a model file, not from the configuration.
|
||||
if slices.Contains(notReference, f) {
|
||||
continue
|
||||
}
|
||||
t.Errorf("%s declares configuration and is not in Sources or notReference", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADocumentedKeyKeepsItsComment(t *testing.T) {
|
||||
// The descriptions are the field comments, which is what makes the
|
||||
// reference worth reading rather than a list of names. A field whose
|
||||
// comment was the only explanation anywhere must not render blank.
|
||||
blocks, err := Collect(sourcePaths(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lease Field
|
||||
for _, b := range blocks {
|
||||
for _, f := range b.Fields {
|
||||
if f.Key == "lease_ttl" {
|
||||
lease = f
|
||||
}
|
||||
}
|
||||
}
|
||||
if lease.Key == "" {
|
||||
t.Fatal("lease_ttl is not in the reference")
|
||||
}
|
||||
if !strings.Contains(lease.Doc, "shared store") {
|
||||
t.Errorf("lease_ttl lost its explanation: %q", lease.Doc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFieldMarkedNotConfigurableIsLeftOut(t *testing.T) {
|
||||
// koanf:"-" means the field is carried in the struct and never read from
|
||||
// a file. Listing it would send somebody looking for a setting that does
|
||||
// not exist.
|
||||
blocks, err := Collect(sourcePaths(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, b := range blocks {
|
||||
for _, f := range b.Fields {
|
||||
if f.Key == "-" || f.Key == "" {
|
||||
t.Errorf("%s lists a non-configurable field: %+v", b.Name, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Command gen writes docs/reference.md from the configuration structs.
|
||||
//
|
||||
// Run by `go generate ./...`, and checked by a test, so the reference cannot
|
||||
// quietly stop describing the code.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// The repository root, found from this file rather than from the working
|
||||
// directory, so `go generate ./...` produces the same document wherever
|
||||
// it is run from.
|
||||
_, self, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
fmt.Fprintln(os.Stderr, "error: cannot locate the generator")
|
||||
os.Exit(1)
|
||||
}
|
||||
root := filepath.Join(filepath.Dir(self), "..", "..", "..")
|
||||
|
||||
paths := make([]string, len(docgen.Sources))
|
||||
for i, s := range docgen.Sources {
|
||||
paths[i] = filepath.Join(root, s)
|
||||
}
|
||||
|
||||
blocks, err := docgen.Collect(paths)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "docs", "reference.md"), []byte(docgen.Render(blocks)), 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Printf("docs/reference.md: %d blocks\n", len(blocks))
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package docgen
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Sources are the files configuration is declared in.
|
||||
//
|
||||
// Listed rather than discovered, because "every file with a koanf tag" would
|
||||
// sweep in structs that are configuration in the Go sense and not in the
|
||||
// operator's — a scoring weight built at wiring time is not something anyone
|
||||
// writes in a file. A test checks the list against what is actually tagged, so
|
||||
// a new configuration file is a failure here rather than a silent omission.
|
||||
var Sources = []string{
|
||||
"internal/config/config.go",
|
||||
"internal/agent/agent.go",
|
||||
"internal/agent/discovery/discover.go",
|
||||
"internal/agent/engines/engine.go",
|
||||
"internal/agent/engines/llamacpp.go",
|
||||
"internal/agent/engines/vllm.go",
|
||||
"internal/agent/engines/sglang.go",
|
||||
"internal/agent/engines/sim.go",
|
||||
"internal/agent/supervisor/supervisor.go",
|
||||
"internal/tlsconf/tlsconf.go",
|
||||
"internal/control/catalog/catalog.go",
|
||||
"internal/router/admission/controller.go",
|
||||
"internal/accounting/cost.go",
|
||||
}
|
||||
|
||||
// notReference are files with koanf tags that an operator never writes.
|
||||
//
|
||||
// Inventory and catalog entries carry them so the same structs can be decoded
|
||||
// from an API payload or a model file; putting them in a configuration
|
||||
// reference would invite somebody to write them into server.yaml.
|
||||
var notReference = []string{
|
||||
"internal/control/inventory/inventory.go",
|
||||
}
|
||||
|
||||
// Render turns collected blocks into the reference document.
|
||||
func Render(blocks []Block) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(`# Configuration reference
|
||||
|
||||
Every setting llmgw reads from a file, with the type it takes and what it is
|
||||
for.
|
||||
|
||||
**Generated from the source.** The descriptions are the comments on the fields
|
||||
themselves, so this cannot describe a setting that does not exist or miss one
|
||||
that does — and a test fails when it has drifted. Do not edit it; edit the
|
||||
comment and run ` + "`go generate ./...`" + `.
|
||||
|
||||
Where a setting takes another block, the block is listed under its own heading
|
||||
rather than repeated.
|
||||
|
||||
`)
|
||||
|
||||
// An index first: the reason to open a reference is usually to find one
|
||||
// key, and scanning thirteen headings for it is the thing this replaces.
|
||||
b.WriteString("## Every key\n\n| Key | Type | In |\n|---|---|---|\n")
|
||||
for _, bl := range blocks {
|
||||
for _, f := range bl.Fields {
|
||||
b.WriteString(fmt.Sprintf("| `%s` | %s | [%s](#%s) |\n",
|
||||
f.Key, code(f.Type), bl.Name, anchor(bl.Name)))
|
||||
}
|
||||
}
|
||||
b.WriteString("\n")
|
||||
|
||||
for _, bl := range blocks {
|
||||
fmt.Fprintf(&b, "## %s\n\n", bl.Name)
|
||||
if bl.Doc != "" {
|
||||
b.WriteString(bl.Doc + "\n\n")
|
||||
}
|
||||
for _, f := range bl.Fields {
|
||||
fmt.Fprintf(&b, "### `%s`\n\n%s\n\n", f.Key, code(f.Type))
|
||||
if f.Doc != "" {
|
||||
b.WriteString(f.Doc + "\n\n")
|
||||
}
|
||||
if f.Nested != "" && hasBlock(blocks, f.Nested) {
|
||||
fmt.Fprintf(&b, "Takes a [%s](#%s) block.\n\n", f.Nested, anchor(f.Nested))
|
||||
}
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hasBlock(blocks []Block, name string) bool {
|
||||
for _, b := range blocks {
|
||||
if b.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func code(s string) string { return "`" + s + "`" }
|
||||
|
||||
// anchor is how GitHub and Gitea slug a heading.
|
||||
func anchor(s string) string { return strings.ToLower(s) }
|
||||
Reference in new issue
Block a user