FreeToken (FlashML-org/FreeToken) serves mixture-of-experts models several times a card's size from one card, experts in RAM, from Hugging Face safetensors checkpoints. Installed like vLLM, from PyPI by uv, on every node it runs on: compute capability 8.0+, driver 580+, x86-64, and the CUDA 13 toolkit it compiles kernels with. Older cards are passed over: on cuda01's Quadro RTX 6000 (7.5) 0.2.1 had no kernel image for the first prefill, and in fp16 hit an illegal memory access on its first request. ft serve gets the model, address, card and served name; the key goes in FREETOKEN_API_KEY. Flags that read files or fetch are refused, as are FREETOKEN_ variables naming paths or the agent's own settings. ctl model context sets --max-seq-len-override. Readiness now also waits for an engine whose /health answers 200 while its body says "loading", as FreeToken's does from the moment its HTTP server is up. FreeToken is followed release by release: a weekly workflow fails when PyPI has a version newer than DefaultEngines installs (scripts/engine_releases.py). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
140 lines
4.8 KiB
Go
140 lines
4.8 KiB
Go
package docgen
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
// Sources are the files configuration is declared in.
|
|
//
|
|
// Listed rather than discovered, because "every file with a koanf tag" would
|
|
// sweep in structs that are configuration in the Go sense and not in the
|
|
// operator's — a scoring weight built at wiring time is not something anyone
|
|
// writes in a file. A test checks the list against what is actually tagged, so
|
|
// a new configuration file is a failure here rather than a silent omission.
|
|
var Sources = []string{
|
|
"internal/config/config.go",
|
|
"internal/agent/config.go",
|
|
"internal/agent/discovery/discover.go",
|
|
"internal/agent/engines/engine.go",
|
|
"internal/agent/engines/llamacpp.go",
|
|
"internal/agent/engines/vllm.go",
|
|
"internal/agent/engines/sglang.go",
|
|
"internal/agent/engines/trtllm.go",
|
|
"internal/agent/engines/sim.go",
|
|
"internal/agent/engines/comfyui.go",
|
|
"internal/agent/engines/strata.go",
|
|
"internal/agent/engines/freetoken.go",
|
|
"internal/agent/supervisor/supervisor.go",
|
|
"internal/tlsconf/tlsconf.go",
|
|
"internal/control/catalog/catalog.go",
|
|
"internal/control/catalog/workflow.go",
|
|
"internal/control/catalog/hardware.go",
|
|
"internal/router/admission/controller.go",
|
|
"internal/accounting/cost.go",
|
|
"internal/guard/policy.go",
|
|
"internal/guard/service.go",
|
|
"internal/guard/judge.go",
|
|
"internal/config/oidc.go",
|
|
"internal/config/ldap.go",
|
|
"internal/config/firewall.go",
|
|
"internal/config/serving.go",
|
|
"internal/config/presets.go",
|
|
"internal/config/ondemand.go",
|
|
"internal/config/agents.go",
|
|
"internal/mail/mail.go",
|
|
"internal/notify/notify.go",
|
|
"internal/secrets/secrets.go",
|
|
"internal/obs/trace.go",
|
|
}
|
|
|
|
// notReference are files with koanf tags that an operator never writes.
|
|
//
|
|
// Inventory and catalog entries carry them so the same structs can be decoded
|
|
// from an API payload or a model file; putting them in a configuration
|
|
// reference would invite somebody to write them into server.yaml.
|
|
var notReference = []string{
|
|
"internal/control/inventory/inventory.go",
|
|
}
|
|
|
|
// Render turns collected blocks into the reference document.
|
|
func Render(blocks []Block) string {
|
|
var b strings.Builder
|
|
b.WriteString(`# Configuration reference
|
|
|
|
Every setting llmgw reads from a file, with the type it takes and what it is
|
|
for.
|
|
|
|
**Generated from the source.** The descriptions are the comments on the fields
|
|
themselves, so this cannot describe a setting that does not exist or miss one
|
|
that does — and a test fails when it has drifted. Do not edit it; edit the
|
|
comment and run ` + "`go generate ./...`" + `.
|
|
|
|
Where a setting takes another block, the block is listed under its own heading
|
|
rather than repeated.
|
|
|
|
`)
|
|
|
|
// An index first: the reason to open a reference is usually to find one
|
|
// key, and scanning thirteen headings for it is the thing this replaces.
|
|
b.WriteString("## Every key\n\n| Key | Type | In |\n|---|---|---|\n")
|
|
for _, bl := range blocks {
|
|
for _, f := range bl.Fields {
|
|
b.WriteString(fmt.Sprintf("| `%s` | %s | [%s](#%s) |\n",
|
|
f.Key, code(f.Type), bl.Title, anchor(bl.Title)))
|
|
}
|
|
}
|
|
b.WriteString("\n")
|
|
|
|
for _, bl := range blocks {
|
|
fmt.Fprintf(&b, "## %s\n\n", bl.Title)
|
|
if bl.Doc != "" {
|
|
b.WriteString(bl.Doc + "\n\n")
|
|
}
|
|
for _, f := range bl.Fields {
|
|
fmt.Fprintf(&b, "### `%s`\n\n%s\n\n", f.Key, code(f.Type))
|
|
if f.Doc != "" {
|
|
b.WriteString(f.Doc + "\n\n")
|
|
}
|
|
if nb, ok := Find(blocks, f.Nested); ok {
|
|
fmt.Fprintf(&b, "Takes a [%s](#%s) block.\n\n", nb.Title, anchor(nb.Title))
|
|
}
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// Find is the block with an ID, as a Field's Nested names it.
|
|
func Find(blocks []Block, id string) (Block, bool) {
|
|
for _, b := range blocks {
|
|
if id != "" && b.ID() == id {
|
|
return b, true
|
|
}
|
|
}
|
|
return Block{}, false
|
|
}
|
|
|
|
func code(s string) string { return "`" + s + "`" }
|
|
|
|
// anchor is how GitHub and Gitea slug a heading: lower case, and punctuation
|
|
// such as the dot in tlsconf.Config dropped.
|
|
func anchor(s string) string { return strings.ToLower(strings.ReplaceAll(s, ".", "")) }
|
|
|
|
// RenderGo writes the blocks as Go, for the settings screens: every key, its
|
|
// type and what it does, from the same source as the reference, so a setting
|
|
// added to the code is on the screens without anyone listing it.
|
|
func RenderGo(blocks []Block) string {
|
|
var b strings.Builder
|
|
b.WriteString("// Code generated by go run ./internal/docgen/gen; DO NOT EDIT.\n\npackage docgen\n\n")
|
|
b.WriteString("// Reference is every configuration block, as Collect reads them.\nvar Reference = []Block{\n")
|
|
for _, bl := range blocks {
|
|
fmt.Fprintf(&b, "\t{Package: %q, Name: %q, Title: %q, Doc: %q, Fields: []Field{\n", bl.Package, bl.Name, bl.Title, bl.Doc)
|
|
for _, f := range bl.Fields {
|
|
fmt.Fprintf(&b, "\t\t{Key: %q, Type: %q, Doc: %q, Nested: %q},\n", f.Key, f.Type, f.Doc, f.Nested)
|
|
}
|
|
b.WriteString("\t}},\n")
|
|
}
|
|
b.WriteString("}\n")
|
|
return b.String()
|
|
}
|