FreeToken (FlashML-org/FreeToken) serves mixture-of-experts models several times a card's size from one card, experts in RAM, from Hugging Face safetensors checkpoints. Installed like vLLM, from PyPI by uv, on every node it runs on: compute capability 8.0+, driver 580+, x86-64, and the CUDA 13 toolkit it compiles kernels with. Older cards are passed over: on cuda01's Quadro RTX 6000 (7.5) 0.2.1 had no kernel image for the first prefill, and in fp16 hit an illegal memory access on its first request. ft serve gets the model, address, card and served name; the key goes in FREETOKEN_API_KEY. Flags that read files or fetch are refused, as are FREETOKEN_ variables naming paths or the agent's own settings. ctl model context sets --max-seq-len-override. Readiness now also waits for an engine whose /health answers 200 while its body says "loading", as FreeToken's does from the moment its HTTP server is up. FreeToken is followed release by release: a weekly workflow fails when PyPI has a version newer than DefaultEngines installs (scripts/engine_releases.py). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c866b3a8ee
commit
f4b34dbf11
25 files changed
+808
-18
No files matched your search
@@ -157,6 +157,9 @@ var Reference = []Block{
|
||||
{Package: "config", Name: "FirewallSettings", Title: "FirewallSettings", Doc: "", Fields: []Field{
|
||||
{Key: "enabled", Type: "bool", Doc: "Enabled keeps llmgw's internal ports — engines, node exporter, the\nagents' stream, the mirror, the store — to the cluster's networks on\nevery machine. False removes llmgw's nftables table everywhere, for a\nsite whose own firewall does that job; the rules are kept, and true\nputs them back. Changed in the cluster's settings, it is in force on\nevery node and control plane within seconds.", Nested: ""},
|
||||
}},
|
||||
{Package: "engines", Name: "FreeTokenConfig", Title: "FreeTokenConfig", Doc: "", Fields: []Field{
|
||||
{Key: "binary", Type: "string", Doc: "Binary is its ft command, in the environment it was installed into.\nThe agent installs one itself on a node whose GPUs it runs on; this\nis for one installed by hand.", Nested: ""},
|
||||
}},
|
||||
{Package: "catalog", Name: "GPUNeed", Title: "GPUNeed", Doc: "", Fields: []Field{
|
||||
{Key: "min_compute_capability", Type: "float64", Doc: "MinComputeCapability is the oldest architecture that runs it: 7.5 is\nTuring, 8.0 Ampere (bf16, flash-attention 2), 8.9 Ada (fp8), 9.0 Hopper.", Nested: ""},
|
||||
{Key: "min_memory_gb", Type: "int", Doc: "MinMemoryGB is the card's own memory it needs.", Nested: ""},
|
||||
@@ -622,6 +625,7 @@ var Reference = []Block{
|
||||
{Key: "sim", Type: "SimConfig", Doc: "", Nested: "engines.SimConfig"},
|
||||
{Key: "comfyui", Type: "ComfyUIConfig", Doc: "ComfyUI makes pictures; offered by a node whose ComfyUI is named here.", Nested: "engines.ComfyUIConfig"},
|
||||
{Key: "strata", Type: "StrataConfig", Doc: "Strata serves a few mixture-of-experts models from RAM and one card;\noffered by a node whose Strata is named here, or that installed it.", Nested: "engines.StrataConfig"},
|
||||
{Key: "freetoken", Type: "FreeTokenConfig", Doc: "FreeToken serves mixture-of-experts models larger than a card from\nRAM and one card of 8.0 or newer; installed by the agent where the\ncards allow it, or named here.", Nested: "engines.FreeTokenConfig"},
|
||||
}},
|
||||
{Package: "guard", Name: "Config", Title: "guard.Config", Doc: "", Fields: []Field{
|
||||
{Key: "enabled", Type: "*bool", Doc: "Enabled turns every policy on or off at once: false checks nothing,\nwhatever is defined or attached. Unset is on.", Nested: ""},
|
||||
|
||||
@@ -24,6 +24,7 @@ var Sources = []string{
|
||||
"internal/agent/engines/sim.go",
|
||||
"internal/agent/engines/comfyui.go",
|
||||
"internal/agent/engines/strata.go",
|
||||
"internal/agent/engines/freetoken.go",
|
||||
"internal/agent/supervisor/supervisor.go",
|
||||
"internal/tlsconf/tlsconf.go",
|
||||
"internal/control/catalog/catalog.go",
|
||||
|
||||
Reference in new issue
Block a user