FreeToken, an engine for models larger than the card
FreeToken (FlashML-org/FreeToken) serves mixture-of-experts models several times a card's size from one card, experts in RAM, from Hugging Face safetensors checkpoints. Installed like vLLM, from PyPI by uv, on every node it runs on: compute capability 8.0+, driver 580+, x86-64, and the CUDA 13 toolkit it compiles kernels with. Older cards are passed over: on cuda01's Quadro RTX 6000 (7.5) 0.2.1 had no kernel image for the first prefill, and in fp16 hit an illegal memory access on its first request. ft serve gets the model, address, card and served name; the key goes in FREETOKEN_API_KEY. Flags that read files or fetch are refused, as are FREETOKEN_ variables naming paths or the agent's own settings. ctl model context sets --max-seq-len-override. Readiness now also waits for an engine whose /health answers 200 while its body says "loading", as FreeToken's does from the moment its HTTP server is up. FreeToken is followed release by release: a weekly workflow fails when PyPI has a version newer than DefaultEngines installs (scripts/engine_releases.py). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c866b3a8ee
commit
f4b34dbf11
25 files changed
+808
-18
No files matched your search
@@ -0,0 +1,53 @@
|
||||
package provision
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
)
|
||||
|
||||
func TestFreeTokenIsInstalledOnlyWhereItsKernelsRun(t *testing.T) {
|
||||
old := freetokenToolkit
|
||||
toolkit := "/usr/local/cuda-13.4"
|
||||
freetokenToolkit = func() string { return toolkit }
|
||||
t.Cleanup(func() { freetokenToolkit = old })
|
||||
|
||||
ampere := discovery.Host{Arch: "amd64", NVIDIADriver: "580.95.05", MinComputeCapability: 8.6}
|
||||
got, err := freetokenCandidates("0.2.1", "", ampere)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := got[0]
|
||||
if b.Binary != "ft" || !slices.Equal(b.Requirements, []string{"freetoken[accel]==0.2.1"}) || !b.GPUCheck || b.Python == "" {
|
||||
t.Errorf("build %+v", b)
|
||||
}
|
||||
|
||||
// Not meant for these, and not reported as a fault: no GPU, a Turing
|
||||
// card (cuda01, where 0.2.1 had no kernel image for it), an Arm node.
|
||||
for name, h := range map[string]discovery.Host{
|
||||
"no gpu": {Arch: "amd64"},
|
||||
"turing": {Arch: "amd64", NVIDIADriver: "615.71.09", MinComputeCapability: 7.5},
|
||||
"aarch64": {Arch: "arm64", NVIDIADriver: "580.95.05", MinComputeCapability: 9.0},
|
||||
} {
|
||||
_, err := freetokenCandidates("0.2.1", "", h)
|
||||
var nh NotHere
|
||||
if !errors.As(err, &nh) {
|
||||
t.Errorf("%s: %v, want NotHere", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Meant for these, and said why not.
|
||||
if _, err := freetokenCandidates("0.2.1", "", discovery.Host{Arch: "amd64", NVIDIADriver: "570.1", MinComputeCapability: 9.0}); err == nil ||
|
||||
!strings.Contains(err.Error(), "driver 580") {
|
||||
t.Errorf("old driver: %v", err)
|
||||
}
|
||||
toolkit = ""
|
||||
_, err = freetokenCandidates("0.2.1", "", ampere)
|
||||
var nh NotHere
|
||||
if err == nil || errors.As(err, &nh) || !strings.Contains(err.Error(), "CUDA 13 toolkit") {
|
||||
t.Errorf("no toolkit: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -126,12 +126,13 @@ type Recipe struct {
|
||||
// from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
|
||||
// checked against its published checksum.
|
||||
var Recipes = map[string]Recipe{
|
||||
"llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates},
|
||||
"vllm": {PyPI: "vllm", Candidates: vllmCandidates},
|
||||
"sglang": {PyPI: "sglang", Candidates: sglangCandidates},
|
||||
"comfyui": {Candidates: comfyuiCandidates},
|
||||
"strata": {Repo: StrataRepo, Candidates: strataCandidates},
|
||||
"uv": {Repo: "astral-sh/uv", Candidates: uvCandidates},
|
||||
"llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates},
|
||||
"vllm": {PyPI: "vllm", Candidates: vllmCandidates},
|
||||
"sglang": {PyPI: "sglang", Candidates: sglangCandidates},
|
||||
"comfyui": {Candidates: comfyuiCandidates},
|
||||
"strata": {Repo: StrataRepo, Candidates: strataCandidates},
|
||||
"freetoken": {PyPI: "freetoken", Candidates: freetokenCandidates},
|
||||
"uv": {Repo: "astral-sh/uv", Candidates: uvCandidates},
|
||||
}
|
||||
|
||||
// Installable are the engines this agent can install, as it tells the
|
||||
@@ -758,3 +759,56 @@ print("strata version", "` + version + `", "engine", "` + rel.engine + `")`},
|
||||
{build("cuda12"), gpuNeeds{575, 6.0}, "Strata " + version + " built for CUDA 12.9"},
|
||||
}, h)
|
||||
}
|
||||
|
||||
// --- FreeToken ---
|
||||
|
||||
// FreeTokenRepo is where FreeToken is developed; its builds are on PyPI.
|
||||
const FreeTokenRepo = "FlashML-org/FreeToken"
|
||||
|
||||
// freetokenMinCompute is the oldest GPU FreeToken runs on. It says RTX 30
|
||||
// and newer, and its kernels agree: on a Quadro RTX 6000 (7.5) version 0.2.1
|
||||
// loaded gpt-oss-20b and failed its first prefill with no kernel image for
|
||||
// the card, and in fp16 with fused experts its first request hit an illegal
|
||||
// memory access. Its FP8 kernels need 8.9 and its NVFP4 ones 12.0, which
|
||||
// the model's quantisation decides.
|
||||
const freetokenMinCompute = 8.0
|
||||
|
||||
// freetokenToolkit finds the CUDA toolkit FreeToken compiles its kernels
|
||||
// with; a variable for tests.
|
||||
var freetokenToolkit = func() string { return discovery.CUDAToolkit(13) }
|
||||
|
||||
// freetokenCandidates are the FreeToken builds for a machine: its PyPI
|
||||
// wheels with the kernel extras (FlashInfer, SGLang's kernels), CUDA 13,
|
||||
// x86-64 only. A node of older cards is not one it is meant for, and is not
|
||||
// reported as a fault; one of new cards without the toolkit is.
|
||||
func freetokenCandidates(version, variant string, h discovery.Host) ([]Build, error) {
|
||||
if _, err := gpuArch("FreeToken", h); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if h.Arch != "amd64" {
|
||||
return nil, NotHere{"FreeToken publishes wheels for x86-64 only; this node is " + h.Arch}
|
||||
}
|
||||
if cc := h.MinComputeCapability; cc > 0 && cc < freetokenMinCompute {
|
||||
return nil, NotHere{fmt.Sprintf("FreeToken runs on GPUs of compute capability %.1f and newer "+
|
||||
"(RTX 30, A100 and later); this node has %.1f", freetokenMinCompute, cc)}
|
||||
}
|
||||
// It compiles kernels the first time a model loads, with nvcc, as its
|
||||
// own install guide says; without a toolkit the installation looks
|
||||
// finished and the first replica fails.
|
||||
if freetokenToolkit() == "" {
|
||||
return nil, fmt.Errorf("FreeToken compiles its kernels where it runs and needs the CUDA 13 toolkit's nvcc " +
|
||||
"(in CUDA_HOME, /usr/local/cuda or /usr/local/cuda-13.*); install cuda-toolkit-13 from NVIDIA's repository")
|
||||
}
|
||||
b := Build{
|
||||
Variant: "cu13",
|
||||
Python: pythonForEngines,
|
||||
Requirements: []string{"freetoken[accel]==" + version},
|
||||
Binary: "ft",
|
||||
Packages: compilerPackages,
|
||||
FreeGB: 20,
|
||||
GPUCheck: true,
|
||||
}
|
||||
return pick("FreeToken "+version, variant, []candidate{
|
||||
{b, gpuNeeds{minDriverForCUDA13, freetokenMinCompute}, "FreeToken " + version},
|
||||
}, h)
|
||||
}
|
||||
Reference in new issue
Block a user