FreeToken, an engine for models larger than the card
CI / test (push) Successful in 9m37s

FreeToken (FlashML-org/FreeToken) serves mixture-of-experts models several
times a card's size from one card, experts in RAM, from Hugging Face
safetensors checkpoints. Installed like vLLM, from PyPI by uv, on every node
it runs on: compute capability 8.0+, driver 580+, x86-64, and the CUDA 13
toolkit it compiles kernels with. Older cards are passed over: on cuda01's
Quadro RTX 6000 (7.5) 0.2.1 had no kernel image for the first prefill, and
in fp16 hit an illegal memory access on its first request.

ft serve gets the model, address, card and served name; the key goes in
FREETOKEN_API_KEY. Flags that read files or fetch are refused, as are
FREETOKEN_ variables naming paths or the agent's own settings. ctl model
context sets --max-seq-len-override.

Readiness now also waits for an engine whose /health answers 200 while its
body says "loading", as FreeToken's does from the moment its HTTP server is up.

FreeToken is followed release by release: a weekly workflow fails when PyPI
has a version newer than DefaultEngines installs (scripts/engine_releases.py).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-31 22:10:05 +01:00
1 parent c866b3a8ee
commit f4b34dbf11
25 files changed
+808 -18

No files matched your search

+93
View File
@@ -0,0 +1,93 @@
package discovery
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
)
// CUDAToolkit is the root of an installed CUDA toolkit of this major version
// — the directory whose bin holds nvcc — or "" when there is none. Engines
// that compile their kernels where they run (FreeToken, ComfyUI's compiled
// nodes) need one matching the CUDA their torch was built for, and a
// toolkit of another major version compiles code that torch cannot load.
//
// CUDA_HOME is looked at first, then /usr/local/cuda, then every
// /usr/local/cuda-<version>, newest first.
func CUDAToolkit(major int) string {
return cudaToolkitIn(major, os.Getenv("CUDA_HOME"), "/usr/local")
}
func cudaToolkitIn(major int, home, root string) string {
var dirs []string
if home != "" {
dirs = append(dirs, home)
}
dirs = append(dirs, filepath.Join(root, "cuda"))
versioned, _ := filepath.Glob(filepath.Join(root, "cuda-*"))
slices.SortFunc(versioned, func(a, b string) int { return versionCompare(b, a) })
dirs = append(dirs, versioned...)
for _, d := range dirs {
if _, err := os.Stat(filepath.Join(d, "bin", "nvcc")); err != nil {
continue
}
if toolkitMajor(d) == major {
return d
}
}
return ""
}
// toolkitMajor is a toolkit's major version, from the version.json every
// toolkit since 11.1 carries, or from nvcc where it has none; 0 when neither
// says.
func toolkitMajor(dir string) int {
if b, err := os.ReadFile(filepath.Join(dir, "version.json")); err == nil { //nolint:gosec // a toolkit's own file
var v struct {
CUDA struct {
Version string `json:"version"`
} `json:"cuda"`
}
if json.Unmarshal(b, &v) == nil && v.CUDA.Version != "" {
n, _ := strconv.Atoi(strings.SplitN(v.CUDA.Version, ".", 2)[0])
return n
}
}
//nolint:gosec // nvcc of a toolkit found on disk
out, err := exec.Command(filepath.Join(dir, "bin", "nvcc"), "--version").Output()
if err != nil {
return 0
}
if m := nvccRelease.FindSubmatch(out); m != nil {
n, _ := strconv.Atoi(string(m[1]))
return n
}
return 0
}
var nvccRelease = regexp.MustCompile(`release (\d+)\.`)
// versionCompare orders two paths ending in dotted versions: cuda-13.10 is
// newer than cuda-13.4, which comparing them as text gets wrong.
func versionCompare(a, b string) int {
pa := strings.Split(strings.TrimPrefix(filepath.Base(a), "cuda-"), ".")
pb := strings.Split(strings.TrimPrefix(filepath.Base(b), "cuda-"), ".")
for i := range max(len(pa), len(pb)) {
var x, y int
if i < len(pa) {
x, _ = strconv.Atoi(pa[i])
}
if i < len(pb) {
y, _ = strconv.Atoi(pb[i])
}
if x != y {
return x - y
}
}
return 0
}
@@ -0,0 +1,48 @@
package discovery
import (
"os"
"path/filepath"
"testing"
)
func toolkit(t *testing.T, root, name, version string) string {
t.Helper()
d := filepath.Join(root, name)
put(t, d, "bin/nvcc", "#!/bin/sh")
if version != "" {
put(t, d, "version.json", `{"cuda":{"name":"CUDA SDK","version":"`+version+`"}}`)
}
return d
}
func TestTheNewestToolkitOfTheMajorVersionAskedForIsFound(t *testing.T) {
root := t.TempDir()
toolkit(t, root, "cuda-12.8", "12.8.1")
toolkit(t, root, "cuda-13.4", "13.4.0")
want := toolkit(t, root, "cuda-13.10", "13.10.0")
// /usr/local/cuda is a 12: a toolkit of the wrong major version is not
// one torch built for 13 can load what it compiles.
toolkit(t, root, "cuda", "12.8.1")
if got := cudaToolkitIn(13, "", root); got != want {
t.Errorf("got %q, want %q", got, want)
}
if got := cudaToolkitIn(12, "", root); got != filepath.Join(root, "cuda") {
t.Errorf("12: got %q", got)
}
// CUDA_HOME first, when it is one.
home := toolkit(t, t.TempDir(), "mine", "13.0.2")
if got := cudaToolkitIn(13, home, root); got != home {
t.Errorf("CUDA_HOME: got %q", got)
}
if got := cudaToolkitIn(14, "", root); got != "" {
t.Errorf("14: got %q", got)
}
// A directory without nvcc is not a toolkit.
if err := os.MkdirAll(filepath.Join(root, "cuda-13.99"), 0o755); err != nil {
t.Fatal(err)
}
if got := cudaToolkitIn(13, "", root); got != want {
t.Errorf("with an empty cuda-13.99: got %q", got)
}
}