Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client. - engines/strata.go: the adapter. The agent starts llmgw strata-serve, which runs Strata's setup the way its desktop app does (STRATA_CONTROL_INSTALL: installs nothing, writes only under the replica's and the node's directories) on the fetched GGUF files, points the configuration it writes at the installed engine, and execs Strata's server. The key goes in STRATA_API_KEY, never on a command line. - A profile's args go to setup: --family, --size (setup's --model, which is the source everywhere else), --context, --parallel, ... Flags that move files or pick the engine or GPU are refused. - provision: the source pinned by digest in a Python environment, and the release's prebuilt engine unpacked beside it, checked against the digest GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9, driver 575). Python builds can now carry release assets. - Metrics are vLLM's names under /metrics/prometheus. - The catalog wants one card and a family, and a directory, not one .gguf. Opt-in: not in DefaultEngines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
77 lines
2.6 KiB
Go
77 lines
2.6 KiB
Go
package strata
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSizeIsSetupsModel(t *testing.T) {
|
|
got, err := SetupArgs([]string{"--family", "qwen", "--size", "IQ2_XS", "--context", "32768"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if want := []string{"--family", "qwen", "--model", "IQ2_XS", "--context", "32768"}; !slices.Equal(got, want) {
|
|
t.Errorf("got %v, want %v", got, want)
|
|
}
|
|
if got, _ := SetupArgs([]string{"--family=qwen36", "--size=UD-Q4_K_XL"}); !slices.Equal(got, []string{"--family=qwen36", "--model=UD-Q4_K_XL"}) {
|
|
t.Errorf("the = forms: %v", got)
|
|
}
|
|
if _, err := SetupArgs([]string{"--size", "IQ2_XS"}); err == nil {
|
|
t.Error("a profile with no family was accepted")
|
|
}
|
|
}
|
|
|
|
func TestTheConfigurationRunsTheInstalledEngineWithNoKey(t *testing.T) {
|
|
state, engine := t.TempDir(), t.TempDir()
|
|
if err := os.WriteFile(filepath.Join(engine, "strata"), []byte("\x7fELF"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old := filepath.Join(state, "strata-q2_0.json")
|
|
path := filepath.Join(state, "strata-iq2_xs.json")
|
|
for _, p := range []string{old, path} {
|
|
b, _ := json.Marshal(map[string]any{"exe": "component:strata-engine", "args": []string{"--pack", "/p"},
|
|
"model_name": "qwen3.8-flash-next-iq2_xs", "api_key": "from-setup", "host": "127.0.0.1"})
|
|
if err := os.WriteFile(p, b, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
past := time.Now().Add(-time.Hour)
|
|
_ = os.Chtimes(old, past, past)
|
|
|
|
got, err := newestConfig(state)
|
|
if err != nil || got != path {
|
|
t.Fatalf("newest configuration %q, %v; want %q", got, err, path)
|
|
}
|
|
c := Config{Engine: engine, Model: "qwen3.8-flash-next"}
|
|
if err := adopt(path, c, "10.0.0.147"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var cfg map[string]any
|
|
b, _ := os.ReadFile(path)
|
|
if err := json.Unmarshal(b, &cfg); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cfg["exe"] != filepath.Join(engine, "strata") || cfg["host"] != "10.0.0.147" || cfg["model_name"] != "qwen3.8-flash-next" {
|
|
t.Errorf("configuration not adopted: %v", cfg)
|
|
}
|
|
if _, ok := cfg["api_key"]; ok {
|
|
t.Error("the configuration keeps a key; it comes in the environment only")
|
|
}
|
|
if args, _ := cfg["args"].([]any); len(args) != 2 {
|
|
t.Errorf("setup's engine arguments were lost: %v", cfg["args"])
|
|
}
|
|
}
|
|
|
|
func TestAMissingEngineIsSaid(t *testing.T) {
|
|
state := t.TempDir()
|
|
path := filepath.Join(state, "strata-x.json")
|
|
_ = os.WriteFile(path, []byte(`{"exe":"component:strata-engine"}`), 0o600)
|
|
if err := adopt(path, Config{Engine: filepath.Join(state, "nowhere")}, "0.0.0.0"); err == nil {
|
|
t.Error("a configuration was pointed at an engine that is not there")
|
|
}
|
|
}
|