Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client. - engines/strata.go: the adapter. The agent starts llmgw strata-serve, which runs Strata's setup the way its desktop app does (STRATA_CONTROL_INSTALL: installs nothing, writes only under the replica's and the node's directories) on the fetched GGUF files, points the configuration it writes at the installed engine, and execs Strata's server. The key goes in STRATA_API_KEY, never on a command line. - A profile's args go to setup: --family, --size (setup's --model, which is the source everywhere else), --context, --parallel, ... Flags that move files or pick the engine or GPU are refused. - provision: the source pinned by digest in a Python environment, and the release's prebuilt engine unpacked beside it, checked against the digest GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9, driver 575). Python builds can now carry release assets. - Metrics are vLLM's names under /metrics/prometheus. - The catalog wants one card and a family, and a directory, not one .gguf. Opt-in: not in DefaultEngines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
70 lines
2.4 KiB
Go
70 lines
2.4 KiB
Go
package engines
|
|
|
|
import (
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestStrataStartsStrataServeWithTheKeyInTheEnvironment(t *testing.T) {
|
|
s := &Strata{}
|
|
s.UseInstalled("/opt/llmgw/engines/strata/0.2.7-cuda13/bin/python")
|
|
spec := ShardSpec{
|
|
ReplicaID: "r1", HTTPPort: 18090, BindAddr: "10.0.0.147", GPUIndices: []int{0},
|
|
APIKey: "secret-key",
|
|
Profile: Profile{Name: "p", Engine: "strata", TP: 1, ModelPath: "/var/lib/llmgw/models/q38",
|
|
ServedModelName: "qwen3.8-flash-next", Args: []string{"--family", "qwen", "--size", "IQ2_XS"}},
|
|
}
|
|
cmds, err := s.Commands(spec)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c := cmds[0]
|
|
args := strings.Join(c.Args, " ")
|
|
env := "/opt/llmgw/engines/strata/0.2.7-cuda13"
|
|
for _, want := range []string{
|
|
"strata-serve", "--listen 10.0.0.147:18090", "--model qwen3.8-flash-next",
|
|
"--weights /var/lib/llmgw/models/q38", "--state " + filepath.Join("strata", "r1"),
|
|
"--python " + env + "/bin/python", "--strata " + env + "/strata", "--engine " + env + "/strata-engine",
|
|
"-- --family qwen --size IQ2_XS",
|
|
} {
|
|
if !strings.Contains(args, want) {
|
|
t.Errorf("args lack %q: %s", want, args)
|
|
}
|
|
}
|
|
if strings.Contains(args, "secret-key") {
|
|
t.Error("the key is on the command line, which every user on the node can read")
|
|
}
|
|
if !slices.Contains(c.Env, "STRATA_API_KEY=secret-key") {
|
|
t.Error("the key is not in the environment, where Strata's server reads it")
|
|
}
|
|
if !slices.Contains(c.Env, "CUDA_VISIBLE_DEVICES=0") {
|
|
t.Error("the shard is not pinned to its GPU")
|
|
}
|
|
}
|
|
|
|
func TestStrataRunsOnOneCard(t *testing.T) {
|
|
s := &Strata{}
|
|
p := Profile{Name: "p", Engine: "strata", TP: 1, GPUsPerNode: 2, ModelPath: "/m"}
|
|
if err := s.Validate(p, nil); err == nil {
|
|
t.Error("a two-card profile was accepted")
|
|
}
|
|
p.GPUsPerNode = 1
|
|
if err := s.Validate(p, []GPUInfo{{Name: "Quadro RTX 6000", ComputeCapability: "7.5"}}); err != nil {
|
|
t.Errorf("one Turing card was refused: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestStrataSetupFlagsThatMoveFilesAreRefused(t *testing.T) {
|
|
for _, bad := range []string{"--gguf-dir", "--gguf", "--data-dir=/etc", "--model", "--mod", "--feed", "--host", "--gpus"} {
|
|
if err := UnsafeArgs("strata", []string{bad, "x"}); err == nil {
|
|
t.Errorf("%s was accepted", bad)
|
|
}
|
|
}
|
|
if err := UnsafeArgs("strata", []string{"--family", "qwen", "--size", "IQ2_XS", "--context", "32768",
|
|
"--parallel", "2", "--vram-reserve-mib", "2048", "--vision", "no"}); err != nil {
|
|
t.Errorf("an ordinary profile was refused: %v", err)
|
|
}
|
|
}
|