Files
llmgw/internal/agent/engines/engines_test.go
T
Ilker ManapandClaude Opus 5.5 c0d5194ea0 Strata runs as an engine: a 125B MoE on one card
Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's
experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it
serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client.

- engines/strata.go: the adapter. The agent starts llmgw strata-serve,
  which runs Strata's setup the way its desktop app does
  (STRATA_CONTROL_INSTALL: installs nothing, writes only under the
  replica's and the node's directories) on the fetched GGUF files, points
  the configuration it writes at the installed engine, and execs Strata's
  server. The key goes in STRATA_API_KEY, never on a command line.
- A profile's args go to setup: --family, --size (setup's --model, which
  is the source everywhere else), --context, --parallel, ... Flags that
  move files or pick the engine or GPU are refused.
- provision: the source pinned by digest in a Python environment, and the
  release's prebuilt engine unpacked beside it, checked against the digest
  GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9,
  driver 575). Python builds can now carry release assets.
- Metrics are vLLM's names under /metrics/prometheus.
- The catalog wants one card and a family, and a directory, not one .gguf.

Opt-in: not in DefaultEngines.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 15:09:13 +01:00

531 lines
17 KiB
Go

package engines
import (
"fmt"
"slices"
"strconv"
"strings"
"testing"
)
// buildOne returns the single command a shard produces, failing if it produced
// a different number.
func buildOne(t *testing.T, e Engine, s ShardSpec) Command {
t.Helper()
cmds := build(t, e, s)
if len(cmds) != 1 {
t.Fatalf("expected 1 command, got %d: %s", len(cmds), summarize(cmds))
}
return cmds[0]
}
// buildEngine returns the last command, which is the engine itself. The ones
// before it are sidecars.
func buildEngine(t *testing.T, e Engine, s ShardSpec) Command {
t.Helper()
cmds := build(t, e, s)
return cmds[len(cmds)-1]
}
func build(t *testing.T, e Engine, s ShardSpec) []Command {
t.Helper()
cmds, err := e.Commands(s)
if err != nil {
t.Fatal(err)
}
if len(cmds) == 0 {
t.Fatal("no commands produced")
}
return cmds
}
// summarize keeps failure output readable: a Command carries the whole
// environment, and printing it buries the actual problem.
func summarize(cmds []Command) string {
var b strings.Builder
for i, c := range cmds {
if i > 0 {
b.WriteString(" | ")
}
fmt.Fprintf(&b, "%s: %s %s", c.Name, c.Path, strings.Join(c.Args, " "))
}
return b.String()
}
// envMap turns exec-form environment back into a map for assertions.
func envMap(env []string) map[string]string {
m := map[string]string{}
for _, kv := range env {
if k, v, ok := strings.Cut(kv, "="); ok {
m[k] = v
}
}
return m
}
// argValue returns the value following flag, or "" if the flag is absent.
func argValue(args []string, flag string) string {
i := slices.Index(args, flag)
if i < 0 || i+1 >= len(args) {
return ""
}
return args[i+1]
}
func singleNodeSpec() ShardSpec {
return ShardSpec{
ReplicaID: "r1",
ShardID: "r1-0",
Rank: 0,
NNodes: 1,
GPUIndices: []int{0, 1, 2, 3},
MgmtInterface: "eth0",
HTTPPort: 20000,
Profile: Profile{
Name: "h100-tp4",
Engine: "vllm",
TP: 4,
ModelPath: "/models/llama",
},
}
}
func multiNodeSpec(rank int) ShardSpec {
s := singleNodeSpec()
s.Rank = rank
s.NNodes = 2
s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7}
s.HCAs = []string{"mlx5_0", "mlx5_1"}
s.LeaderAddr = "10.0.0.10"
s.LeaderPort = 20001
s.Profile.TP = 16
s.Profile.GPUsPerNode = 8
s.Profile.Nodes = 2
return s
}
// --- environment generation: the part that breaks multi-node runs ---
func TestSingleNodeDisablesIB(t *testing.T) {
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
env := envMap(cmd.Env)
if env["NCCL_IB_DISABLE"] != "1" {
t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there")
}
if _, set := env["NCCL_IB_HCA"]; set {
t.Error("NCCL_IB_HCA was set without any HCAs")
}
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
}
}
func TestMultiNodePinsFabric(t *testing.T) {
cmd := buildEngine(t, &VLLM{}, multiNodeSpec(0))
env := envMap(cmd.Env)
// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
// adapter on another fabric, where the peer is unreachable, and the job
// hangs rather than failing.
if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" {
t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"])
}
if env["NCCL_IB_DISABLE"] != "0" {
t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"])
}
// Bootstrap must stay on the management network; NCCL otherwise probes
// every interface, including IB-over-IP that cannot route between fabrics.
if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" {
t.Errorf("socket interfaces not pinned: %q / %q",
env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"])
}
}
func TestProfileEnvOverridesGenerated(t *testing.T) {
s := multiNodeSpec(0)
s.Profile.Env = map[string]string{
"NCCL_IB_HCA": "mlx5_9",
"NCCL_NET_GDR_LEVEL": "SYS",
}
cmd := buildEngine(t, &VLLM{}, s)
env := envMap(cmd.Env)
if env["NCCL_IB_HCA"] != "mlx5_9" {
t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning")
}
if env["NCCL_NET_GDR_LEVEL"] != "SYS" {
t.Error("an extra profile variable was dropped")
}
}
func TestProcessEnvironmentIsInherited(t *testing.T) {
// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
// file set up.
t.Setenv("LLMGW_TEST_MARKER", "present")
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
t.Error("the process environment was not inherited")
}
}
func TestEnvIsSorted(t *testing.T) {
cmd := buildOne(t, &VLLM{}, singleNodeSpec())
if !slices.IsSorted(cmd.Env) {
t.Error("environment is not sorted; launches would not be reproducible in logs")
}
}
// --- vLLM ---
func TestVLLMSingleNodeArgs(t *testing.T) {
cmd := buildOne(t, &VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}, singleNodeSpec())
if cmd.Path != "/opt/vllm/bin/vllm" {
t.Errorf("Path=%q", cmd.Path)
}
if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" {
t.Errorf("args start wrong: %v", cmd.Args[:2])
}
if argValue(cmd.Args, "--tensor-parallel-size") != "4" {
t.Errorf("TP not passed: %v", cmd.Args)
}
if argValue(cmd.Args, "--port") != "20000" {
t.Errorf("port not passed: %v", cmd.Args)
}
// Single node means no distributed backend flag at all.
if slices.Contains(cmd.Args, "--distributed-executor-backend") {
t.Error("a distributed backend was selected for a single-node profile")
}
}
func TestVLLMRayLeaderRunsTwoProcesses(t *testing.T) {
// The leader starts a Ray head and then the server against it. A single
// command would mean running vLLM against a cluster that does not exist.
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
cmds, err := v.Commands(multiNodeSpec(0))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 2 {
t.Fatalf("expected 2 commands for the Ray leader, got %d", len(cmds))
}
if cmds[0].Name != "ray" || !slices.Contains(cmds[0].Args, "--head") {
t.Errorf("first command is not the Ray head: %+v", cmds[0])
}
if !cmds[0].WaitForLeaderPort {
t.Error("the server would start before the head is listening")
}
if cmds[1].Name != "vllm" || argValue(cmds[1].Args, "--distributed-executor-backend") != "ray" {
t.Errorf("second command is not the vLLM server: %+v", cmds[1])
}
if got := envMap(cmds[1].Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
t.Errorf("RAY_ADDRESS=%q", got)
}
}
func TestVLLMRayWorkerRunsOnlyRay(t *testing.T) {
// A Ray worker runs no vLLM process: the cluster hands its GPUs to the
// leader. Starting a second server there would bind a port nobody uses and
// compete for the same cards.
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
cmds, err := v.Commands(multiNodeSpec(1))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 {
t.Fatalf("expected 1 command for a Ray worker, got %d: %+v", len(cmds), cmds)
}
if cmds[0].Name != "ray" || slices.Contains(cmds[0].Args, "--head") {
t.Errorf("worker command wrong: %+v", cmds[0])
}
if argValue(cmds[0].Args, "--address") != "10.0.0.10:20001" {
t.Errorf("worker is not joining the leader: %v", cmds[0].Args)
}
}
func TestVLLMMPMode(t *testing.T) {
// vLLM's multi-node launch: every node the same serve with its rank and
// the leader's address, workers headless. Workers used to be given the
// leader's address as their own VLLM_HOST_IP, and no rank at all.
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
for _, rank := range []int{0, 1} {
cmd := buildOne(t, v, multiNodeSpec(rank))
for flag, want := range map[string]string{
"--distributed-executor-backend": "mp", "--nnodes": "2",
"--node-rank": strconv.Itoa(rank), "--master-addr": "10.0.0.10", "--master-port": "20001",
} {
if got := argValue(cmd.Args, flag); got != want {
t.Errorf("rank %d: %s = %q, want %q (%v)", rank, flag, got, want, cmd.Args)
}
}
if headless := slices.Contains(cmd.Args, "--headless"); headless != (rank > 0) {
t.Errorf("rank %d: headless = %v", rank, headless)
}
if ip, ok := envMap(cmd.Env)["VLLM_HOST_IP"]; ok && ip == "10.0.0.10" && rank > 0 {
t.Errorf("a worker was told the leader's address is its own")
}
}
}
func TestRoCEGetsItsGIDIndex(t *testing.T) {
s := multiNodeSpec(0)
idx := 3
s.GIDIndex = &idx
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
if env["NCCL_IB_GID_INDEX"] != "3" {
t.Errorf("NCCL_IB_GID_INDEX = %q", env["NCCL_IB_GID_INDEX"])
}
// InfiniBand needs none, and must not be given one.
env = envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env)
if _, ok := env["NCCL_IB_GID_INDEX"]; ok {
t.Error("an InfiniBand shard was given a GID index")
}
}
func TestVLLMUnknownLaunchMode(t *testing.T) {
v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
if _, err := v.Commands(multiNodeSpec(0)); err == nil {
t.Error("an unknown launch mode was accepted")
}
}
func TestOnlyLeaderServesHTTP(t *testing.T) {
// The fix for the bug that stopped multi-node replicas coming up at all:
// health-checking a worker's port times out and fails a healthy replica.
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
if !e.ServesHTTP(multiNodeSpec(0)) {
t.Errorf("%s: the leader must serve HTTP", e.Name())
}
if e.ServesHTTP(multiNodeSpec(1)) {
t.Errorf("%s: a worker must not be expected to serve HTTP", e.Name())
}
if !e.ServesHTTP(singleNodeSpec()) {
t.Errorf("%s: a single-node shard must serve HTTP", e.Name())
}
}
}
func TestNoRayForSingleNodeOrMPMode(t *testing.T) {
ray := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
if cmds := buildOne(t, ray, singleNodeSpec()); cmds.Name != "vllm" {
t.Errorf("a single-node profile was given a Ray process: %+v", cmds)
}
mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
if cmds := buildOne(t, mp, multiNodeSpec(0)); cmds.Name != "vllm" {
t.Errorf("mp mode was given a Ray process: %+v", cmds)
}
}
// --- SGLang ---
func TestSGLangMultiNodeArgs(t *testing.T) {
cmd := buildOne(t, &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}, multiNodeSpec(1))
if cmd.Path != "/opt/sglang/bin/python" {
t.Errorf("Path=%q", cmd.Path)
}
if cmd.Args[0] != "-c" || !strings.Contains(cmd.Args[1], `run_module("sglang.launch_server"`) {
t.Errorf("not launched as the module: %v", cmd.Args[:2])
}
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
t.Errorf("multi-node flags wrong: %v", cmd.Args)
}
if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" {
t.Errorf("dist-init-addr wrong: %v", cmd.Args)
}
}
func TestSGLangsKeyIsNotOnItsCommandLine(t *testing.T) {
s := singleNodeSpec()
s.APIKey = "k1"
cmd := buildOne(t, &SGLang{}, s)
if slices.Contains(cmd.Args, "k1") || slices.Contains(cmd.Args, "--api-key") {
t.Errorf("the key is in the arguments: %v", cmd.Args)
}
if !slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
t.Errorf("the key is not in the environment")
}
// A worker serves no HTTP, and is not given it.
w := multiNodeSpec(1)
w.APIKey = "k1"
if cmd := buildOne(t, &SGLang{}, w); slices.Contains(cmd.Env, sglangKeyEnv+"=k1") {
t.Error("a worker was given the key")
}
}
func TestEnginesListenOnTheAddressTheyAreGiven(t *testing.T) {
s := singleNodeSpec()
for _, e := range []Engine{&VLLM{}, &SGLang{}} {
s.BindAddr = ""
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "0.0.0.0" {
t.Errorf("%s: without an address listens on %q", e.Name(), got)
}
s.BindAddr = "10.0.0.147"
if got := argValue(buildOne(t, e, s).Args, "--host"); got != "10.0.0.147" {
t.Errorf("%s: listens on %q, expected the management address", e.Name(), got)
}
}
}
func TestSGLangIsStartedServingItsMetrics(t *testing.T) {
// Without --enable-metrics SGLang has no /metrics, and every scrape of
// the replica fails as though it were unreachable.
if cmd := buildOne(t, &SGLang{}, singleNodeSpec()); !slices.Contains(cmd.Args, "--enable-metrics") {
t.Errorf("not started with its metrics: %v", cmd.Args)
}
}
func TestSGLangIsAskedForItsHelpThroughItsModule(t *testing.T) {
g := &SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}
g.UseInstalled("/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python")
path, args := g.HelpCommand(Command{})
if path != "/opt/llmgw/engines/sglang/0.5.21-cu13/bin/python" {
t.Errorf("asked %s, not the interpreter it was installed into", path)
}
if !slices.Equal(args, []string{"-m", "sglang.launch_server", "--help"}) {
t.Errorf("asked %v", args)
}
}
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
cmd := buildOne(t, &SGLang{}, singleNodeSpec())
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
if slices.Contains(cmd.Args, flag) {
t.Errorf("%s was passed for a single-node profile", flag)
}
}
}
func TestProfileArgsComeLast(t *testing.T) {
// Operators must be able to override anything the adapter generated.
s := singleNodeSpec()
s.Profile.Args = []string{"--max-model-len", "32768"}
cmd := buildEngine(t, &VLLM{}, s)
i := slices.Index(cmd.Args, "--max-model-len")
j := slices.Index(cmd.Args, "--tensor-parallel-size")
if i < j {
t.Errorf("profile args came before generated ones: %v", cmd.Args)
}
}
// --- validation ---
func h100(n int) []GPUInfo {
out := make([]GPUInfo, n)
for i := range out {
out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559}
}
return out
}
func TestValidateAcceptsMatchingProfile(t *testing.T) {
p := Profile{Name: "ok", TP: 4, ModelPath: "/m"}
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
t.Errorf("a valid profile was rejected: %v", err)
}
}
func TestValidateRejectsMixedGPUTypes(t *testing.T) {
// The invariant from plan §1.1: a TP group never mixes GPU types.
gpus := h100(2)
gpus[1].Name = "NVIDIA A100-SXM4-40GB"
err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus)
if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") {
t.Errorf("mixed GPU types accepted: %v", err)
}
}
func TestValidateRejectsFP8OnOldHardware(t *testing.T) {
gpus := h100(4)
for i := range gpus {
gpus[i].Name = "NVIDIA A100-SXM4-80GB"
gpus[i].ComputeCapability = "8.0"
}
p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}}
err := (&VLLM{}).Validate(p, gpus)
if err == nil || !strings.Contains(err.Error(), "compute capability") {
t.Errorf("fp8 accepted on compute capability 8.0: %v", err)
}
// The same profile is fine on Hopper.
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
t.Errorf("fp8 rejected on H100: %v", err)
}
}
func TestValidateRejectsInconsistentParallelism(t *testing.T) {
// 2 nodes x 8 GPUs cannot serve TP 4.
p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"}
if err := (&VLLM{}).Validate(p, h100(8)); err == nil {
t.Error("inconsistent node/GPU/TP arithmetic accepted")
}
}
func TestValidateRejectsTooFewGPUs(t *testing.T) {
if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil {
t.Error("a TP 8 profile was accepted on a 4-GPU node")
}
}
func TestValidateRequiresModelPath(t *testing.T) {
if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil {
t.Error("a profile without a model path was accepted")
}
}
func TestRegistry(t *testing.T) {
r := NewRegistry(Config{})
if got := r.Names(); !slices.Equal(got, []string{"comfyui", "llamacpp", "sglang", "sim", "strata", "trtllm", "vllm"}) {
t.Errorf("registry contents: %v", got)
}
if _, err := r.Get("vllm"); err != nil {
t.Errorf("vllm missing: %v", err)
}
err := func() error { _, err := r.Get("nope"); return err }()
if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, sim, strata, trtllm, vllm") {
t.Errorf("an unknown engine should list what is available: %v", err)
}
}
func TestQuantizationOf(t *testing.T) {
for _, args := range [][]string{
{"--quantization", "fp8"},
{"--quantization=fp8"},
{"--max-model-len", "100", "--quantization", "fp8"},
} {
if got := quantizationOf(args); got != "fp8" {
t.Errorf("quantizationOf(%v)=%q", args, got)
}
}
if got := quantizationOf([]string{"--quantization"}); got != "" {
t.Errorf("a dangling flag should yield nothing, got %q", got)
}
}
func TestAReplicaInAnNVLinkDomainIsToldToUseIt(t *testing.T) {
s := multiNodeSpec(0)
s.Fabric = "nvl:76b1ad76-7c24-4b6b-9f6c-2a1e0d8c11aa/7"
env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, s).Env)
if env["NCCL_MNNVL_ENABLE"] != "1" {
t.Errorf("NCCL_MNNVL_ENABLE = %q", env["NCCL_MNNVL_ENABLE"])
}
if env := envMap(buildOne(t, &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}, multiNodeSpec(0)).Env); env["NCCL_MNNVL_ENABLE"] != "" {
t.Error("a replica on InfiniBand was told to use multi-node NVLink")
}
}
func TestMultiNodeVLLMDefaultsToItsOwnLaunch(t *testing.T) {
// mp is what vLLM documents and needs nothing but vLLM; Ray was the
// default and was never checked against a real cluster.
cmds, err := (&VLLM{}).Commands(multiNodeSpec(1))
if err != nil {
t.Fatal(err)
}
if len(cmds) != 1 || cmds[0].Name != "vllm" || argValue(cmds[0].Args, "--distributed-executor-backend") != "mp" ||
!slices.Contains(cmds[0].Args, "--headless") {
t.Errorf("default worker = %+v", cmds)
}
}