package engines import ( "slices" "strings" "testing" ) // envMap turns exec-form environment back into a map for assertions. func envMap(env []string) map[string]string { m := map[string]string{} for _, kv := range env { if k, v, ok := strings.Cut(kv, "="); ok { m[k] = v } } return m } // argValue returns the value following flag, or "" if the flag is absent. func argValue(args []string, flag string) string { i := slices.Index(args, flag) if i < 0 || i+1 >= len(args) { return "" } return args[i+1] } func singleNodeSpec() ShardSpec { return ShardSpec{ ReplicaID: "r1", ShardID: "r1-0", Rank: 0, NNodes: 1, GPUIndices: []int{0, 1, 2, 3}, MgmtInterface: "eth0", HTTPPort: 20000, Profile: Profile{ Name: "h100-tp4", Engine: "vllm", TP: 4, ModelPath: "/models/llama", }, } } func multiNodeSpec(rank int) ShardSpec { s := singleNodeSpec() s.Rank = rank s.NNodes = 2 s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7} s.HCAs = []string{"mlx5_0", "mlx5_1"} s.LeaderAddr = "10.0.0.10" s.LeaderPort = 20001 s.Profile.TP = 16 s.Profile.GPUsPerNode = 8 s.Profile.Nodes = 2 return s } // --- environment generation: the part that breaks multi-node runs --- func TestSingleNodeDisablesIB(t *testing.T) { cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec()) if err != nil { t.Fatal(err) } env := envMap(cmd.Env) if env["NCCL_IB_DISABLE"] != "1" { t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there") } if _, set := env["NCCL_IB_HCA"]; set { t.Error("NCCL_IB_HCA was set without any HCAs") } if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" { t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"]) } } func TestMultiNodePinsFabric(t *testing.T) { cmd, err := (&VLLM{}).BuildCommand(multiNodeSpec(0)) if err != nil { t.Fatal(err) } env := envMap(cmd.Env) // Only the chosen fabric's adapters. Leaving this open lets NCCL pick an // adapter on another fabric, where the peer is unreachable, and the job // hangs rather than failing. if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" { t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"]) } if env["NCCL_IB_DISABLE"] != "0" { t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"]) } // Bootstrap must stay on the management network; NCCL otherwise probes // every interface, including IB-over-IP that cannot route between fabrics. if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" { t.Errorf("socket interfaces not pinned: %q / %q", env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"]) } } func TestProfileEnvOverridesGenerated(t *testing.T) { s := multiNodeSpec(0) s.Profile.Env = map[string]string{ "NCCL_IB_HCA": "mlx5_9", "NCCL_NET_GDR_LEVEL": "SYS", } cmd, err := (&VLLM{}).BuildCommand(s) if err != nil { t.Fatal(err) } env := envMap(cmd.Env) if env["NCCL_IB_HCA"] != "mlx5_9" { t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning") } if env["NCCL_NET_GDR_LEVEL"] != "SYS" { t.Error("an extra profile variable was dropped") } } func TestProcessEnvironmentIsInherited(t *testing.T) { // Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit // file set up. t.Setenv("LLMGW_TEST_MARKER", "present") cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec()) if err != nil { t.Fatal(err) } if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" { t.Error("the process environment was not inherited") } } func TestEnvIsSorted(t *testing.T) { cmd, _ := (&VLLM{}).BuildCommand(singleNodeSpec()) if !slices.IsSorted(cmd.Env) { t.Error("environment is not sorted; launches would not be reproducible in logs") } } // --- vLLM --- func TestVLLMSingleNodeArgs(t *testing.T) { cmd, err := (&VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}).BuildCommand(singleNodeSpec()) if err != nil { t.Fatal(err) } if cmd.Path != "/opt/vllm/bin/vllm" { t.Errorf("Path=%q", cmd.Path) } if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" { t.Errorf("args start wrong: %v", cmd.Args[:2]) } if argValue(cmd.Args, "--tensor-parallel-size") != "4" { t.Errorf("TP not passed: %v", cmd.Args) } if argValue(cmd.Args, "--port") != "20000" { t.Errorf("port not passed: %v", cmd.Args) } // Single node means no distributed backend flag at all. if slices.Contains(cmd.Args, "--distributed-executor-backend") { t.Error("a distributed backend was selected for a single-node profile") } } func TestVLLMRayMode(t *testing.T) { v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}} cmd, err := v.BuildCommand(multiNodeSpec(1)) if err != nil { t.Fatal(err) } if argValue(cmd.Args, "--distributed-executor-backend") != "ray" { t.Errorf("backend not set: %v", cmd.Args) } if got := envMap(cmd.Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" { t.Errorf("RAY_ADDRESS=%q", got) } } func TestVLLMMPMode(t *testing.T) { v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}} cmd, err := v.BuildCommand(multiNodeSpec(0)) if err != nil { t.Fatal(err) } if argValue(cmd.Args, "--distributed-executor-backend") != "mp" { t.Errorf("backend not set: %v", cmd.Args) } env := envMap(cmd.Env) if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" { t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"]) } } func TestVLLMUnknownLaunchMode(t *testing.T) { v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}} if _, err := v.BuildCommand(multiNodeSpec(0)); err == nil { t.Error("an unknown launch mode was accepted") } } func TestVLLMRayCommand(t *testing.T) { v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}} head, ok := v.RayCommand(multiNodeSpec(0)) if !ok { t.Fatal("no ray command for the leader") } if !slices.Contains(head.Args, "--head") { t.Errorf("rank 0 must start the head: %v", head.Args) } worker, ok := v.RayCommand(multiNodeSpec(1)) if !ok { t.Fatal("no ray command for the worker") } if slices.Contains(worker.Args, "--head") { t.Errorf("rank 1 must not start a head: %v", worker.Args) } if argValue(worker.Args, "--address") != "10.0.0.10:20001" { t.Errorf("worker address wrong: %v", worker.Args) } // Single node needs no Ray at all. if _, ok := v.RayCommand(singleNodeSpec()); ok { t.Error("a single-node profile was given a Ray command") } // Neither does mp mode. mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}} if _, ok := mp.RayCommand(multiNodeSpec(0)); ok { t.Error("mp mode was given a Ray command") } } // --- SGLang --- func TestSGLangMultiNodeArgs(t *testing.T) { cmd, err := (&SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}).BuildCommand(multiNodeSpec(1)) if err != nil { t.Fatal(err) } if cmd.Path != "/opt/sglang/bin/python" { t.Errorf("Path=%q", cmd.Path) } if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" { t.Errorf("not launched as a module: %v", cmd.Args[:2]) } if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" { t.Errorf("multi-node flags wrong: %v", cmd.Args) } if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" { t.Errorf("dist-init-addr wrong: %v", cmd.Args) } } func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) { cmd, _ := (&SGLang{}).BuildCommand(singleNodeSpec()) for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} { if slices.Contains(cmd.Args, flag) { t.Errorf("%s was passed for a single-node profile", flag) } } } func TestProfileArgsComeLast(t *testing.T) { // Operators must be able to override anything the adapter generated. s := singleNodeSpec() s.Profile.Args = []string{"--max-model-len", "32768"} cmd, _ := (&VLLM{}).BuildCommand(s) i := slices.Index(cmd.Args, "--max-model-len") j := slices.Index(cmd.Args, "--tensor-parallel-size") if i < j { t.Errorf("profile args came before generated ones: %v", cmd.Args) } } // --- validation --- func h100(n int) []GPUInfo { out := make([]GPUInfo, n) for i := range out { out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559} } return out } func TestValidateAcceptsMatchingProfile(t *testing.T) { p := Profile{Name: "ok", TP: 4, ModelPath: "/m"} if err := (&VLLM{}).Validate(p, h100(4)); err != nil { t.Errorf("a valid profile was rejected: %v", err) } } func TestValidateRejectsMixedGPUTypes(t *testing.T) { // The invariant from plan ยง1.1: a TP group never mixes GPU types. gpus := h100(2) gpus[1].Name = "NVIDIA A100-SXM4-40GB" err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus) if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") { t.Errorf("mixed GPU types accepted: %v", err) } } func TestValidateRejectsFP8OnOldHardware(t *testing.T) { gpus := h100(4) for i := range gpus { gpus[i].Name = "NVIDIA A100-SXM4-80GB" gpus[i].ComputeCapability = "8.0" } p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}} err := (&VLLM{}).Validate(p, gpus) if err == nil || !strings.Contains(err.Error(), "compute capability") { t.Errorf("fp8 accepted on compute capability 8.0: %v", err) } // The same profile is fine on Hopper. if err := (&VLLM{}).Validate(p, h100(4)); err != nil { t.Errorf("fp8 rejected on H100: %v", err) } } func TestValidateRejectsInconsistentParallelism(t *testing.T) { // 2 nodes x 8 GPUs cannot serve TP 4. p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"} if err := (&VLLM{}).Validate(p, h100(8)); err == nil { t.Error("inconsistent node/GPU/TP arithmetic accepted") } } func TestValidateRejectsTooFewGPUs(t *testing.T) { if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil { t.Error("a TP 8 profile was accepted on a 4-GPU node") } } func TestValidateRequiresModelPath(t *testing.T) { if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil { t.Error("a profile without a model path was accepted") } } func TestRegistry(t *testing.T) { r := NewRegistry(Config{}) if got := r.Names(); len(got) != 2 || got[0] != "sglang" || got[1] != "vllm" { t.Errorf("registry contents: %v", got) } if _, err := r.Get("vllm"); err != nil { t.Errorf("vllm missing: %v", err) } err := func() error { _, err := r.Get("nope"); return err }() if err == nil || !strings.Contains(err.Error(), "sglang, vllm") { t.Errorf("an unknown engine should list what is available: %v", err) } } func TestQuantizationOf(t *testing.T) { for _, args := range [][]string{ {"--quantization", "fp8"}, {"--quantization=fp8"}, {"--max-model-len", "100", "--quantization", "fp8"}, } { if got := quantizationOf(args); got != "fp8" { t.Errorf("quantizationOf(%v)=%q", args, got) } } if got := quantizationOf([]string{"--quantization"}); got != "" { t.Errorf("a dangling flag should yield nothing, got %q", got) } }