Files
llmgw/internal/agent/engines/engines_test.go
T
ilkerandClaude Opus 5 6a13e3dddf
CI / test (push) Successful in 45s
Phase 2 (1/2): node discovery, engine adapters, process supervisor
Everything here works without GPUs and is covered by tests. The control plane
connection is not built yet, so llmgw agent itself is still a stub and the phase
exit criterion is not met.

- discovery: IB from /sys/class/infiniband, parsing the formats sysfs actually
  uses and telling native InfiniBand from RoCE; GPUs via nvidia-smi, the
  cgo-free path, with health flags for uncorrected ECC and memory already held
  by another process. A node without IB is not an error, it is a single-node
  island.
- engines: vLLM and SGLang command and environment construction, kept free of
  process handling so it stays testable. The NCCL environment is the part the
  plan marks critical: NCCL_IB_HCA lists only the chosen fabric's adapters,
  because leaving it open lets NCCL pick an adapter on a fabric where the peer
  is unreachable, and the job then hangs instead of failing.
- validation before launch: no mixed GPU types in a tensor-parallel group, the
  node/GPU/TP arithmetic has to add up, and fp8 is refused below compute
  capability 8.9 rather than silently falling back to a slower path.
- supervisor: process groups, because engines fork workers that hold GPU memory
  and outlive a signal sent only to the leader; a bounded log tail, since engine
  output is the only diagnosis when loading fails; Starting separate from
  Loading, because a 70B model legitimately takes minutes; and orphan adoption
  through pid files fingerprinted with the command line, since pids get reused
  and adopting an unrelated process is worse than leaking the original.
- readiness is two questions: /health says the process is up, /v1/models says
  the weights loaded. Routing on /health alone sends traffic into a loading model.
- llmgw agent inventory reports what the scheduler would see on a node.

One plan correction: GPU class sizes come from the driver in GiB, which does not
always match the marketing name. An L40S is sold as 48GB and reports 44.99 GiB,
so its class is l40s-45g, not l40s-48g. Consistency matters more than matching
the box, because that number decides whether a model fits. Plan §2 and §5.1
updated.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 02:01:59 +02:00

377 lines
11 KiB
Go

package engines
import (
"slices"
"strings"
"testing"
)
// envMap turns exec-form environment back into a map for assertions.
func envMap(env []string) map[string]string {
m := map[string]string{}
for _, kv := range env {
if k, v, ok := strings.Cut(kv, "="); ok {
m[k] = v
}
}
return m
}
// argValue returns the value following flag, or "" if the flag is absent.
func argValue(args []string, flag string) string {
i := slices.Index(args, flag)
if i < 0 || i+1 >= len(args) {
return ""
}
return args[i+1]
}
func singleNodeSpec() ShardSpec {
return ShardSpec{
ReplicaID: "r1",
ShardID: "r1-0",
Rank: 0,
NNodes: 1,
GPUIndices: []int{0, 1, 2, 3},
MgmtInterface: "eth0",
HTTPPort: 20000,
Profile: Profile{
Name: "h100-tp4",
Engine: "vllm",
TP: 4,
ModelPath: "/models/llama",
},
}
}
func multiNodeSpec(rank int) ShardSpec {
s := singleNodeSpec()
s.Rank = rank
s.NNodes = 2
s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7}
s.HCAs = []string{"mlx5_0", "mlx5_1"}
s.LeaderAddr = "10.0.0.10"
s.LeaderPort = 20001
s.Profile.TP = 16
s.Profile.GPUsPerNode = 8
s.Profile.Nodes = 2
return s
}
// --- environment generation: the part that breaks multi-node runs ---
func TestSingleNodeDisablesIB(t *testing.T) {
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
if err != nil {
t.Fatal(err)
}
env := envMap(cmd.Env)
if env["NCCL_IB_DISABLE"] != "1" {
t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there")
}
if _, set := env["NCCL_IB_HCA"]; set {
t.Error("NCCL_IB_HCA was set without any HCAs")
}
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
}
}
func TestMultiNodePinsFabric(t *testing.T) {
cmd, err := (&VLLM{}).BuildCommand(multiNodeSpec(0))
if err != nil {
t.Fatal(err)
}
env := envMap(cmd.Env)
// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
// adapter on another fabric, where the peer is unreachable, and the job
// hangs rather than failing.
if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" {
t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"])
}
if env["NCCL_IB_DISABLE"] != "0" {
t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"])
}
// Bootstrap must stay on the management network; NCCL otherwise probes
// every interface, including IB-over-IP that cannot route between fabrics.
if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" {
t.Errorf("socket interfaces not pinned: %q / %q",
env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"])
}
}
func TestProfileEnvOverridesGenerated(t *testing.T) {
s := multiNodeSpec(0)
s.Profile.Env = map[string]string{
"NCCL_IB_HCA": "mlx5_9",
"NCCL_NET_GDR_LEVEL": "SYS",
}
cmd, err := (&VLLM{}).BuildCommand(s)
if err != nil {
t.Fatal(err)
}
env := envMap(cmd.Env)
if env["NCCL_IB_HCA"] != "mlx5_9" {
t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning")
}
if env["NCCL_NET_GDR_LEVEL"] != "SYS" {
t.Error("an extra profile variable was dropped")
}
}
func TestProcessEnvironmentIsInherited(t *testing.T) {
// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
// file set up.
t.Setenv("LLMGW_TEST_MARKER", "present")
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
if err != nil {
t.Fatal(err)
}
if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
t.Error("the process environment was not inherited")
}
}
func TestEnvIsSorted(t *testing.T) {
cmd, _ := (&VLLM{}).BuildCommand(singleNodeSpec())
if !slices.IsSorted(cmd.Env) {
t.Error("environment is not sorted; launches would not be reproducible in logs")
}
}
// --- vLLM ---
func TestVLLMSingleNodeArgs(t *testing.T) {
cmd, err := (&VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}).BuildCommand(singleNodeSpec())
if err != nil {
t.Fatal(err)
}
if cmd.Path != "/opt/vllm/bin/vllm" {
t.Errorf("Path=%q", cmd.Path)
}
if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" {
t.Errorf("args start wrong: %v", cmd.Args[:2])
}
if argValue(cmd.Args, "--tensor-parallel-size") != "4" {
t.Errorf("TP not passed: %v", cmd.Args)
}
if argValue(cmd.Args, "--port") != "20000" {
t.Errorf("port not passed: %v", cmd.Args)
}
// Single node means no distributed backend flag at all.
if slices.Contains(cmd.Args, "--distributed-executor-backend") {
t.Error("a distributed backend was selected for a single-node profile")
}
}
func TestVLLMRayMode(t *testing.T) {
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
cmd, err := v.BuildCommand(multiNodeSpec(1))
if err != nil {
t.Fatal(err)
}
if argValue(cmd.Args, "--distributed-executor-backend") != "ray" {
t.Errorf("backend not set: %v", cmd.Args)
}
if got := envMap(cmd.Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
t.Errorf("RAY_ADDRESS=%q", got)
}
}
func TestVLLMMPMode(t *testing.T) {
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
cmd, err := v.BuildCommand(multiNodeSpec(0))
if err != nil {
t.Fatal(err)
}
if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
t.Errorf("backend not set: %v", cmd.Args)
}
env := envMap(cmd.Env)
if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
}
}
func TestVLLMUnknownLaunchMode(t *testing.T) {
v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
if _, err := v.BuildCommand(multiNodeSpec(0)); err == nil {
t.Error("an unknown launch mode was accepted")
}
}
func TestVLLMRayCommand(t *testing.T) {
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
head, ok := v.RayCommand(multiNodeSpec(0))
if !ok {
t.Fatal("no ray command for the leader")
}
if !slices.Contains(head.Args, "--head") {
t.Errorf("rank 0 must start the head: %v", head.Args)
}
worker, ok := v.RayCommand(multiNodeSpec(1))
if !ok {
t.Fatal("no ray command for the worker")
}
if slices.Contains(worker.Args, "--head") {
t.Errorf("rank 1 must not start a head: %v", worker.Args)
}
if argValue(worker.Args, "--address") != "10.0.0.10:20001" {
t.Errorf("worker address wrong: %v", worker.Args)
}
// Single node needs no Ray at all.
if _, ok := v.RayCommand(singleNodeSpec()); ok {
t.Error("a single-node profile was given a Ray command")
}
// Neither does mp mode.
mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
if _, ok := mp.RayCommand(multiNodeSpec(0)); ok {
t.Error("mp mode was given a Ray command")
}
}
// --- SGLang ---
func TestSGLangMultiNodeArgs(t *testing.T) {
cmd, err := (&SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}).BuildCommand(multiNodeSpec(1))
if err != nil {
t.Fatal(err)
}
if cmd.Path != "/opt/sglang/bin/python" {
t.Errorf("Path=%q", cmd.Path)
}
if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
t.Errorf("not launched as a module: %v", cmd.Args[:2])
}
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
t.Errorf("multi-node flags wrong: %v", cmd.Args)
}
if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" {
t.Errorf("dist-init-addr wrong: %v", cmd.Args)
}
}
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
cmd, _ := (&SGLang{}).BuildCommand(singleNodeSpec())
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
if slices.Contains(cmd.Args, flag) {
t.Errorf("%s was passed for a single-node profile", flag)
}
}
}
func TestProfileArgsComeLast(t *testing.T) {
// Operators must be able to override anything the adapter generated.
s := singleNodeSpec()
s.Profile.Args = []string{"--max-model-len", "32768"}
cmd, _ := (&VLLM{}).BuildCommand(s)
i := slices.Index(cmd.Args, "--max-model-len")
j := slices.Index(cmd.Args, "--tensor-parallel-size")
if i < j {
t.Errorf("profile args came before generated ones: %v", cmd.Args)
}
}
// --- validation ---
func h100(n int) []GPUInfo {
out := make([]GPUInfo, n)
for i := range out {
out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559}
}
return out
}
func TestValidateAcceptsMatchingProfile(t *testing.T) {
p := Profile{Name: "ok", TP: 4, ModelPath: "/m"}
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
t.Errorf("a valid profile was rejected: %v", err)
}
}
func TestValidateRejectsMixedGPUTypes(t *testing.T) {
// The invariant from plan §1.1: a TP group never mixes GPU types.
gpus := h100(2)
gpus[1].Name = "NVIDIA A100-SXM4-40GB"
err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus)
if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") {
t.Errorf("mixed GPU types accepted: %v", err)
}
}
func TestValidateRejectsFP8OnOldHardware(t *testing.T) {
gpus := h100(4)
for i := range gpus {
gpus[i].Name = "NVIDIA A100-SXM4-80GB"
gpus[i].ComputeCapability = "8.0"
}
p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}}
err := (&VLLM{}).Validate(p, gpus)
if err == nil || !strings.Contains(err.Error(), "compute capability") {
t.Errorf("fp8 accepted on compute capability 8.0: %v", err)
}
// The same profile is fine on Hopper.
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
t.Errorf("fp8 rejected on H100: %v", err)
}
}
func TestValidateRejectsInconsistentParallelism(t *testing.T) {
// 2 nodes x 8 GPUs cannot serve TP 4.
p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"}
if err := (&VLLM{}).Validate(p, h100(8)); err == nil {
t.Error("inconsistent node/GPU/TP arithmetic accepted")
}
}
func TestValidateRejectsTooFewGPUs(t *testing.T) {
if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil {
t.Error("a TP 8 profile was accepted on a 4-GPU node")
}
}
func TestValidateRequiresModelPath(t *testing.T) {
if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil {
t.Error("a profile without a model path was accepted")
}
}
func TestRegistry(t *testing.T) {
r := NewRegistry(Config{})
if got := r.Names(); len(got) != 2 || got[0] != "sglang" || got[1] != "vllm" {
t.Errorf("registry contents: %v", got)
}
if _, err := r.Get("vllm"); err != nil {
t.Errorf("vllm missing: %v", err)
}
err := func() error { _, err := r.Get("nope"); return err }()
if err == nil || !strings.Contains(err.Error(), "sglang, vllm") {
t.Errorf("an unknown engine should list what is available: %v", err)
}
}
func TestQuantizationOf(t *testing.T) {
for _, args := range [][]string{
{"--quantization", "fp8"},
{"--quantization=fp8"},
{"--max-model-len", "100", "--quantization", "fp8"},
} {
if got := quantizationOf(args); got != "fp8" {
t.Errorf("quantizationOf(%v)=%q", args, got)
}
}
if got := quantizationOf([]string{"--quantization"}); got != "" {
t.Errorf("a dangling flag should yield nothing, got %q", got)
}
}