Everything here works without GPUs and is covered by tests. The control plane connection is not built yet, so llmgw agent itself is still a stub and the phase exit criterion is not met. - discovery: IB from /sys/class/infiniband, parsing the formats sysfs actually uses and telling native InfiniBand from RoCE; GPUs via nvidia-smi, the cgo-free path, with health flags for uncorrected ECC and memory already held by another process. A node without IB is not an error, it is a single-node island. - engines: vLLM and SGLang command and environment construction, kept free of process handling so it stays testable. The NCCL environment is the part the plan marks critical: NCCL_IB_HCA lists only the chosen fabric's adapters, because leaving it open lets NCCL pick an adapter on a fabric where the peer is unreachable, and the job then hangs instead of failing. - validation before launch: no mixed GPU types in a tensor-parallel group, the node/GPU/TP arithmetic has to add up, and fp8 is refused below compute capability 8.9 rather than silently falling back to a slower path. - supervisor: process groups, because engines fork workers that hold GPU memory and outlive a signal sent only to the leader; a bounded log tail, since engine output is the only diagnosis when loading fails; Starting separate from Loading, because a 70B model legitimately takes minutes; and orphan adoption through pid files fingerprinted with the command line, since pids get reused and adopting an unrelated process is worse than leaking the original. - readiness is two questions: /health says the process is up, /v1/models says the weights loaded. Routing on /health alone sends traffic into a loading model. - llmgw agent inventory reports what the scheduler would see on a node. One plan correction: GPU class sizes come from the driver in GiB, which does not always match the marketing name. An L40S is sold as 48GB and reports 44.99 GiB, so its class is l40s-45g, not l40s-48g. Consistency matters more than matching the box, because that number decides whether a model fits. Plan §2 and §5.1 updated. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
377 lines
11 KiB
Go
377 lines
11 KiB
Go
package engines
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// envMap turns exec-form environment back into a map for assertions.
|
|
func envMap(env []string) map[string]string {
|
|
m := map[string]string{}
|
|
for _, kv := range env {
|
|
if k, v, ok := strings.Cut(kv, "="); ok {
|
|
m[k] = v
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// argValue returns the value following flag, or "" if the flag is absent.
|
|
func argValue(args []string, flag string) string {
|
|
i := slices.Index(args, flag)
|
|
if i < 0 || i+1 >= len(args) {
|
|
return ""
|
|
}
|
|
return args[i+1]
|
|
}
|
|
|
|
func singleNodeSpec() ShardSpec {
|
|
return ShardSpec{
|
|
ReplicaID: "r1",
|
|
ShardID: "r1-0",
|
|
Rank: 0,
|
|
NNodes: 1,
|
|
GPUIndices: []int{0, 1, 2, 3},
|
|
MgmtInterface: "eth0",
|
|
HTTPPort: 20000,
|
|
Profile: Profile{
|
|
Name: "h100-tp4",
|
|
Engine: "vllm",
|
|
TP: 4,
|
|
ModelPath: "/models/llama",
|
|
},
|
|
}
|
|
}
|
|
|
|
func multiNodeSpec(rank int) ShardSpec {
|
|
s := singleNodeSpec()
|
|
s.Rank = rank
|
|
s.NNodes = 2
|
|
s.GPUIndices = []int{0, 1, 2, 3, 4, 5, 6, 7}
|
|
s.HCAs = []string{"mlx5_0", "mlx5_1"}
|
|
s.LeaderAddr = "10.0.0.10"
|
|
s.LeaderPort = 20001
|
|
s.Profile.TP = 16
|
|
s.Profile.GPUsPerNode = 8
|
|
s.Profile.Nodes = 2
|
|
return s
|
|
}
|
|
|
|
// --- environment generation: the part that breaks multi-node runs ---
|
|
|
|
func TestSingleNodeDisablesIB(t *testing.T) {
|
|
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env := envMap(cmd.Env)
|
|
|
|
if env["NCCL_IB_DISABLE"] != "1" {
|
|
t.Error("NCCL_IB_DISABLE must be 1 without a fabric, or NCCL waits for one that is not there")
|
|
}
|
|
if _, set := env["NCCL_IB_HCA"]; set {
|
|
t.Error("NCCL_IB_HCA was set without any HCAs")
|
|
}
|
|
if env["CUDA_VISIBLE_DEVICES"] != "0,1,2,3" {
|
|
t.Errorf("CUDA_VISIBLE_DEVICES=%q", env["CUDA_VISIBLE_DEVICES"])
|
|
}
|
|
}
|
|
|
|
func TestMultiNodePinsFabric(t *testing.T) {
|
|
cmd, err := (&VLLM{}).BuildCommand(multiNodeSpec(0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env := envMap(cmd.Env)
|
|
|
|
// Only the chosen fabric's adapters. Leaving this open lets NCCL pick an
|
|
// adapter on another fabric, where the peer is unreachable, and the job
|
|
// hangs rather than failing.
|
|
if env["NCCL_IB_HCA"] != "mlx5_0,mlx5_1" {
|
|
t.Errorf("NCCL_IB_HCA=%q", env["NCCL_IB_HCA"])
|
|
}
|
|
if env["NCCL_IB_DISABLE"] != "0" {
|
|
t.Errorf("NCCL_IB_DISABLE=%q with a fabric present", env["NCCL_IB_DISABLE"])
|
|
}
|
|
// Bootstrap must stay on the management network; NCCL otherwise probes
|
|
// every interface, including IB-over-IP that cannot route between fabrics.
|
|
if env["NCCL_SOCKET_IFNAME"] != "eth0" || env["GLOO_SOCKET_IFNAME"] != "eth0" {
|
|
t.Errorf("socket interfaces not pinned: %q / %q",
|
|
env["NCCL_SOCKET_IFNAME"], env["GLOO_SOCKET_IFNAME"])
|
|
}
|
|
}
|
|
|
|
func TestProfileEnvOverridesGenerated(t *testing.T) {
|
|
s := multiNodeSpec(0)
|
|
s.Profile.Env = map[string]string{
|
|
"NCCL_IB_HCA": "mlx5_9",
|
|
"NCCL_NET_GDR_LEVEL": "SYS",
|
|
}
|
|
cmd, err := (&VLLM{}).BuildCommand(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
env := envMap(cmd.Env)
|
|
|
|
if env["NCCL_IB_HCA"] != "mlx5_9" {
|
|
t.Error("a profile override did not win; it is the escape hatch for per-cluster tuning")
|
|
}
|
|
if env["NCCL_NET_GDR_LEVEL"] != "SYS" {
|
|
t.Error("an extra profile variable was dropped")
|
|
}
|
|
}
|
|
|
|
func TestProcessEnvironmentIsInherited(t *testing.T) {
|
|
// Engines need PATH, HF_HOME, LD_LIBRARY_PATH and whatever else the unit
|
|
// file set up.
|
|
t.Setenv("LLMGW_TEST_MARKER", "present")
|
|
cmd, err := (&VLLM{}).BuildCommand(singleNodeSpec())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if envMap(cmd.Env)["LLMGW_TEST_MARKER"] != "present" {
|
|
t.Error("the process environment was not inherited")
|
|
}
|
|
}
|
|
|
|
func TestEnvIsSorted(t *testing.T) {
|
|
cmd, _ := (&VLLM{}).BuildCommand(singleNodeSpec())
|
|
if !slices.IsSorted(cmd.Env) {
|
|
t.Error("environment is not sorted; launches would not be reproducible in logs")
|
|
}
|
|
}
|
|
|
|
// --- vLLM ---
|
|
|
|
func TestVLLMSingleNodeArgs(t *testing.T) {
|
|
cmd, err := (&VLLM{cfg: VLLMConfig{Bin: "/opt/vllm/bin/vllm"}}).BuildCommand(singleNodeSpec())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cmd.Path != "/opt/vllm/bin/vllm" {
|
|
t.Errorf("Path=%q", cmd.Path)
|
|
}
|
|
if cmd.Args[0] != "serve" || cmd.Args[1] != "/models/llama" {
|
|
t.Errorf("args start wrong: %v", cmd.Args[:2])
|
|
}
|
|
if argValue(cmd.Args, "--tensor-parallel-size") != "4" {
|
|
t.Errorf("TP not passed: %v", cmd.Args)
|
|
}
|
|
if argValue(cmd.Args, "--port") != "20000" {
|
|
t.Errorf("port not passed: %v", cmd.Args)
|
|
}
|
|
// Single node means no distributed backend flag at all.
|
|
if slices.Contains(cmd.Args, "--distributed-executor-backend") {
|
|
t.Error("a distributed backend was selected for a single-node profile")
|
|
}
|
|
}
|
|
|
|
func TestVLLMRayMode(t *testing.T) {
|
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
|
cmd, err := v.BuildCommand(multiNodeSpec(1))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if argValue(cmd.Args, "--distributed-executor-backend") != "ray" {
|
|
t.Errorf("backend not set: %v", cmd.Args)
|
|
}
|
|
if got := envMap(cmd.Env)["RAY_ADDRESS"]; got != "10.0.0.10:20001" {
|
|
t.Errorf("RAY_ADDRESS=%q", got)
|
|
}
|
|
}
|
|
|
|
func TestVLLMMPMode(t *testing.T) {
|
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
|
cmd, err := v.BuildCommand(multiNodeSpec(0))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if argValue(cmd.Args, "--distributed-executor-backend") != "mp" {
|
|
t.Errorf("backend not set: %v", cmd.Args)
|
|
}
|
|
env := envMap(cmd.Env)
|
|
if env["VLLM_HOST_IP"] != "10.0.0.10" || env["VLLM_PORT"] != "20001" {
|
|
t.Errorf("dist init not configured: %q %q", env["VLLM_HOST_IP"], env["VLLM_PORT"])
|
|
}
|
|
}
|
|
|
|
func TestVLLMUnknownLaunchMode(t *testing.T) {
|
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "sorcery"}}
|
|
if _, err := v.BuildCommand(multiNodeSpec(0)); err == nil {
|
|
t.Error("an unknown launch mode was accepted")
|
|
}
|
|
}
|
|
|
|
func TestVLLMRayCommand(t *testing.T) {
|
|
v := &VLLM{cfg: VLLMConfig{LaunchMode: "ray"}}
|
|
|
|
head, ok := v.RayCommand(multiNodeSpec(0))
|
|
if !ok {
|
|
t.Fatal("no ray command for the leader")
|
|
}
|
|
if !slices.Contains(head.Args, "--head") {
|
|
t.Errorf("rank 0 must start the head: %v", head.Args)
|
|
}
|
|
|
|
worker, ok := v.RayCommand(multiNodeSpec(1))
|
|
if !ok {
|
|
t.Fatal("no ray command for the worker")
|
|
}
|
|
if slices.Contains(worker.Args, "--head") {
|
|
t.Errorf("rank 1 must not start a head: %v", worker.Args)
|
|
}
|
|
if argValue(worker.Args, "--address") != "10.0.0.10:20001" {
|
|
t.Errorf("worker address wrong: %v", worker.Args)
|
|
}
|
|
|
|
// Single node needs no Ray at all.
|
|
if _, ok := v.RayCommand(singleNodeSpec()); ok {
|
|
t.Error("a single-node profile was given a Ray command")
|
|
}
|
|
// Neither does mp mode.
|
|
mp := &VLLM{cfg: VLLMConfig{LaunchMode: "mp"}}
|
|
if _, ok := mp.RayCommand(multiNodeSpec(0)); ok {
|
|
t.Error("mp mode was given a Ray command")
|
|
}
|
|
}
|
|
|
|
// --- SGLang ---
|
|
|
|
func TestSGLangMultiNodeArgs(t *testing.T) {
|
|
cmd, err := (&SGLang{cfg: SGLangConfig{Python: "/opt/sglang/bin/python"}}).BuildCommand(multiNodeSpec(1))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cmd.Path != "/opt/sglang/bin/python" {
|
|
t.Errorf("Path=%q", cmd.Path)
|
|
}
|
|
if cmd.Args[0] != "-m" || cmd.Args[1] != "sglang.launch_server" {
|
|
t.Errorf("not launched as a module: %v", cmd.Args[:2])
|
|
}
|
|
if argValue(cmd.Args, "--nnodes") != "2" || argValue(cmd.Args, "--node-rank") != "1" {
|
|
t.Errorf("multi-node flags wrong: %v", cmd.Args)
|
|
}
|
|
if argValue(cmd.Args, "--dist-init-addr") != "10.0.0.10:20001" {
|
|
t.Errorf("dist-init-addr wrong: %v", cmd.Args)
|
|
}
|
|
}
|
|
|
|
func TestSGLangSingleNodeOmitsMultiNodeFlags(t *testing.T) {
|
|
cmd, _ := (&SGLang{}).BuildCommand(singleNodeSpec())
|
|
for _, flag := range []string{"--nnodes", "--node-rank", "--dist-init-addr"} {
|
|
if slices.Contains(cmd.Args, flag) {
|
|
t.Errorf("%s was passed for a single-node profile", flag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProfileArgsComeLast(t *testing.T) {
|
|
// Operators must be able to override anything the adapter generated.
|
|
s := singleNodeSpec()
|
|
s.Profile.Args = []string{"--max-model-len", "32768"}
|
|
cmd, _ := (&VLLM{}).BuildCommand(s)
|
|
|
|
i := slices.Index(cmd.Args, "--max-model-len")
|
|
j := slices.Index(cmd.Args, "--tensor-parallel-size")
|
|
if i < j {
|
|
t.Errorf("profile args came before generated ones: %v", cmd.Args)
|
|
}
|
|
}
|
|
|
|
// --- validation ---
|
|
|
|
func h100(n int) []GPUInfo {
|
|
out := make([]GPUInfo, n)
|
|
for i := range out {
|
|
out[i] = GPUInfo{Index: i, Name: "NVIDIA H100 80GB HBM3", ComputeCapability: "9.0", MemoryTotalMB: 81559}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestValidateAcceptsMatchingProfile(t *testing.T) {
|
|
p := Profile{Name: "ok", TP: 4, ModelPath: "/m"}
|
|
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
|
|
t.Errorf("a valid profile was rejected: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsMixedGPUTypes(t *testing.T) {
|
|
// The invariant from plan §1.1: a TP group never mixes GPU types.
|
|
gpus := h100(2)
|
|
gpus[1].Name = "NVIDIA A100-SXM4-40GB"
|
|
|
|
err := (&VLLM{}).Validate(Profile{Name: "mixed", TP: 2, ModelPath: "/m"}, gpus)
|
|
if err == nil || !strings.Contains(err.Error(), "may not mix GPU types") {
|
|
t.Errorf("mixed GPU types accepted: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsFP8OnOldHardware(t *testing.T) {
|
|
gpus := h100(4)
|
|
for i := range gpus {
|
|
gpus[i].Name = "NVIDIA A100-SXM4-80GB"
|
|
gpus[i].ComputeCapability = "8.0"
|
|
}
|
|
p := Profile{Name: "fp8", TP: 4, ModelPath: "/m", Args: []string{"--quantization", "fp8"}}
|
|
|
|
err := (&VLLM{}).Validate(p, gpus)
|
|
if err == nil || !strings.Contains(err.Error(), "compute capability") {
|
|
t.Errorf("fp8 accepted on compute capability 8.0: %v", err)
|
|
}
|
|
|
|
// The same profile is fine on Hopper.
|
|
if err := (&VLLM{}).Validate(p, h100(4)); err != nil {
|
|
t.Errorf("fp8 rejected on H100: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsInconsistentParallelism(t *testing.T) {
|
|
// 2 nodes x 8 GPUs cannot serve TP 4.
|
|
p := Profile{Name: "bad", TP: 4, Nodes: 2, GPUsPerNode: 8, ModelPath: "/m"}
|
|
if err := (&VLLM{}).Validate(p, h100(8)); err == nil {
|
|
t.Error("inconsistent node/GPU/TP arithmetic accepted")
|
|
}
|
|
}
|
|
|
|
func TestValidateRejectsTooFewGPUs(t *testing.T) {
|
|
if err := (&VLLM{}).Validate(Profile{Name: "big", TP: 8, ModelPath: "/m"}, h100(4)); err == nil {
|
|
t.Error("a TP 8 profile was accepted on a 4-GPU node")
|
|
}
|
|
}
|
|
|
|
func TestValidateRequiresModelPath(t *testing.T) {
|
|
if err := (&VLLM{}).Validate(Profile{Name: "nopath", TP: 1}, h100(1)); err == nil {
|
|
t.Error("a profile without a model path was accepted")
|
|
}
|
|
}
|
|
|
|
func TestRegistry(t *testing.T) {
|
|
r := NewRegistry(Config{})
|
|
if got := r.Names(); len(got) != 2 || got[0] != "sglang" || got[1] != "vllm" {
|
|
t.Errorf("registry contents: %v", got)
|
|
}
|
|
if _, err := r.Get("vllm"); err != nil {
|
|
t.Errorf("vllm missing: %v", err)
|
|
}
|
|
err := func() error { _, err := r.Get("nope"); return err }()
|
|
if err == nil || !strings.Contains(err.Error(), "sglang, vllm") {
|
|
t.Errorf("an unknown engine should list what is available: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestQuantizationOf(t *testing.T) {
|
|
for _, args := range [][]string{
|
|
{"--quantization", "fp8"},
|
|
{"--quantization=fp8"},
|
|
{"--max-model-len", "100", "--quantization", "fp8"},
|
|
} {
|
|
if got := quantizationOf(args); got != "fp8" {
|
|
t.Errorf("quantizationOf(%v)=%q", args, got)
|
|
}
|
|
}
|
|
if got := quantizationOf([]string{"--quantization"}); got != "" {
|
|
t.Errorf("a dangling flag should yield nothing, got %q", got)
|
|
}
|
|
}
|