Files
llmgw/internal/slurm/slurm_test.go
T
Ilker ManapandClaude Opus 5.5 d439d507d7 Slurm beside llmgw, P1: its settings and the files a controller runs on
slurm.{enabled,cluster_name,controller,nodes,lent,partition} in the cluster's
settings, checked; internal/slurm writes slurm.conf (configless), gres.conf
and cgroup.conf from the nodes' slurmd -C lines and GPUs, derives the munge
key from a cluster secret, and reads scontrol and version output.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:20:31 +01:00

152 lines
4.9 KiB
Go

package slurm
import (
"bytes"
"strings"
"testing"
)
func TestMungeKey(t *testing.T) {
s := strings.Repeat("ab", 32)
a, err := MungeKey(s)
if err != nil {
t.Fatal(err)
}
b, _ := MungeKey(s)
if len(a) != MungeKeyLen || !bytes.Equal(a, b) {
t.Fatalf("a key is %d bytes and the same each time: got %d, equal %v", MungeKeyLen, len(a), bytes.Equal(a, b))
}
c, _ := MungeKey(strings.Repeat("cd", 32))
if bytes.Equal(a, c) {
t.Fatal("two secrets made one key")
}
if _, err := MungeKey("short"); err == nil {
t.Fatal("a short secret was taken")
}
}
func TestParseHardware(t *testing.T) {
out := "NodeName=cuda01 CPUs=16 Boards=1 SocketsPerBoard=2 CoresPerSocket=4 ThreadsPerCore=2 RealMemory=64302\n" +
"UpTime=3-01:02:03\n"
h, err := ParseHardware(out)
if err != nil {
t.Fatal(err)
}
want := Hardware{CPUs: 16, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 4, ThreadsPerCore: 2, RealMemory: 64302}
if h != want {
t.Fatalf("got %+v", h)
}
if _, err := ParseHardware("slurmd: error: something\n"); err == nil {
t.Fatal("an error was read as hardware")
}
}
func TestGPUType(t *testing.T) {
for name, want := range map[string]string{
"NVIDIA A100-SXM4-80GB": "a100",
"NVIDIA H100 80GB HBM3": "h100",
"Tesla V100-SXM2-32GB": "v100",
"Tesla T4": "t4",
"NVIDIA GeForce RTX 3090": "rtx3090",
"NVIDIA RTX A6000": "rtxa6000",
"NVIDIA L40S": "l40s",
"Quadro RTX 8000": "rtx8000",
"NVIDIA GeForce GTX 1080": "gtx1080",
"": "gpu",
"Some Accelerator Unknown": "gpu",
} {
if got := GPUType(name); got != want {
t.Errorf("GPUType(%q) = %q, want %q", name, got, want)
}
}
}
func testCluster() Cluster {
return Cluster{Name: "llmgw", Controller: "node06", ControllerAddr: "10.0.0.146", Partition: "gpu",
Nodes: []Node{
{Name: "site2", Addr: "10.1.0.5",
Hardware: Hardware{CPUs: 32, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 8, ThreadsPerCore: 2, RealMemory: 100000},
GPUs: []GPU{{Minor: 2, Name: "Tesla T4"}, {Minor: 0, Name: "Tesla V100-SXM2-32GB"}, {Minor: 1, Name: "Tesla V100-SXM2-32GB"}}},
{Name: "cuda01", Addr: "10.0.0.147", Hardware: Hardware{CPUs: 16, RealMemory: 64000},
GPUs: []GPU{{Minor: 0, Name: "NVIDIA GeForce RTX 3090"}}},
{Name: "waiting", Addr: "10.0.0.9"},
}}
}
func TestSlurmConf(t *testing.T) {
conf := testCluster().SlurmConf()
for _, want := range []string{
"ClusterName=llmgw\n",
"SlurmctldHost=node06(10.0.0.146)\n",
"SlurmctldParameters=enable_configless\n",
"NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=UNKNOWN\n",
"NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=UNKNOWN\n",
"PartitionName=gpu Nodes=cuda01,site2 Default=YES",
} {
if !strings.Contains(conf, want) {
t.Errorf("slurm.conf has no %q:\n%s", want, conf)
}
}
if strings.Contains(conf, "waiting") {
t.Error("a node that has not reported its hardware is in slurm.conf")
}
if strings.Index(conf, "NodeName=cuda01") > strings.Index(conf, "NodeName=site2") {
t.Error("nodes are not in order")
}
}
func TestSlurmConfNoNodes(t *testing.T) {
c := testCluster()
c.Nodes = nil
if strings.Contains(c.SlurmConf(), "PartitionName") {
t.Error("a partition with no nodes would stop slurmctld starting")
}
}
func TestGresConf(t *testing.T) {
g := testCluster().GresConf()
for _, want := range []string{
"NodeName=cuda01 Name=gpu Type=rtx3090 File=/dev/nvidia0\n",
"NodeName=site2 Name=gpu Type=t4 File=/dev/nvidia2\n",
"NodeName=site2 Name=gpu Type=v100 File=/dev/nvidia0,/dev/nvidia1\n",
} {
if !strings.Contains(g, want) {
t.Errorf("gres.conf has no %q:\n%s", want, g)
}
}
}
func TestParseNodeState(t *testing.T) {
out := "NodeName=cuda01 Arch=x86_64 CoresPerSocket=8 CPUAlloc=0 State=IDLE+DRAIN ThreadsPerCore=2 " +
"Partitions=gpu Reason=llmgw: serving models [root@2026-11-11T10:00:00]\n"
s, err := ParseNodeState(out)
if err != nil {
t.Fatal(err)
}
if s.State != "IDLE+DRAIN" || s.Reason != "llmgw: serving models" || !s.Drained() {
t.Fatalf("got %+v", s)
}
s, _ = ParseNodeState("NodeName=cuda01 State=MIXED Partitions=gpu\n")
if s.Drained() || s.Reason != "" {
t.Fatalf("got %+v", s)
}
if _, err := ParseNodeState("Node cuda02 not found\n"); err == nil {
t.Fatal("an unknown node was read")
}
}
func TestParseVersion(t *testing.T) {
for out, want := range map[string][2]int{"slurm 23.11.4\n": {23, 11}, "slurm-wlm 21.08.5": {21, 8}, "slurm 22.05.9": {22, 5}} {
v, err := ParseVersion(out)
if err != nil || v != want {
t.Errorf("ParseVersion(%q) = %v, %v", out, v, err)
}
}
if !TooOld([2]int{21, 8}) || TooOld([2]int{22, 5}) || TooOld([2]int{23, 2}) {
t.Error("TooOld is wrong")
}
if _, err := ParseVersion("command not found"); err == nil {
t.Error("no version was read as one")
}
}