Files
llmgw/internal/slurm/slurm_test.go
T
Ilker ManapandClaude Opus 5.5 35de47444c Slurm beside llmgw, P2: the agent keeps its node's part
A Slurm message says whether the node runs slurmctld on the leader's files,
slurmd configless, both or neither, and whether it is lent. The agent
installs munge and Slurm from the machine's sources (22.05 or newer, for
cgroup v2), writes the munge key, starts slurmd through a drop-in with
--conf-server, resumes a lent node that llmgw drained and drains one that is
not, never touching a drain a Slurm administrator made, and reports its
version, slurmd -C, state and jobs in SlurmStatus. Told it has no part, it
stops what it started. New nodes enter slurm.conf drained.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:25:41 +01:00

152 lines
4.9 KiB
Go

package slurm
import (
"bytes"
"strings"
"testing"
)
func TestMungeKey(t *testing.T) {
s := strings.Repeat("ab", 32)
a, err := MungeKey(s)
if err != nil {
t.Fatal(err)
}
b, _ := MungeKey(s)
if len(a) != MungeKeyLen || !bytes.Equal(a, b) {
t.Fatalf("a key is %d bytes and the same each time: got %d, equal %v", MungeKeyLen, len(a), bytes.Equal(a, b))
}
c, _ := MungeKey(strings.Repeat("cd", 32))
if bytes.Equal(a, c) {
t.Fatal("two secrets made one key")
}
if _, err := MungeKey("short"); err == nil {
t.Fatal("a short secret was taken")
}
}
func TestParseHardware(t *testing.T) {
out := "NodeName=cuda01 CPUs=16 Boards=1 SocketsPerBoard=2 CoresPerSocket=4 ThreadsPerCore=2 RealMemory=64302\n" +
"UpTime=3-01:02:03\n"
h, err := ParseHardware(out)
if err != nil {
t.Fatal(err)
}
want := Hardware{CPUs: 16, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 4, ThreadsPerCore: 2, RealMemory: 64302}
if h != want {
t.Fatalf("got %+v", h)
}
if _, err := ParseHardware("slurmd: error: something\n"); err == nil {
t.Fatal("an error was read as hardware")
}
}
func TestGPUType(t *testing.T) {
for name, want := range map[string]string{
"NVIDIA A100-SXM4-80GB": "a100",
"NVIDIA H100 80GB HBM3": "h100",
"Tesla V100-SXM2-32GB": "v100",
"Tesla T4": "t4",
"NVIDIA GeForce RTX 3090": "rtx3090",
"NVIDIA RTX A6000": "rtxa6000",
"NVIDIA L40S": "l40s",
"Quadro RTX 8000": "rtx8000",
"NVIDIA GeForce GTX 1080": "gtx1080",
"": "gpu",
"Some Accelerator Unknown": "gpu",
} {
if got := GPUType(name); got != want {
t.Errorf("GPUType(%q) = %q, want %q", name, got, want)
}
}
}
func testCluster() Cluster {
return Cluster{Name: "llmgw", Controller: "node06", ControllerAddr: "10.0.0.146", Partition: "gpu",
Nodes: []Node{
{Name: "site2", Addr: "10.1.0.5",
Hardware: Hardware{CPUs: 32, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 8, ThreadsPerCore: 2, RealMemory: 100000},
GPUs: []GPU{{Minor: 2, Name: "Tesla T4"}, {Minor: 0, Name: "Tesla V100-SXM2-32GB"}, {Minor: 1, Name: "Tesla V100-SXM2-32GB"}}},
{Name: "cuda01", Addr: "10.0.0.147", Hardware: Hardware{CPUs: 16, RealMemory: 64000},
GPUs: []GPU{{Minor: 0, Name: "NVIDIA GeForce RTX 3090"}}},
{Name: "waiting", Addr: "10.0.0.9"},
}}
}
func TestSlurmConf(t *testing.T) {
conf := testCluster().SlurmConf()
for _, want := range []string{
"ClusterName=llmgw\n",
"SlurmctldHost=node06(10.0.0.146)\n",
"SlurmctldParameters=enable_configless\n",
"NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"PartitionName=gpu Nodes=cuda01,site2 Default=YES",
} {
if !strings.Contains(conf, want) {
t.Errorf("slurm.conf has no %q:\n%s", want, conf)
}
}
if strings.Contains(conf, "waiting") {
t.Error("a node that has not reported its hardware is in slurm.conf")
}
if strings.Index(conf, "NodeName=cuda01") > strings.Index(conf, "NodeName=site2") {
t.Error("nodes are not in order")
}
}
func TestSlurmConfNoNodes(t *testing.T) {
c := testCluster()
c.Nodes = nil
if strings.Contains(c.SlurmConf(), "PartitionName") {
t.Error("a partition with no nodes would stop slurmctld starting")
}
}
func TestGresConf(t *testing.T) {
g := testCluster().GresConf()
for _, want := range []string{
"NodeName=cuda01 Name=gpu Type=rtx3090 File=/dev/nvidia0\n",
"NodeName=site2 Name=gpu Type=t4 File=/dev/nvidia2\n",
"NodeName=site2 Name=gpu Type=v100 File=/dev/nvidia0,/dev/nvidia1\n",
} {
if !strings.Contains(g, want) {
t.Errorf("gres.conf has no %q:\n%s", want, g)
}
}
}
func TestParseNodeState(t *testing.T) {
out := "NodeName=cuda01 Arch=x86_64 CoresPerSocket=8 CPUAlloc=0 State=IDLE+DRAIN ThreadsPerCore=2 " +
"Partitions=gpu Reason=llmgw: serving models [root@2026-11-11T10:00:00]\n"
s, err := ParseNodeState(out)
if err != nil {
t.Fatal(err)
}
if s.State != "IDLE+DRAIN" || s.Reason != "llmgw: serving models" || !s.Drained() {
t.Fatalf("got %+v", s)
}
s, _ = ParseNodeState("NodeName=cuda01 State=MIXED Partitions=gpu\n")
if s.Drained() || s.Reason != "" {
t.Fatalf("got %+v", s)
}
if _, err := ParseNodeState("Node cuda02 not found\n"); err == nil {
t.Fatal("an unknown node was read")
}
}
func TestParseVersion(t *testing.T) {
for out, want := range map[string][2]int{"slurm 23.11.4\n": {23, 11}, "slurm-wlm 21.08.5": {21, 8}, "slurm 22.05.9": {22, 5}} {
v, err := ParseVersion(out)
if err != nil || v != want {
t.Errorf("ParseVersion(%q) = %v, %v", out, v, err)
}
}
if !TooOld([2]int{21, 8}) || TooOld([2]int{22, 5}) || TooOld([2]int{23, 2}) {
t.Error("TooOld is wrong")
}
if _, err := ParseVersion("command not found"); err == nil {
t.Error("no version was read as one")
}
}