Files
llmgw/internal/slurm/slurm_test.go
T
Ilker ManapandClaude Opus 5.5 5db920bf2c
CI / test (push) Successful in 10m34s
Slurm: a job that asks for no memory gets a share, not the whole node
Tried on the PoC: five one-second jobs on an 8-CPU node started 10 s
apart. Each asked for no memory, so each was given the node's all, ran
alone, and the next waited for backfill's pass. Not accounting, as
thought. The partition's DefMemPerCPU is now the leanest node's memory
per CPU.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 23:11:24 +01:00

154 lines
5.0 KiB
Go

package slurm
import (
"bytes"
"strings"
"testing"
)
func TestMungeKey(t *testing.T) {
s := strings.Repeat("ab", 32)
a, err := MungeKey(s)
if err != nil {
t.Fatal(err)
}
b, _ := MungeKey(s)
if len(a) != MungeKeyLen || !bytes.Equal(a, b) {
t.Fatalf("a key is %d bytes and the same each time: got %d, equal %v", MungeKeyLen, len(a), bytes.Equal(a, b))
}
c, _ := MungeKey(strings.Repeat("cd", 32))
if bytes.Equal(a, c) {
t.Fatal("two secrets made one key")
}
if _, err := MungeKey("short"); err == nil {
t.Fatal("a short secret was taken")
}
}
func TestParseHardware(t *testing.T) {
out := "NodeName=cuda01 CPUs=16 Boards=1 SocketsPerBoard=2 CoresPerSocket=4 ThreadsPerCore=2 RealMemory=64302\n" +
"UpTime=3-01:02:03\n"
h, err := ParseHardware(out)
if err != nil {
t.Fatal(err)
}
want := Hardware{CPUs: 16, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 4, ThreadsPerCore: 2, RealMemory: 64302}
if h != want {
t.Fatalf("got %+v", h)
}
if _, err := ParseHardware("slurmd: error: something\n"); err == nil {
t.Fatal("an error was read as hardware")
}
}
func TestGPUType(t *testing.T) {
for name, want := range map[string]string{
"NVIDIA A100-SXM4-80GB": "a100",
"NVIDIA H100 80GB HBM3": "h100",
"Tesla V100-SXM2-32GB": "v100",
"Tesla T4": "t4",
"NVIDIA GeForce RTX 3090": "rtx3090",
"NVIDIA RTX A6000": "rtxa6000",
"NVIDIA L40S": "l40s",
"Quadro RTX 8000": "rtx8000",
"NVIDIA GeForce GTX 1080": "gtx1080",
"": "gpu",
"Some Accelerator Unknown": "gpu",
} {
if got := GPUType(name); got != want {
t.Errorf("GPUType(%q) = %q, want %q", name, got, want)
}
}
}
func testCluster() Cluster {
return Cluster{Name: "llmgw", Controller: "node06", ControllerAddr: "10.0.0.146", Partition: "gpu",
Nodes: []Node{
{Name: "site2", Addr: "10.1.0.5",
Hardware: Hardware{CPUs: 32, Boards: 1, SocketsPerBoard: 2, CoresPerSocket: 8, ThreadsPerCore: 2, RealMemory: 100000},
GPUs: []GPU{{Minor: 2, Name: "Tesla T4"}, {Minor: 0, Name: "Tesla V100-SXM2-32GB"}, {Minor: 1, Name: "Tesla V100-SXM2-32GB"}}},
{Name: "cuda01", Addr: "10.0.0.147", Hardware: Hardware{CPUs: 16, RealMemory: 64000},
GPUs: []GPU{{Minor: 0, Name: "NVIDIA GeForce RTX 3090"}}},
{Name: "waiting", Addr: "10.0.0.9"},
}}
}
func TestSlurmConf(t *testing.T) {
conf := testCluster().SlurmConf()
for _, want := range []string{
"ClusterName=llmgw\n",
"SlurmctldHost=node06(10.0.0.146)\n",
"SlurmctldParameters=enable_configless\n",
"NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"NodeName=site2 NodeAddr=10.1.0.5 CPUs=32 Boards=1 SocketsPerBoard=2 CoresPerSocket=8 ThreadsPerCore=2 RealMemory=97000 Gres=gpu:t4:1,gpu:v100:2 State=DRAIN Reason=\"llmgw: not lent yet\"\n",
"PartitionName=gpu Nodes=cuda01,site2 Default=YES",
// site2's share, the leaner: 97000 MiB for 32 CPUs.
" DefMemPerCPU=3031\n",
} {
if !strings.Contains(conf, want) {
t.Errorf("slurm.conf has no %q:\n%s", want, conf)
}
}
if strings.Contains(conf, "waiting") {
t.Error("a node that has not reported its hardware is in slurm.conf")
}
if strings.Index(conf, "NodeName=cuda01") > strings.Index(conf, "NodeName=site2") {
t.Error("nodes are not in order")
}
}
func TestSlurmConfNoNodes(t *testing.T) {
c := testCluster()
c.Nodes = nil
if strings.Contains(c.SlurmConf(), "PartitionName") {
t.Error("a partition with no nodes would stop slurmctld starting")
}
}
func TestGresConf(t *testing.T) {
g := testCluster().GresConf()
for _, want := range []string{
"NodeName=cuda01 Name=gpu Type=rtx3090 File=/dev/nvidia0\n",
"NodeName=site2 Name=gpu Type=t4 File=/dev/nvidia2\n",
"NodeName=site2 Name=gpu Type=v100 File=/dev/nvidia0,/dev/nvidia1\n",
} {
if !strings.Contains(g, want) {
t.Errorf("gres.conf has no %q:\n%s", want, g)
}
}
}
func TestParseNodeState(t *testing.T) {
out := "NodeName=cuda01 Arch=x86_64 CoresPerSocket=8 CPUAlloc=0 State=IDLE+DRAIN ThreadsPerCore=2 " +
"Partitions=gpu Reason=llmgw: serving models [root@2026-11-11T10:00:00]\n"
s, err := ParseNodeState(out)
if err != nil {
t.Fatal(err)
}
if s.State != "IDLE+DRAIN" || s.Reason != "llmgw: serving models" || !s.Drained() {
t.Fatalf("got %+v", s)
}
s, _ = ParseNodeState("NodeName=cuda01 State=MIXED Partitions=gpu\n")
if s.Drained() || s.Reason != "" {
t.Fatalf("got %+v", s)
}
if _, err := ParseNodeState("Node cuda02 not found\n"); err == nil {
t.Fatal("an unknown node was read")
}
}
func TestParseVersion(t *testing.T) {
for out, want := range map[string][2]int{"slurm 23.11.4\n": {23, 11}, "slurm-wlm 21.08.5": {21, 8}, "slurm 22.05.9": {22, 5}} {
v, err := ParseVersion(out)
if err != nil || v != want {
t.Errorf("ParseVersion(%q) = %v, %v", out, v, err)
}
}
if !TooOld([2]int{21, 8}) || TooOld([2]int{22, 5}) || TooOld([2]int{23, 2}) {
t.Error("TooOld is wrong")
}
if _, err := ParseVersion("command not found"); err == nil {
t.Error("no version was read as one")
}
}