package slurmsvc import ( "context" "strings" "testing" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) type fakeInv struct { nodes map[string]*inventory.Node allocs map[string]int } func (f *fakeInv) Get(id string) (inventory.Node, bool) { n, ok := f.nodes[id] if !ok { return inventory.Node{}, false } return *n, true } func (f *fakeInv) SetState(id string, st inventory.NodeState) bool { n, ok := f.nodes[id] if ok { n.State = st } return ok } func (f *fakeInv) Allocations(id string) []inventory.Allocation { return make([]inventory.Allocation, f.allocs[id]) } type fakeAgents struct { conn []string got map[string][]*llmgwv1.Slurm } func (f *fakeAgents) Send(node string, m *llmgwv1.ControlMessage) error { f.got[node] = append(f.got[node], m.GetSlurm()) return nil } func (f *fakeAgents) ConnectedNodes() []string { return f.conn } func (f *fakeAgents) last(node string) *llmgwv1.Slurm { ms := f.got[node] if len(ms) == 0 { return nil } return ms[len(ms)-1] } type rig struct { set config.SlurmSettings inv *fakeInv agents *fakeAgents svc *Service } func newRig() *rig { r := &rig{ set: config.SlurmSettings{Enabled: true, Controller: "node06", Nodes: []string{"cuda01"}}, inv: &fakeInv{nodes: map[string]*inventory.Node{ "node06": {ID: "node06", MgmtAddr: "10.0.0.146"}, "cuda01": {ID: "cuda01", MgmtAddr: "10.0.0.147", GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA GeForce RTX 3090"}}}, }, allocs: map[string]int{"cuda01": 2}}, agents: &fakeAgents{conn: []string{"node06", "cuda01"}, got: map[string][]*llmgwv1.Slurm{}}, } r.svc = New(func() config.SlurmSettings { return r.set }, r.inv, r.agents, func(context.Context) ([]byte, error) { return []byte(strings.Repeat("k", 1024)), nil }, nil) return r } func (r *rig) report(node string, st *llmgwv1.SlurmStatus) { r.svc.OnSlurmStatus(node, st) } func (r *rig) tick() { r.svc.Tick(context.Background()) } const hw = "NodeName=cuda01 CPUs=16 RealMemory=64000" func TestMessagesAndFiles(t *testing.T) { r := newRig() r.tick() ctl, node := r.agents.last("node06"), r.agents.last("cuda01") if !ctl.GetController() || ctl.GetNode() || len(ctl.GetMungeKey()) != 1024 { t.Fatalf("controller told %v", ctl) } if !node.GetNode() || node.GetController() || node.GetConfServer() != "10.0.0.146:6817" || node.GetLent() || node.GetName() != "cuda01" || len(node.GetFiles()) != 0 { t.Fatalf("node told %v", node) } if strings.Contains(ctl.GetFiles()["slurm.conf"], "NodeName=cuda01") { t.Fatal("a node with no hardware reported is in slurm.conf") } if w := r.svc.Status().Waiting; len(w) != 1 || w[0] != "cuda01" { t.Fatalf("waiting %v", w) } // Same again: nobody is told twice. r.tick() if len(r.agents.got["node06"]) != 1 || len(r.agents.got["cuda01"]) != 1 { t.Fatal("told again with nothing changed") } r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"}) r.tick() conf := r.agents.last("node06").GetFiles()["slurm.conf"] if !strings.Contains(conf, "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1") { t.Fatalf("slurm.conf:\n%s", conf) } if !strings.Contains(r.agents.last("node06").GetFiles()["gres.conf"], "File=/dev/nvidia0") { t.Fatal("gres.conf has no card") } if len(r.agents.got["cuda01"]) != 1 { t.Fatal("a node was told again when only the controller's files changed") } // A new stream is told again. r.svc.OnRegister("cuda01") r.tick() if len(r.agents.got["cuda01"]) != 2 { t.Fatal("a new stream was not told") } if r.inv.nodes["cuda01"].State != inventory.Ready { t.Fatal("a node drained in Slurm with no job was taken from llmgw") } } func TestLendAndReclaim(t *testing.T) { r := newRig() r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"}) r.set.Lent = []string{"cuda01"} r.tick() if r.inv.nodes["cuda01"].State != inventory.Draining { t.Fatal("a lent node was not drained of its replicas") } if r.agents.last("cuda01").GetLent() { t.Fatal("lent to Slurm with replicas still on it") } if p := r.svc.Status().Nodes[0].Phase; p != Lending { t.Fatalf("phase %s", p) } r.inv.allocs["cuda01"] = 0 r.tick() if !r.agents.last("cuda01").GetLent() || r.svc.Status().Nodes[0].Phase != Lent { t.Fatal("not lent once its replicas were gone") } r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED", Jobs: 3}) // Taken back: Slurm drains it; llmgw keeps off until its jobs end. r.set.Lent = nil r.tick() if r.agents.last("cuda01").GetLent() { t.Fatal("still lent") } if s := r.inv.nodes["cuda01"].State; s != inventory.Draining && s != inventory.Cordoned { t.Fatalf("llmgw may place on a node with Slurm jobs: %v", s) } r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED+DRAIN", Jobs: 1}) r.tick() if r.svc.Status().Nodes[0].Phase != Reclaiming || r.inv.nodes["cuda01"].State == inventory.Ready { t.Fatal("given back with a job left") } r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: serving models"}) r.tick() if r.inv.nodes["cuda01"].State != inventory.Ready || r.svc.Status().Nodes[0].Phase != Llmgw { t.Fatal("not llmgw's when its last job ended") } } func TestNewLeaderKeepsOffUntilHeard(t *testing.T) { r := newRig() r.inv.allocs["cuda01"] = 0 r.tick() // Nothing heard yet: cordoned, nothing moved. if r.inv.nodes["cuda01"].State != inventory.Cordoned { t.Fatalf("state %v before the agent said anything", r.inv.nodes["cuda01"].State) } r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "MIXED", Jobs: 2}) r.tick() if r.inv.nodes["cuda01"].State == inventory.Ready { t.Fatal("ready with Slurm jobs on it") } } func TestBrokenSlurmLeavesTheNodeToLlmgw(t *testing.T) { r := newRig() r.report("cuda01", &llmgwv1.SlurmStatus{Error: "installing Slurm: no package", Jobs: 0}) r.tick() if r.inv.nodes["cuda01"].State != inventory.Ready { t.Fatal("a Slurm that does not work took a node from llmgw") } } func TestOperatorsCordonIsLeft(t *testing.T) { r := newRig() r.inv.nodes["cuda01"].State = inventory.Cordoned r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN"}) r.tick() if r.inv.nodes["cuda01"].State != inventory.Cordoned { t.Fatal("llmgw uncordoned a node an operator cordoned") } } func TestOffAndOut(t *testing.T) { r := newRig() r.inv.allocs["cuda01"] = 0 r.set.Lent = []string{"cuda01"} r.tick() r.set.Enabled = false r.tick() if m := r.agents.last("cuda01"); m.GetNode() || m.GetController() { t.Fatalf("told %v with Slurm off", m) } if r.inv.nodes["cuda01"].State != inventory.Ready { t.Fatal("not llmgw's with Slurm off") } r = newRig() r.inv.allocs["cuda01"] = 0 r.set.Lent = []string{"cuda01"} r.tick() r.set.Nodes, r.set.Lent = nil, nil r.tick() if r.inv.nodes["cuda01"].State != inventory.Ready || r.agents.last("cuda01").GetNode() { t.Fatal("a node taken out of slurm.nodes is not llmgw's") } if !r.agents.last("node06").GetController() { t.Fatal("the controller stopped with no nodes") } } func TestNoControllerYet(t *testing.T) { r := newRig() delete(r.inv.nodes, "node06") r.agents.conn = []string{"cuda01"} r.tick() if m := r.agents.last("cuda01"); m.GetNode() { t.Fatalf("a node was started with no controller to fetch its files from: %v", m) } }