CI / test (push) Successful in 9m40s
internal/control/slurmsvc tells each node what it is, writes slurm.conf and gres.conf from the nodes' reports and the inventory, and moves each node of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until its last job step ends. Kept every tick from the settings and the agents' reports, so a new leader carries on. A broken Slurm never keeps a node from llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
244 lines
7.4 KiB
Go
244 lines
7.4 KiB
Go
package slurmsvc
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
type fakeInv struct {
|
|
nodes map[string]*inventory.Node
|
|
allocs map[string]int
|
|
}
|
|
|
|
func (f *fakeInv) Get(id string) (inventory.Node, bool) {
|
|
n, ok := f.nodes[id]
|
|
if !ok {
|
|
return inventory.Node{}, false
|
|
}
|
|
return *n, true
|
|
}
|
|
|
|
func (f *fakeInv) SetState(id string, st inventory.NodeState) bool {
|
|
n, ok := f.nodes[id]
|
|
if ok {
|
|
n.State = st
|
|
}
|
|
return ok
|
|
}
|
|
|
|
func (f *fakeInv) Allocations(id string) []inventory.Allocation {
|
|
return make([]inventory.Allocation, f.allocs[id])
|
|
}
|
|
|
|
type fakeAgents struct {
|
|
conn []string
|
|
got map[string][]*llmgwv1.Slurm
|
|
}
|
|
|
|
func (f *fakeAgents) Send(node string, m *llmgwv1.ControlMessage) error {
|
|
f.got[node] = append(f.got[node], m.GetSlurm())
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeAgents) ConnectedNodes() []string { return f.conn }
|
|
|
|
func (f *fakeAgents) last(node string) *llmgwv1.Slurm {
|
|
ms := f.got[node]
|
|
if len(ms) == 0 {
|
|
return nil
|
|
}
|
|
return ms[len(ms)-1]
|
|
}
|
|
|
|
type rig struct {
|
|
set config.SlurmSettings
|
|
inv *fakeInv
|
|
agents *fakeAgents
|
|
svc *Service
|
|
}
|
|
|
|
func newRig() *rig {
|
|
r := &rig{
|
|
set: config.SlurmSettings{Enabled: true, Controller: "node06", Nodes: []string{"cuda01"}},
|
|
inv: &fakeInv{nodes: map[string]*inventory.Node{
|
|
"node06": {ID: "node06", MgmtAddr: "10.0.0.146"},
|
|
"cuda01": {ID: "cuda01", MgmtAddr: "10.0.0.147", GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA GeForce RTX 3090"}}},
|
|
}, allocs: map[string]int{"cuda01": 2}},
|
|
agents: &fakeAgents{conn: []string{"node06", "cuda01"}, got: map[string][]*llmgwv1.Slurm{}},
|
|
}
|
|
r.svc = New(func() config.SlurmSettings { return r.set }, r.inv, r.agents,
|
|
func(context.Context) ([]byte, error) { return []byte(strings.Repeat("k", 1024)), nil }, nil)
|
|
return r
|
|
}
|
|
|
|
func (r *rig) report(node string, st *llmgwv1.SlurmStatus) { r.svc.OnSlurmStatus(node, st) }
|
|
|
|
func (r *rig) tick() { r.svc.Tick(context.Background()) }
|
|
|
|
const hw = "NodeName=cuda01 CPUs=16 RealMemory=64000"
|
|
|
|
func TestMessagesAndFiles(t *testing.T) {
|
|
r := newRig()
|
|
r.tick()
|
|
ctl, node := r.agents.last("node06"), r.agents.last("cuda01")
|
|
if !ctl.GetController() || ctl.GetNode() || len(ctl.GetMungeKey()) != 1024 {
|
|
t.Fatalf("controller told %v", ctl)
|
|
}
|
|
if !node.GetNode() || node.GetController() || node.GetConfServer() != "10.0.0.146:6817" || node.GetLent() ||
|
|
node.GetName() != "cuda01" || len(node.GetFiles()) != 0 {
|
|
t.Fatalf("node told %v", node)
|
|
}
|
|
if strings.Contains(ctl.GetFiles()["slurm.conf"], "NodeName=cuda01") {
|
|
t.Fatal("a node with no hardware reported is in slurm.conf")
|
|
}
|
|
if w := r.svc.Status().Waiting; len(w) != 1 || w[0] != "cuda01" {
|
|
t.Fatalf("waiting %v", w)
|
|
}
|
|
// Same again: nobody is told twice.
|
|
r.tick()
|
|
if len(r.agents.got["node06"]) != 1 || len(r.agents.got["cuda01"]) != 1 {
|
|
t.Fatal("told again with nothing changed")
|
|
}
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"})
|
|
r.tick()
|
|
conf := r.agents.last("node06").GetFiles()["slurm.conf"]
|
|
if !strings.Contains(conf, "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1") {
|
|
t.Fatalf("slurm.conf:\n%s", conf)
|
|
}
|
|
if !strings.Contains(r.agents.last("node06").GetFiles()["gres.conf"], "File=/dev/nvidia0") {
|
|
t.Fatal("gres.conf has no card")
|
|
}
|
|
if len(r.agents.got["cuda01"]) != 1 {
|
|
t.Fatal("a node was told again when only the controller's files changed")
|
|
}
|
|
// A new stream is told again.
|
|
r.svc.OnRegister("cuda01")
|
|
r.tick()
|
|
if len(r.agents.got["cuda01"]) != 2 {
|
|
t.Fatal("a new stream was not told")
|
|
}
|
|
if r.inv.nodes["cuda01"].State != inventory.Ready {
|
|
t.Fatal("a node drained in Slurm with no job was taken from llmgw")
|
|
}
|
|
}
|
|
|
|
func TestLendAndReclaim(t *testing.T) {
|
|
r := newRig()
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"})
|
|
r.set.Lent = []string{"cuda01"}
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State != inventory.Draining {
|
|
t.Fatal("a lent node was not drained of its replicas")
|
|
}
|
|
if r.agents.last("cuda01").GetLent() {
|
|
t.Fatal("lent to Slurm with replicas still on it")
|
|
}
|
|
if p := r.svc.Status().Nodes[0].Phase; p != Lending {
|
|
t.Fatalf("phase %s", p)
|
|
}
|
|
r.inv.allocs["cuda01"] = 0
|
|
r.tick()
|
|
if !r.agents.last("cuda01").GetLent() || r.svc.Status().Nodes[0].Phase != Lent {
|
|
t.Fatal("not lent once its replicas were gone")
|
|
}
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED", Jobs: 3})
|
|
|
|
// Taken back: Slurm drains it; llmgw keeps off until its jobs end.
|
|
r.set.Lent = nil
|
|
r.tick()
|
|
if r.agents.last("cuda01").GetLent() {
|
|
t.Fatal("still lent")
|
|
}
|
|
if s := r.inv.nodes["cuda01"].State; s != inventory.Draining && s != inventory.Cordoned {
|
|
t.Fatalf("llmgw may place on a node with Slurm jobs: %v", s)
|
|
}
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED+DRAIN", Jobs: 1})
|
|
r.tick()
|
|
if r.svc.Status().Nodes[0].Phase != Reclaiming || r.inv.nodes["cuda01"].State == inventory.Ready {
|
|
t.Fatal("given back with a job left")
|
|
}
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: serving models"})
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State != inventory.Ready || r.svc.Status().Nodes[0].Phase != Llmgw {
|
|
t.Fatal("not llmgw's when its last job ended")
|
|
}
|
|
}
|
|
|
|
func TestNewLeaderKeepsOffUntilHeard(t *testing.T) {
|
|
r := newRig()
|
|
r.inv.allocs["cuda01"] = 0
|
|
r.tick()
|
|
// Nothing heard yet: cordoned, nothing moved.
|
|
if r.inv.nodes["cuda01"].State != inventory.Cordoned {
|
|
t.Fatalf("state %v before the agent said anything", r.inv.nodes["cuda01"].State)
|
|
}
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "MIXED", Jobs: 2})
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State == inventory.Ready {
|
|
t.Fatal("ready with Slurm jobs on it")
|
|
}
|
|
}
|
|
|
|
func TestBrokenSlurmLeavesTheNodeToLlmgw(t *testing.T) {
|
|
r := newRig()
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Error: "installing Slurm: no package", Jobs: 0})
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State != inventory.Ready {
|
|
t.Fatal("a Slurm that does not work took a node from llmgw")
|
|
}
|
|
}
|
|
|
|
func TestOperatorsCordonIsLeft(t *testing.T) {
|
|
r := newRig()
|
|
r.inv.nodes["cuda01"].State = inventory.Cordoned
|
|
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN"})
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State != inventory.Cordoned {
|
|
t.Fatal("llmgw uncordoned a node an operator cordoned")
|
|
}
|
|
}
|
|
|
|
func TestOffAndOut(t *testing.T) {
|
|
r := newRig()
|
|
r.inv.allocs["cuda01"] = 0
|
|
r.set.Lent = []string{"cuda01"}
|
|
r.tick()
|
|
r.set.Enabled = false
|
|
r.tick()
|
|
if m := r.agents.last("cuda01"); m.GetNode() || m.GetController() {
|
|
t.Fatalf("told %v with Slurm off", m)
|
|
}
|
|
if r.inv.nodes["cuda01"].State != inventory.Ready {
|
|
t.Fatal("not llmgw's with Slurm off")
|
|
}
|
|
|
|
r = newRig()
|
|
r.inv.allocs["cuda01"] = 0
|
|
r.set.Lent = []string{"cuda01"}
|
|
r.tick()
|
|
r.set.Nodes, r.set.Lent = nil, nil
|
|
r.tick()
|
|
if r.inv.nodes["cuda01"].State != inventory.Ready || r.agents.last("cuda01").GetNode() {
|
|
t.Fatal("a node taken out of slurm.nodes is not llmgw's")
|
|
}
|
|
if !r.agents.last("node06").GetController() {
|
|
t.Fatal("the controller stopped with no nodes")
|
|
}
|
|
}
|
|
|
|
func TestNoControllerYet(t *testing.T) {
|
|
r := newRig()
|
|
delete(r.inv.nodes, "node06")
|
|
r.agents.conn = []string{"cuda01"}
|
|
r.tick()
|
|
if m := r.agents.last("cuda01"); m.GetNode() {
|
|
t.Fatalf("a node was started with no controller to fetch its files from: %v", m)
|
|
}
|
|
}
|