// Package slurmsvc is the leader's side of the Slurm cluster run beside // llmgw (BACKLOG P): it tells each node's agent what it is, writes the // controller's files from what the nodes report, and hands nodes between // llmgw and Slurm as the cluster's settings say. // // A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does, // and lent to Slurm once none is left. A node taken out of slurm.lent is // drained in Slurm, and is llmgw's again when its last job has ended. Both // are kept in force every tick, from the settings and what the agents say, // so a leader that takes over carries on where the last one was. package slurmsvc import ( "context" "fmt" "log/slog" "maps" "net" "slices" "strconv" "strings" "sync" "time" "google.golang.org/protobuf/proto" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) // Agents is how the service reaches the nodes. type Agents interface { Send(node string, msg *llmgwv1.ControlMessage) error ConnectedNodes() []string } // Inventory is what the service reads and changes of llmgw's nodes. type Inventory interface { Get(id string) (inventory.Node, bool) SetState(id string, st inventory.NodeState) bool Allocations(nodeID string) []inventory.Allocation Nodes() []inventory.Node } // Phase is where a node of slurm.nodes is between llmgw and Slurm. type Phase string const ( // Llmgw: drained in Slurm with no job left, llmgw's to place on. Llmgw Phase = "llmgw" // Lending: in slurm.lent, its replicas being moved off. Lending Phase = "lending" // Lent: Slurm's; llmgw places nothing there. Lent Phase = "lent" // Reclaiming: out of slurm.lent, drained in Slurm, jobs still running. Reclaiming Phase = "reclaiming" // Unknown: its agent has not said how it is in Slurm yet. Unknown Phase = "unknown" ) // NodeStatus is one node as the service sees it. type NodeStatus struct { Node string `json:"node"` Phase Phase `json:"phase"` Replicas int `json:"replicas"` Status *Reported `json:"slurm,omitempty"` Connected bool `json:"connected"` } // Reported is the agent's SlurmStatus, for the admin API. type Reported struct { Version string `json:"version,omitempty"` Hardware string `json:"hardware,omitempty"` SlurmdRunning bool `json:"slurmd_running"` SlurmctldRunning bool `json:"slurmctld_running"` State string `json:"state,omitempty"` Reason string `json:"reason,omitempty"` Jobs int `json:"jobs"` Error string `json:"error,omitempty"` FilesSHA256 string `json:"files_sha256,omitempty"` At time.Time `json:"at"` } // Status is the whole Slurm cluster, for `ctl slurm status`. type Status struct { Enabled bool `json:"enabled"` Cluster string `json:"cluster"` Controller string `json:"controller"` Partition string `json:"partition"` Ctld *Reported `json:"controller_status,omitempty"` Nodes []NodeStatus `json:"nodes"` // Waiting are nodes left out of slurm.conf until they report their // hardware. Waiting []string `json:"waiting,omitempty"` } // Service keeps the Slurm cluster. type Service struct { settings func() config.SlurmSettings inv Inventory agents Agents key func(ctx context.Context) ([]byte, error) log *slog.Logger mu sync.Mutex // status is each node's last SlurmStatus. status map[string]*Reported // drained are nodes this service put in Draining, which it alone makes // ready again. drained map[string]bool // sent is what each node was last told, so it is told again only when // that changes or its stream is new. sent map[string]*llmgwv1.Slurm mungeKy []byte kick chan struct{} // mem keeps slurm.conf's nodes and the controller's address for the // next leader; known is what it held when read, and what was kept // since. loaded is false until it was read: until then the controller // is sent no files that leave out a node, which would drop that node // and its jobs. mem Memory known Kept loaded bool // started is the first pass, from which a node neither heard from nor // kept is waited for, for HoldFor. started time.Time } // HoldFor is how long a new leader waits for a node it neither has heard // from nor finds kept before it writes the controller's files without it: // on a cluster that kept nothing yet (one upgraded to this), the nodes // report within seconds of a new leader, and the controller keeps the // node and its jobs meanwhile. var HoldFor = 2 * time.Minute // New is a Service; key gives the munge key, made once from a cluster // secret. func New(settings func() config.SlurmSettings, inv Inventory, agents Agents, key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service { if log == nil { log = slog.Default() } return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log, status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{}, kick: make(chan struct{}, 1)} } // SetMemory has slurm.conf's nodes kept in mem, and read from it first. func (s *Service) SetMemory(mem Memory) { s.mu.Lock() s.mem = mem s.mu.Unlock() } // current is the settings, slurm.all_nodes resolved to the nodes the // inventory has. func (s *Service) current() config.SlurmSettings { set := s.settings() if !set.AllNodes { return set } var ids []string for _, n := range s.inv.Nodes() { ids = append(ids, n.ID) } return set.WithNodes(ids) } // Every is how often the service looks again without being asked. var Every = 10 * time.Second // Run keeps the cluster until ctx ends. func (s *Service) Run(ctx context.Context) { t := time.NewTicker(Every) defer t.Stop() for { s.Tick(ctx) select { case <-ctx.Done(): return case <-t.C: case <-s.kick: } } } // Kick has the service look again now: the settings changed. func (s *Service) Kick() { select { case s.kick <- struct{}{}: default: } } // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() prev := s.status[node] s.status[node] = r s.mu.Unlock() if r.Error != "" && (prev == nil || prev.Error != r.Error) { s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error) } if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs { s.Kick() } } // OnRegister is a node's new stream: it is told again what it is. func (s *Service) OnRegister(node string) { s.mu.Lock() delete(s.sent, node) s.mu.Unlock() s.Kick() } // Tick does one pass: the nodes handed over as far as they can be, and // each told what it is. func (s *Service) Tick(ctx context.Context) { set := s.current() s.mu.Lock() defer s.mu.Unlock() s.recall(ctx) if s.started.IsZero() { s.started = time.Now() } phases := s.handOver(set) var key []byte if set.Enabled { if s.mungeKy == nil { k, err := s.key(ctx) if err != nil { s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err) return } s.mungeKy = k } key = s.mungeKy } files, waiting := s.files(set) // Not yet knowing what the last leader knew, a slurm.conf without some // node would take it, and its jobs, from the controller: kept back. hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor) for _, node := range s.agents.ConnectedNodes() { if hold && node == set.Controller { continue } m := s.message(set, node, phases[node] == Lent, key, files) if m == nil { continue } if old, ok := s.sent[node]; ok && proto.Equal(old, m) { continue } if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil { s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err) continue } s.sent[node] = m } s.remember(ctx, set) } // recall reads what the last leader kept, once. Under s.mu. func (s *Service) recall(ctx context.Context) { if s.mem == nil || s.loaded { return } k, err := s.mem.Recall(ctx) if err != nil { s.log.Warn("could not read Slurm's nodes from the store; the controller's files wait for every node", "err", err) return } s.known, s.loaded = k, true } // remember keeps slurm.conf's nodes and the controller's address when they // changed. Under s.mu. func (s *Service) remember(ctx context.Context, set config.SlurmSettings) { if s.mem == nil || !s.loaded || !set.Enabled { return } c, _ := s.cluster(set) now := Kept{Controller: set.Controller, ControllerAddr: c.ControllerAddr, Nodes: map[string]slurm.Node{}} for _, n := range c.Nodes { now.Nodes[n.Name] = n } if now.equal(s.known) { return } if err := s.mem.Keep(ctx, now); err != nil { s.log.Warn("could not keep Slurm's nodes in the store", "err", err) return } s.known = now } // controllerAddr is where the nodes reach slurmctld: the controller's // management address, else the one kept. Under s.mu. func (s *Service) controllerAddr(set config.SlurmSettings) string { if n, ok := s.inv.Get(set.Controller); ok && n.MgmtAddr != "" { return n.MgmtAddr } if s.known.Controller == set.Controller { return s.known.ControllerAddr } return "" } // handOver moves each node of slurm.nodes a step between llmgw and Slurm, // answering where each is. Under s.mu. func (s *Service) handOver(set config.SlurmSettings) map[string]Phase { out := map[string]Phase{} if !set.Enabled { // Off: every node is llmgw's again. What Slurm still runs is left // to finish where it is; turning Slurm off with work lent is the // operator's choice. for node := range s.drained { s.ready(node) } return out } for _, node := range set.Nodes { out[node] = s.phase(set, node) } // A node no longer in slurm.nodes is llmgw's. for node := range s.drained { if !slices.Contains(set.Nodes, node) { s.ready(node) } } return out } func (s *Service) phase(set config.SlurmSettings, node string) Phase { n, ok := s.inv.Get(node) st := s.status[node] if set.IsLent(node) { if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) { s.inv.SetState(node, inventory.Draining) s.drained[node] = true s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node) } if !ok || len(s.inv.Allocations(node)) > 0 { return Lending } return Lent } // Not lent. Until the agent says Slurm has drained it with no job left, // llmgw places nothing new on it: a new leader starts with every node // ready. Cordoned, not drained: llmgw has nothing there to move. if st == nil || slurmBusy(st) { if ok && n.State == inventory.Ready { s.inv.SetState(node, inventory.Cordoned) s.drained[node] = true if st != nil { s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end", "node", node, "jobs", st.Jobs) } } if st == nil { return Unknown } return Reclaiming } if s.drained[node] { s.ready(node) } return Llmgw } // slurmBusy is whether Slurm may still run work on a node: a job step left // on it (or no telling), or a controller that would still give it one. A // node the controller does not know of, or cannot be asked about, gets no // job from it, so a Slurm that is broken never keeps a node from llmgw. func slurmBusy(st *Reported) bool { if st.Jobs != 0 { return true } if st.State == "" { return false } return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN") } func (s *Service) ready(node string) { if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) { s.inv.SetState(node, inventory.Ready) s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node) } delete(s.drained, node) } // files are the controller's files, and the nodes left out of them for not // having reported their hardware. Under s.mu. func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) { c, waiting := s.cluster(set) if c.ControllerAddr == "" { return nil, waiting } return c.Files(), waiting } // cluster is what slurm.conf says: each node as it last reported, else as // kept from before, and the nodes left out for having neither. Under s.mu. func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) { c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName()} if !set.Enabled { return c, nil } c.ControllerAddr = s.controllerAddr(set) var waiting []string for _, node := range set.Nodes { if sn, ok := s.reported(node); ok { c.Nodes = append(c.Nodes, sn) } else if sn, ok := s.known.Nodes[node]; ok { c.Nodes = append(c.Nodes, sn) } else { waiting = append(waiting, node) } } return c, waiting } // reported is a node as its agent and the inventory say now. Under s.mu. func (s *Service) reported(node string) (slurm.Node, bool) { n, ok := s.inv.Get(node) st := s.status[node] if !ok || n.MgmtAddr == "" || st == nil || st.Hardware == "" { return slurm.Node{}, false } hw, err := slurm.ParseHardware(st.Hardware) if err != nil { return slurm.Node{}, false } sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw} for _, g := range n.GPUs { sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name}) } return sn, true } // message is what a node is told. Under s.mu. func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte, files map[string]string) *llmgwv1.Slurm { m := &llmgwv1.Slurm{} if !set.Enabled { return m } isNode := slices.Contains(set.Nodes, node) isCtl := set.Controller == node && files != nil if !isNode && !isCtl { return m } m.Name, m.MungeKey, m.Lent = node, key, lent m.Node = isNode if addr := s.controllerAddr(set); addr != "" { m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort)) } if isNode && m.ConfServer == "" { // Without its controller a node cannot start; it is told when the // controller registers. Told nothing until then, not "no part": a // node still running Slurm's jobs would stop them. return nil } if isCtl { m.Controller = true m.Files = maps.Clone(files) } return m } // Status is the cluster as the service sees it now. func (s *Service) Status() Status { set := s.current() s.mu.Lock() defer s.mu.Unlock() out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName(), Ctld: s.status[set.Controller]} if !set.Enabled { return out } _, out.Waiting = s.files(set) conn := s.agents.ConnectedNodes() for _, node := range set.Nodes { ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)), Connected: slices.Contains(conn, node)} ns.Phase = s.peek(set, node) out.Nodes = append(out.Nodes, ns) } return out } // peek is phase without changing anything. Under s.mu. func (s *Service) peek(set config.SlurmSettings, node string) Phase { st := s.status[node] if set.IsLent(node) { if len(s.inv.Allocations(node)) > 0 { return Lending } return Lent } switch { case st == nil: return Unknown case slurmBusy(st): return Reclaiming } return Llmgw } // Describe is one line for a log or a screen. func (n NodeStatus) Describe() string { switch n.Phase { case Lending: return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas) case Lent: return "lent to Slurm" case Reclaiming: if n.Status != nil { return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs) } return "being taken back" case Llmgw: return "llmgw's" } return "not heard from yet" }