Files
llmgw/internal/control/slurmsvc/service.go
T
Ilker ManapandClaude Opus 5.5 410fe545ce
CI / test (push) Successful in 9m40s
Slurm beside llmgw, P3 and P4: the leader hands nodes over; ctl slurm
internal/control/slurmsvc tells each node what it is, writes slurm.conf
and gres.conf from the nodes' reports and the inventory, and moves each node
of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then
resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until
its last job step ends. Kept every tick from the settings and the agents'
reports, so a new leader carries on. A broken Slurm never keeps a node from
llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 12:33:14 +01:00

418 lines
12 KiB
Go

// Package slurmsvc is the leader's side of the Slurm cluster run beside
// llmgw (BACKLOG P): it tells each node's agent what it is, writes the
// controller's files from what the nodes report, and hands nodes between
// llmgw and Slurm as the cluster's settings say.
//
// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does,
// and lent to Slurm once none is left. A node taken out of slurm.lent is
// drained in Slurm, and is llmgw's again when its last job has ended. Both
// are kept in force every tick, from the settings and what the agents say,
// so a leader that takes over carries on where the last one was.
package slurmsvc
import (
"context"
"fmt"
"log/slog"
"maps"
"net"
"slices"
"strconv"
"strings"
"sync"
"time"
"google.golang.org/protobuf/proto"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// Agents is how the service reaches the nodes.
type Agents interface {
Send(node string, msg *llmgwv1.ControlMessage) error
ConnectedNodes() []string
}
// Inventory is what the service reads and changes of llmgw's nodes.
type Inventory interface {
Get(id string) (inventory.Node, bool)
SetState(id string, st inventory.NodeState) bool
Allocations(nodeID string) []inventory.Allocation
}
// Phase is where a node of slurm.nodes is between llmgw and Slurm.
type Phase string
const (
// Llmgw: drained in Slurm with no job left, llmgw's to place on.
Llmgw Phase = "llmgw"
// Lending: in slurm.lent, its replicas being moved off.
Lending Phase = "lending"
// Lent: Slurm's; llmgw places nothing there.
Lent Phase = "lent"
// Reclaiming: out of slurm.lent, drained in Slurm, jobs still running.
Reclaiming Phase = "reclaiming"
// Unknown: its agent has not said how it is in Slurm yet.
Unknown Phase = "unknown"
)
// NodeStatus is one node as the service sees it.
type NodeStatus struct {
Node string `json:"node"`
Phase Phase `json:"phase"`
Replicas int `json:"replicas"`
Status *Reported `json:"slurm,omitempty"`
Connected bool `json:"connected"`
}
// Reported is the agent's SlurmStatus, for the admin API.
type Reported struct {
Version string `json:"version,omitempty"`
Hardware string `json:"hardware,omitempty"`
SlurmdRunning bool `json:"slurmd_running"`
SlurmctldRunning bool `json:"slurmctld_running"`
State string `json:"state,omitempty"`
Reason string `json:"reason,omitempty"`
Jobs int `json:"jobs"`
Error string `json:"error,omitempty"`
FilesSHA256 string `json:"files_sha256,omitempty"`
At time.Time `json:"at"`
}
// Status is the whole Slurm cluster, for `ctl slurm status`.
type Status struct {
Enabled bool `json:"enabled"`
Cluster string `json:"cluster"`
Controller string `json:"controller"`
Partition string `json:"partition"`
Ctld *Reported `json:"controller_status,omitempty"`
Nodes []NodeStatus `json:"nodes"`
// Waiting are nodes left out of slurm.conf until they report their
// hardware.
Waiting []string `json:"waiting,omitempty"`
}
// Service keeps the Slurm cluster.
type Service struct {
settings func() config.SlurmSettings
inv Inventory
agents Agents
key func(ctx context.Context) ([]byte, error)
log *slog.Logger
mu sync.Mutex
// status is each node's last SlurmStatus.
status map[string]*Reported
// drained are nodes this service put in Draining, which it alone makes
// ready again.
drained map[string]bool
// sent is what each node was last told, so it is told again only when
// that changes or its stream is new.
sent map[string]*llmgwv1.Slurm
mungeKy []byte
kick chan struct{}
}
// New is a Service; key gives the munge key, made once from a cluster
// secret.
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service {
if log == nil {
log = slog.Default()
}
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{},
kick: make(chan struct{}, 1)}
}
// Every is how often the service looks again without being asked.
var Every = 10 * time.Second
// Run keeps the cluster until ctx ends.
func (s *Service) Run(ctx context.Context) {
t := time.NewTicker(Every)
defer t.Stop()
for {
s.Tick(ctx)
select {
case <-ctx.Done():
return
case <-t.C:
case <-s.kick:
}
}
}
// Kick has the service look again now: the settings changed.
func (s *Service) Kick() {
select {
case s.kick <- struct{}{}:
default:
}
}
// OnSlurmStatus records what a node's agent says.
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(),
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
s.mu.Lock()
prev := s.status[node]
s.status[node] = r
s.mu.Unlock()
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
}
if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs {
s.Kick()
}
}
// OnRegister is a node's new stream: it is told again what it is.
func (s *Service) OnRegister(node string) {
s.mu.Lock()
delete(s.sent, node)
s.mu.Unlock()
s.Kick()
}
// Tick does one pass: the nodes handed over as far as they can be, and
// each told what it is.
func (s *Service) Tick(ctx context.Context) {
set := s.settings()
s.mu.Lock()
defer s.mu.Unlock()
phases := s.handOver(set)
var key []byte
if set.Enabled {
if s.mungeKy == nil {
k, err := s.key(ctx)
if err != nil {
s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err)
return
}
s.mungeKy = k
}
key = s.mungeKy
}
files, _ := s.files(set)
for _, node := range s.agents.ConnectedNodes() {
m := s.message(set, node, phases[node] == Lent, key, files)
if old, ok := s.sent[node]; ok && proto.Equal(old, m) {
continue
}
if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil {
s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err)
continue
}
s.sent[node] = m
}
}
// handOver moves each node of slurm.nodes a step between llmgw and Slurm,
// answering where each is. Under s.mu.
func (s *Service) handOver(set config.SlurmSettings) map[string]Phase {
out := map[string]Phase{}
if !set.Enabled {
// Off: every node is llmgw's again. What Slurm still runs is left
// to finish where it is; turning Slurm off with work lent is the
// operator's choice.
for node := range s.drained {
s.ready(node)
}
return out
}
for _, node := range set.Nodes {
out[node] = s.phase(set, node)
}
// A node no longer in slurm.nodes is llmgw's.
for node := range s.drained {
if !slices.Contains(set.Nodes, node) {
s.ready(node)
}
}
return out
}
func (s *Service) phase(set config.SlurmSettings, node string) Phase {
n, ok := s.inv.Get(node)
st := s.status[node]
if set.IsLent(node) {
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
s.inv.SetState(node, inventory.Draining)
s.drained[node] = true
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node)
}
if !ok || len(s.inv.Allocations(node)) > 0 {
return Lending
}
return Lent
}
// Not lent. Until the agent says Slurm has drained it with no job left,
// llmgw places nothing new on it: a new leader starts with every node
// ready. Cordoned, not drained: llmgw has nothing there to move.
if st == nil || slurmBusy(st) {
if ok && n.State == inventory.Ready {
s.inv.SetState(node, inventory.Cordoned)
s.drained[node] = true
if st != nil {
s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end",
"node", node, "jobs", st.Jobs)
}
}
if st == nil {
return Unknown
}
return Reclaiming
}
if s.drained[node] {
s.ready(node)
}
return Llmgw
}
// slurmBusy is whether Slurm may still run work on a node: a job step left
// on it (or no telling), or a controller that would still give it one. A
// node the controller does not know of, or cannot be asked about, gets no
// job from it, so a Slurm that is broken never keeps a node from llmgw.
func slurmBusy(st *Reported) bool {
if st.Jobs != 0 {
return true
}
if st.State == "" {
return false
}
return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN")
}
func (s *Service) ready(node string) {
if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) {
s.inv.SetState(node, inventory.Ready)
s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node)
}
delete(s.drained, node)
}
// files are the controller's files, and the nodes left out of them for not
// having reported their hardware. Under s.mu.
func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) {
ctl, ok := s.inv.Get(set.Controller)
if !set.Enabled || !ok {
return nil, nil
}
c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, ControllerAddr: ctl.MgmtAddr,
Partition: set.PartitionName()}
var waiting []string
for _, node := range set.Nodes {
n, ok := s.inv.Get(node)
st := s.status[node]
if !ok || st == nil || st.Hardware == "" {
waiting = append(waiting, node)
continue
}
hw, err := slurm.ParseHardware(st.Hardware)
if err != nil {
waiting = append(waiting, node)
continue
}
sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw}
for _, g := range n.GPUs {
sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name})
}
c.Nodes = append(c.Nodes, sn)
}
return c.Files(), waiting
}
// message is what a node is told. Under s.mu.
func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte,
files map[string]string) *llmgwv1.Slurm {
m := &llmgwv1.Slurm{}
if !set.Enabled {
return m
}
isNode := slices.Contains(set.Nodes, node)
isCtl := set.Controller == node && files != nil
if !isNode && !isCtl {
return m
}
m.Name, m.MungeKey, m.Lent = node, key, lent
m.Node = isNode
if ctl, ok := s.inv.Get(set.Controller); ok {
m.ConfServer = net.JoinHostPort(ctl.MgmtAddr, strconv.Itoa(slurm.CtldPort))
}
if isNode && m.ConfServer == "" {
// Without its controller a node cannot start; it is told when the
// controller registers.
m.Node = false
}
if isCtl {
m.Controller = true
m.Files = maps.Clone(files)
}
if !m.Node && !m.Controller {
return &llmgwv1.Slurm{}
}
return m
}
// Status is the cluster as the service sees it now.
func (s *Service) Status() Status {
set := s.settings()
s.mu.Lock()
defer s.mu.Unlock()
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
Partition: set.PartitionName(), Ctld: s.status[set.Controller]}
if !set.Enabled {
return out
}
_, out.Waiting = s.files(set)
conn := s.agents.ConnectedNodes()
for _, node := range set.Nodes {
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
Connected: slices.Contains(conn, node)}
ns.Phase = s.peek(set, node)
out.Nodes = append(out.Nodes, ns)
}
return out
}
// peek is phase without changing anything. Under s.mu.
func (s *Service) peek(set config.SlurmSettings, node string) Phase {
st := s.status[node]
if set.IsLent(node) {
if len(s.inv.Allocations(node)) > 0 {
return Lending
}
return Lent
}
switch {
case st == nil:
return Unknown
case slurmBusy(st):
return Reclaiming
}
return Llmgw
}
// Describe is one line for a log or a screen.
func (n NodeStatus) Describe() string {
switch n.Phase {
case Lending:
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
case Lent:
return "lent to Slurm"
case Reclaiming:
if n.Status != nil {
return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs)
}
return "being taken back"
case Llmgw:
return "llmgw's"
}
return "not heard from yet"
}