CI / test (push) Successful in 9m40s
internal/control/slurmsvc tells each node what it is, writes slurm.conf and gres.conf from the nodes' reports and the inventory, and moves each node of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until its last job step ends. Kept every tick from the settings and the agents' reports, so a new leader carries on. A broken Slurm never keeps a node from llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
418 lines
12 KiB
Go
418 lines
12 KiB
Go
// Package slurmsvc is the leader's side of the Slurm cluster run beside
|
|
// llmgw (BACKLOG P): it tells each node's agent what it is, writes the
|
|
// controller's files from what the nodes report, and hands nodes between
|
|
// llmgw and Slurm as the cluster's settings say.
|
|
//
|
|
// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does,
|
|
// and lent to Slurm once none is left. A node taken out of slurm.lent is
|
|
// drained in Slurm, and is llmgw's again when its last job has ended. Both
|
|
// are kept in force every tick, from the settings and what the agents say,
|
|
// so a leader that takes over carries on where the last one was.
|
|
package slurmsvc
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"maps"
|
|
"net"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
// Agents is how the service reaches the nodes.
|
|
type Agents interface {
|
|
Send(node string, msg *llmgwv1.ControlMessage) error
|
|
ConnectedNodes() []string
|
|
}
|
|
|
|
// Inventory is what the service reads and changes of llmgw's nodes.
|
|
type Inventory interface {
|
|
Get(id string) (inventory.Node, bool)
|
|
SetState(id string, st inventory.NodeState) bool
|
|
Allocations(nodeID string) []inventory.Allocation
|
|
}
|
|
|
|
// Phase is where a node of slurm.nodes is between llmgw and Slurm.
|
|
type Phase string
|
|
|
|
const (
|
|
// Llmgw: drained in Slurm with no job left, llmgw's to place on.
|
|
Llmgw Phase = "llmgw"
|
|
// Lending: in slurm.lent, its replicas being moved off.
|
|
Lending Phase = "lending"
|
|
// Lent: Slurm's; llmgw places nothing there.
|
|
Lent Phase = "lent"
|
|
// Reclaiming: out of slurm.lent, drained in Slurm, jobs still running.
|
|
Reclaiming Phase = "reclaiming"
|
|
// Unknown: its agent has not said how it is in Slurm yet.
|
|
Unknown Phase = "unknown"
|
|
)
|
|
|
|
// NodeStatus is one node as the service sees it.
|
|
type NodeStatus struct {
|
|
Node string `json:"node"`
|
|
Phase Phase `json:"phase"`
|
|
Replicas int `json:"replicas"`
|
|
Status *Reported `json:"slurm,omitempty"`
|
|
Connected bool `json:"connected"`
|
|
}
|
|
|
|
// Reported is the agent's SlurmStatus, for the admin API.
|
|
type Reported struct {
|
|
Version string `json:"version,omitempty"`
|
|
Hardware string `json:"hardware,omitempty"`
|
|
SlurmdRunning bool `json:"slurmd_running"`
|
|
SlurmctldRunning bool `json:"slurmctld_running"`
|
|
State string `json:"state,omitempty"`
|
|
Reason string `json:"reason,omitempty"`
|
|
Jobs int `json:"jobs"`
|
|
Error string `json:"error,omitempty"`
|
|
FilesSHA256 string `json:"files_sha256,omitempty"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
// Status is the whole Slurm cluster, for `ctl slurm status`.
|
|
type Status struct {
|
|
Enabled bool `json:"enabled"`
|
|
Cluster string `json:"cluster"`
|
|
Controller string `json:"controller"`
|
|
Partition string `json:"partition"`
|
|
Ctld *Reported `json:"controller_status,omitempty"`
|
|
Nodes []NodeStatus `json:"nodes"`
|
|
// Waiting are nodes left out of slurm.conf until they report their
|
|
// hardware.
|
|
Waiting []string `json:"waiting,omitempty"`
|
|
}
|
|
|
|
// Service keeps the Slurm cluster.
|
|
type Service struct {
|
|
settings func() config.SlurmSettings
|
|
inv Inventory
|
|
agents Agents
|
|
key func(ctx context.Context) ([]byte, error)
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
// status is each node's last SlurmStatus.
|
|
status map[string]*Reported
|
|
// drained are nodes this service put in Draining, which it alone makes
|
|
// ready again.
|
|
drained map[string]bool
|
|
// sent is what each node was last told, so it is told again only when
|
|
// that changes or its stream is new.
|
|
sent map[string]*llmgwv1.Slurm
|
|
mungeKy []byte
|
|
kick chan struct{}
|
|
}
|
|
|
|
// New is a Service; key gives the munge key, made once from a cluster
|
|
// secret.
|
|
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
|
key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
|
|
status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{},
|
|
kick: make(chan struct{}, 1)}
|
|
}
|
|
|
|
// Every is how often the service looks again without being asked.
|
|
var Every = 10 * time.Second
|
|
|
|
// Run keeps the cluster until ctx ends.
|
|
func (s *Service) Run(ctx context.Context) {
|
|
t := time.NewTicker(Every)
|
|
defer t.Stop()
|
|
for {
|
|
s.Tick(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
case <-s.kick:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Kick has the service look again now: the settings changed.
|
|
func (s *Service) Kick() {
|
|
select {
|
|
case s.kick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// OnSlurmStatus records what a node's agent says.
|
|
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
|
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
|
|
SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(),
|
|
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
|
|
s.mu.Lock()
|
|
prev := s.status[node]
|
|
s.status[node] = r
|
|
s.mu.Unlock()
|
|
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
|
|
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
|
|
}
|
|
if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs {
|
|
s.Kick()
|
|
}
|
|
}
|
|
|
|
// OnRegister is a node's new stream: it is told again what it is.
|
|
func (s *Service) OnRegister(node string) {
|
|
s.mu.Lock()
|
|
delete(s.sent, node)
|
|
s.mu.Unlock()
|
|
s.Kick()
|
|
}
|
|
|
|
// Tick does one pass: the nodes handed over as far as they can be, and
|
|
// each told what it is.
|
|
func (s *Service) Tick(ctx context.Context) {
|
|
set := s.settings()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
phases := s.handOver(set)
|
|
var key []byte
|
|
if set.Enabled {
|
|
if s.mungeKy == nil {
|
|
k, err := s.key(ctx)
|
|
if err != nil {
|
|
s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err)
|
|
return
|
|
}
|
|
s.mungeKy = k
|
|
}
|
|
key = s.mungeKy
|
|
}
|
|
files, _ := s.files(set)
|
|
for _, node := range s.agents.ConnectedNodes() {
|
|
m := s.message(set, node, phases[node] == Lent, key, files)
|
|
if old, ok := s.sent[node]; ok && proto.Equal(old, m) {
|
|
continue
|
|
}
|
|
if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil {
|
|
s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err)
|
|
continue
|
|
}
|
|
s.sent[node] = m
|
|
}
|
|
}
|
|
|
|
// handOver moves each node of slurm.nodes a step between llmgw and Slurm,
|
|
// answering where each is. Under s.mu.
|
|
func (s *Service) handOver(set config.SlurmSettings) map[string]Phase {
|
|
out := map[string]Phase{}
|
|
if !set.Enabled {
|
|
// Off: every node is llmgw's again. What Slurm still runs is left
|
|
// to finish where it is; turning Slurm off with work lent is the
|
|
// operator's choice.
|
|
for node := range s.drained {
|
|
s.ready(node)
|
|
}
|
|
return out
|
|
}
|
|
for _, node := range set.Nodes {
|
|
out[node] = s.phase(set, node)
|
|
}
|
|
// A node no longer in slurm.nodes is llmgw's.
|
|
for node := range s.drained {
|
|
if !slices.Contains(set.Nodes, node) {
|
|
s.ready(node)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Service) phase(set config.SlurmSettings, node string) Phase {
|
|
n, ok := s.inv.Get(node)
|
|
st := s.status[node]
|
|
if set.IsLent(node) {
|
|
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
|
|
s.inv.SetState(node, inventory.Draining)
|
|
s.drained[node] = true
|
|
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node)
|
|
}
|
|
if !ok || len(s.inv.Allocations(node)) > 0 {
|
|
return Lending
|
|
}
|
|
return Lent
|
|
}
|
|
// Not lent. Until the agent says Slurm has drained it with no job left,
|
|
// llmgw places nothing new on it: a new leader starts with every node
|
|
// ready. Cordoned, not drained: llmgw has nothing there to move.
|
|
if st == nil || slurmBusy(st) {
|
|
if ok && n.State == inventory.Ready {
|
|
s.inv.SetState(node, inventory.Cordoned)
|
|
s.drained[node] = true
|
|
if st != nil {
|
|
s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end",
|
|
"node", node, "jobs", st.Jobs)
|
|
}
|
|
}
|
|
if st == nil {
|
|
return Unknown
|
|
}
|
|
return Reclaiming
|
|
}
|
|
if s.drained[node] {
|
|
s.ready(node)
|
|
}
|
|
return Llmgw
|
|
}
|
|
|
|
// slurmBusy is whether Slurm may still run work on a node: a job step left
|
|
// on it (or no telling), or a controller that would still give it one. A
|
|
// node the controller does not know of, or cannot be asked about, gets no
|
|
// job from it, so a Slurm that is broken never keeps a node from llmgw.
|
|
func slurmBusy(st *Reported) bool {
|
|
if st.Jobs != 0 {
|
|
return true
|
|
}
|
|
if st.State == "" {
|
|
return false
|
|
}
|
|
return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN")
|
|
}
|
|
|
|
func (s *Service) ready(node string) {
|
|
if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) {
|
|
s.inv.SetState(node, inventory.Ready)
|
|
s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node)
|
|
}
|
|
delete(s.drained, node)
|
|
}
|
|
|
|
// files are the controller's files, and the nodes left out of them for not
|
|
// having reported their hardware. Under s.mu.
|
|
func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) {
|
|
ctl, ok := s.inv.Get(set.Controller)
|
|
if !set.Enabled || !ok {
|
|
return nil, nil
|
|
}
|
|
c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, ControllerAddr: ctl.MgmtAddr,
|
|
Partition: set.PartitionName()}
|
|
var waiting []string
|
|
for _, node := range set.Nodes {
|
|
n, ok := s.inv.Get(node)
|
|
st := s.status[node]
|
|
if !ok || st == nil || st.Hardware == "" {
|
|
waiting = append(waiting, node)
|
|
continue
|
|
}
|
|
hw, err := slurm.ParseHardware(st.Hardware)
|
|
if err != nil {
|
|
waiting = append(waiting, node)
|
|
continue
|
|
}
|
|
sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw}
|
|
for _, g := range n.GPUs {
|
|
sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name})
|
|
}
|
|
c.Nodes = append(c.Nodes, sn)
|
|
}
|
|
return c.Files(), waiting
|
|
}
|
|
|
|
// message is what a node is told. Under s.mu.
|
|
func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte,
|
|
files map[string]string) *llmgwv1.Slurm {
|
|
m := &llmgwv1.Slurm{}
|
|
if !set.Enabled {
|
|
return m
|
|
}
|
|
isNode := slices.Contains(set.Nodes, node)
|
|
isCtl := set.Controller == node && files != nil
|
|
if !isNode && !isCtl {
|
|
return m
|
|
}
|
|
m.Name, m.MungeKey, m.Lent = node, key, lent
|
|
m.Node = isNode
|
|
if ctl, ok := s.inv.Get(set.Controller); ok {
|
|
m.ConfServer = net.JoinHostPort(ctl.MgmtAddr, strconv.Itoa(slurm.CtldPort))
|
|
}
|
|
if isNode && m.ConfServer == "" {
|
|
// Without its controller a node cannot start; it is told when the
|
|
// controller registers.
|
|
m.Node = false
|
|
}
|
|
if isCtl {
|
|
m.Controller = true
|
|
m.Files = maps.Clone(files)
|
|
}
|
|
if !m.Node && !m.Controller {
|
|
return &llmgwv1.Slurm{}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Status is the cluster as the service sees it now.
|
|
func (s *Service) Status() Status {
|
|
set := s.settings()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
|
|
Partition: set.PartitionName(), Ctld: s.status[set.Controller]}
|
|
if !set.Enabled {
|
|
return out
|
|
}
|
|
_, out.Waiting = s.files(set)
|
|
conn := s.agents.ConnectedNodes()
|
|
for _, node := range set.Nodes {
|
|
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
|
|
Connected: slices.Contains(conn, node)}
|
|
ns.Phase = s.peek(set, node)
|
|
out.Nodes = append(out.Nodes, ns)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// peek is phase without changing anything. Under s.mu.
|
|
func (s *Service) peek(set config.SlurmSettings, node string) Phase {
|
|
st := s.status[node]
|
|
if set.IsLent(node) {
|
|
if len(s.inv.Allocations(node)) > 0 {
|
|
return Lending
|
|
}
|
|
return Lent
|
|
}
|
|
switch {
|
|
case st == nil:
|
|
return Unknown
|
|
case slurmBusy(st):
|
|
return Reclaiming
|
|
}
|
|
return Llmgw
|
|
}
|
|
|
|
// Describe is one line for a log or a screen.
|
|
func (n NodeStatus) Describe() string {
|
|
switch n.Phase {
|
|
case Lending:
|
|
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
|
|
case Lent:
|
|
return "lent to Slurm"
|
|
case Reclaiming:
|
|
if n.Status != nil {
|
|
return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs)
|
|
}
|
|
return "being taken back"
|
|
case Llmgw:
|
|
return "llmgw's"
|
|
}
|
|
return "not heard from yet"
|
|
}
|