CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
869 lines
26 KiB
Go
869 lines
26 KiB
Go
// Package slurmsvc is the leader's side of the Slurm cluster run beside
|
|
// llmgw (BACKLOG P): it tells each node's agent what it is, writes the
|
|
// controller's files from what the nodes report, and hands nodes between
|
|
// llmgw and Slurm as the cluster's settings say.
|
|
//
|
|
// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does,
|
|
// and lent to Slurm once none is left. A node taken out of slurm.lent is
|
|
// drained in Slurm, and is llmgw's again when its last job has ended. Both
|
|
// are kept in force every tick, from the settings and what the agents say,
|
|
// so a leader that takes over carries on where the last one was.
|
|
package slurmsvc
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"maps"
|
|
"net"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"google.golang.org/protobuf/proto"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
// Agents is how the service reaches the nodes.
|
|
type Agents interface {
|
|
Send(node string, msg *llmgwv1.ControlMessage) error
|
|
ConnectedNodes() []string
|
|
}
|
|
|
|
// Inventory is what the service reads and changes of llmgw's nodes.
|
|
type Inventory interface {
|
|
Get(id string) (inventory.Node, bool)
|
|
SetState(id string, st inventory.NodeState) bool
|
|
Allocations(nodeID string) []inventory.Allocation
|
|
Nodes() []inventory.Node
|
|
}
|
|
|
|
// Phase is where a node of slurm.nodes is between llmgw and Slurm.
|
|
type Phase string
|
|
|
|
const (
|
|
// Llmgw: drained in Slurm with no job left, llmgw's to place on.
|
|
Llmgw Phase = "llmgw"
|
|
// Lending: in slurm.lent, its replicas being moved off.
|
|
Lending Phase = "lending"
|
|
// Lent: Slurm's; llmgw places nothing there.
|
|
Lent Phase = "lent"
|
|
// Reclaiming: out of slurm.lent, drained in Slurm, jobs still running.
|
|
Reclaiming Phase = "reclaiming"
|
|
// Unknown: its agent has not said how it is in Slurm yet.
|
|
Unknown Phase = "unknown"
|
|
)
|
|
|
|
// NodeStatus is one node as the service sees it.
|
|
type NodeStatus struct {
|
|
Node string `json:"node"`
|
|
Phase Phase `json:"phase"`
|
|
Replicas int `json:"replicas"`
|
|
// StopsBy is, while it is being lent, when a replica still on it is
|
|
// stopped even with nowhere else to go: the drain's grace after the
|
|
// lend began. Zero when not known (a leader that took over meanwhile).
|
|
StopsBy time.Time `json:"stops_by,omitempty"`
|
|
Status *Reported `json:"slurm,omitempty"`
|
|
Connected bool `json:"connected"`
|
|
}
|
|
|
|
// Reported is the agent's SlurmStatus, for the admin API.
|
|
type Reported struct {
|
|
Version string `json:"version,omitempty"`
|
|
Hardware string `json:"hardware,omitempty"`
|
|
SlurmdRunning bool `json:"slurmd_running"`
|
|
SlurmctldRunning bool `json:"slurmctld_running"`
|
|
SlurmdbdRunning bool `json:"slurmdbd_running"`
|
|
// Shared is the shared path here: "mounted", "exported", "missing".
|
|
Shared string `json:"shared,omitempty"`
|
|
// Containers is what the machine has for jobs in containers, empty
|
|
// while not both of pyxis and enroot.
|
|
Containers string `json:"containers,omitempty"`
|
|
// Users is how many of llmgw's users the machine has.
|
|
Users int `json:"users"`
|
|
State string `json:"state,omitempty"`
|
|
Reason string `json:"reason,omitempty"`
|
|
Jobs int `json:"jobs"`
|
|
Error string `json:"error,omitempty"`
|
|
FilesSHA256 string `json:"files_sha256,omitempty"`
|
|
At time.Time `json:"at"`
|
|
}
|
|
|
|
// Status is the whole Slurm cluster, for `ctl slurm status`.
|
|
type Status struct {
|
|
Enabled bool `json:"enabled"`
|
|
Cluster string `json:"cluster"`
|
|
Controller string `json:"controller"`
|
|
Partition string `json:"partition"`
|
|
Ctld *Reported `json:"controller_status,omitempty"`
|
|
Nodes []NodeStatus `json:"nodes"`
|
|
// Waiting are nodes left out of slurm.conf until they report their
|
|
// hardware.
|
|
Waiting []string `json:"waiting,omitempty"`
|
|
// Shared is where the shared files are and come from; Users who keeps
|
|
// the users, and how many llmgw has.
|
|
Shared config.SharedStorage `json:"shared"`
|
|
Users string `json:"users,omitempty"`
|
|
UserCount int `json:"user_count,omitempty"`
|
|
// Containers is whether jobs may run in containers: "on" once every
|
|
// machine has pyxis and enroot, "installing" until then, empty when
|
|
// slurm.containers is off.
|
|
Containers string `json:"containers,omitempty"`
|
|
// Accounts is how many of Slurm's accounts are llmgw's projects'.
|
|
Accounts int `json:"accounts,omitempty"`
|
|
}
|
|
|
|
// User is a Linux user made from an llmgw account.
|
|
type User struct {
|
|
Login string `json:"login"`
|
|
UID int `json:"uid"`
|
|
Name string `json:"name"`
|
|
Email string `json:"email"`
|
|
SSHKeys []string `json:"ssh_keys,omitempty"`
|
|
}
|
|
|
|
// Service keeps the Slurm cluster.
|
|
type Service struct {
|
|
settings func() config.SlurmSettings
|
|
inv Inventory
|
|
agents Agents
|
|
key func(ctx context.Context) ([]byte, error)
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
// status is each node's last SlurmStatus.
|
|
status map[string]*Reported
|
|
// drained are nodes this service put in Draining, which it alone makes
|
|
// ready again.
|
|
drained map[string]bool
|
|
// lendingAt is when this leader began moving a node's replicas off.
|
|
lendingAt map[string]time.Time
|
|
// sent is what each node was last told, so it is told again only when
|
|
// that changes or its stream is new.
|
|
sent map[string]*llmgwv1.Slurm
|
|
mungeKy []byte
|
|
kick chan struct{}
|
|
// mem keeps slurm.conf's nodes and the controller's address for the
|
|
// next leader; known is what it held when read, and what was kept
|
|
// since. loaded is false until it was read: until then the controller
|
|
// is sent no files that leave out a node, which would drop that node
|
|
// and its jobs.
|
|
mem Memory
|
|
known Kept
|
|
loaded bool
|
|
// started is the first pass, from which a node neither heard from nor
|
|
// kept is waited for, for HoldFor.
|
|
started time.Time
|
|
// userList reads the users from the accounts; users is what it last
|
|
// said, read at usersAt. Until it has been read once no node is told
|
|
// anything of users: an empty list would take every user away.
|
|
userList func(ctx context.Context) ([]User, error)
|
|
users []User
|
|
usersAt time.Time
|
|
usersOK bool
|
|
// accountList reads Slurm's accounts from the projects, with the users;
|
|
// accountsOK is false until it has been read, and the controller is
|
|
// told nothing of accounts until then: none would take every one away.
|
|
accountList func(ctx context.Context) ([]slurm.Account, error)
|
|
accounts []slurm.Account
|
|
accountsOK bool
|
|
// jobSink records the jobs that ended; recorded are those it took,
|
|
// recording whether a recording is under way.
|
|
jobSink func(ctx context.Context, jobs []JobRecord) error
|
|
recorded map[string]time.Time
|
|
recording bool
|
|
}
|
|
|
|
// JobRecord is a job that ended, for llmgw's accounting.
|
|
type JobRecord struct {
|
|
ID, Login, Account, Partition, State string
|
|
Start, End time.Time
|
|
Elapsed time.Duration
|
|
CPUs, GPUs int
|
|
}
|
|
|
|
// SetAccounts has Slurm's accounts read with list, beside the users.
|
|
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
|
|
s.mu.Lock()
|
|
s.accountList = list
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// SetJobs has the jobs that ended recorded with record; each is given to it
|
|
// once it took it without an error.
|
|
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
|
|
s.mu.Lock()
|
|
s.jobSink = record
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// AccountsOf are the accounts a login may run jobs on.
|
|
func (s *Service) AccountsOf(login string) []string {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
var out []string
|
|
for _, a := range s.accounts {
|
|
if slices.Contains(a.Users, login) {
|
|
out = append(out, a.Name)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// UsersEvery is how often the users are read again without being asked.
|
|
var UsersEvery = 30 * time.Second
|
|
|
|
// SetUsers has the users read with list, for slurm.users "llmgw".
|
|
func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) {
|
|
s.mu.Lock()
|
|
s.userList = list
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// UsersChanged has the users read again now: an account or its keys
|
|
// changed.
|
|
func (s *Service) UsersChanged() {
|
|
s.mu.Lock()
|
|
s.usersAt = time.Time{}
|
|
s.mu.Unlock()
|
|
s.Kick()
|
|
}
|
|
|
|
// readUsers reads the users when they are due. Under s.mu.
|
|
func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
|
|
if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery {
|
|
return
|
|
}
|
|
us, err := s.userList(ctx)
|
|
s.usersAt = time.Now()
|
|
if err != nil {
|
|
s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err)
|
|
return
|
|
}
|
|
s.users, s.usersOK = us, true
|
|
if s.accountList == nil {
|
|
return
|
|
}
|
|
as, err := s.accountList(ctx)
|
|
if err != nil {
|
|
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
|
|
return
|
|
}
|
|
s.accounts, s.accountsOK = as, true
|
|
}
|
|
|
|
// takeJobs records the jobs a controller says ended that are not recorded
|
|
// yet, away from the agent's stream.
|
|
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
|
|
s.mu.Lock()
|
|
if s.jobSink == nil || s.recording {
|
|
s.mu.Unlock()
|
|
return
|
|
}
|
|
if s.recorded == nil {
|
|
s.recorded = map[string]time.Time{}
|
|
}
|
|
for id, at := range s.recorded {
|
|
if time.Since(at) > 3*24*time.Hour {
|
|
delete(s.recorded, id)
|
|
}
|
|
}
|
|
var jobs []JobRecord
|
|
for _, j := range ended {
|
|
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
|
|
continue
|
|
}
|
|
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
|
|
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
|
|
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
|
|
}
|
|
if len(jobs) == 0 {
|
|
s.mu.Unlock()
|
|
return
|
|
}
|
|
sink := s.jobSink
|
|
s.recording = true
|
|
s.mu.Unlock()
|
|
go func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
err := sink(ctx, jobs)
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.recording = false
|
|
if err != nil {
|
|
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
|
|
return
|
|
}
|
|
for _, j := range jobs {
|
|
s.recorded[j.ID] = time.Now()
|
|
}
|
|
}()
|
|
}
|
|
|
|
// HoldFor is how long a new leader waits for a node it neither has heard
|
|
// from nor finds kept before it writes the controller's files without it:
|
|
// on a cluster that kept nothing yet (one upgraded to this), the nodes
|
|
// report within seconds of a new leader, and the controller keeps the
|
|
// node and its jobs meanwhile.
|
|
var HoldFor = 2 * time.Minute
|
|
|
|
// New is a Service; key gives the munge key, made once from a cluster
|
|
// secret.
|
|
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
|
key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service {
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
|
|
status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{},
|
|
kick: make(chan struct{}, 1)}
|
|
}
|
|
|
|
// SetMemory has slurm.conf's nodes kept in mem, and read from it first.
|
|
func (s *Service) SetMemory(mem Memory) {
|
|
s.mu.Lock()
|
|
s.mem = mem
|
|
s.mu.Unlock()
|
|
}
|
|
|
|
// current is the settings, slurm.all_nodes resolved to the nodes the
|
|
// inventory has.
|
|
func (s *Service) current() config.SlurmSettings {
|
|
set := s.settings()
|
|
if !set.AllNodes {
|
|
return set
|
|
}
|
|
var ids []string
|
|
for _, n := range s.inv.Nodes() {
|
|
ids = append(ids, n.ID)
|
|
}
|
|
return set.WithNodes(ids)
|
|
}
|
|
|
|
// Every is how often the service looks again without being asked.
|
|
var Every = 10 * time.Second
|
|
|
|
// Run keeps the cluster until ctx ends.
|
|
func (s *Service) Run(ctx context.Context) {
|
|
t := time.NewTicker(Every)
|
|
defer t.Stop()
|
|
for {
|
|
s.Tick(ctx)
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
case <-s.kick:
|
|
}
|
|
}
|
|
}
|
|
|
|
// Kick has the service look again now: the settings changed.
|
|
func (s *Service) Kick() {
|
|
select {
|
|
case s.kick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// OnSlurmStatus records what a node's agent says.
|
|
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
|
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
|
|
Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
|
|
SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
|
|
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
|
|
s.mu.Lock()
|
|
prev := s.status[node]
|
|
s.status[node] = r
|
|
isCtl := s.settings().Controller == node
|
|
s.mu.Unlock()
|
|
if isCtl && len(st.GetEnded()) > 0 {
|
|
s.takeJobs(st.GetEnded())
|
|
}
|
|
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
|
|
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
|
|
}
|
|
if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs {
|
|
s.Kick()
|
|
}
|
|
}
|
|
|
|
// OnRegister is a node's new stream: it is told again what it is.
|
|
func (s *Service) OnRegister(node string) {
|
|
s.mu.Lock()
|
|
delete(s.sent, node)
|
|
s.mu.Unlock()
|
|
s.Kick()
|
|
}
|
|
|
|
// Tick does one pass: the nodes handed over as far as they can be, and
|
|
// each told what it is.
|
|
func (s *Service) Tick(ctx context.Context) {
|
|
set := s.current()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.recall(ctx)
|
|
s.readUsers(ctx, set)
|
|
if s.started.IsZero() {
|
|
s.started = time.Now()
|
|
}
|
|
phases := s.handOver(set)
|
|
var key []byte
|
|
if set.Enabled {
|
|
if s.mungeKy == nil {
|
|
k, err := s.key(ctx)
|
|
if err != nil {
|
|
s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err)
|
|
return
|
|
}
|
|
s.mungeKy = k
|
|
}
|
|
key = s.mungeKy
|
|
}
|
|
files, waiting := s.files(set)
|
|
// Not yet knowing what the last leader knew, a slurm.conf without some
|
|
// node would take it, and its jobs, from the controller: kept back.
|
|
hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
|
|
for _, node := range s.agents.ConnectedNodes() {
|
|
if hold && node == set.Controller {
|
|
continue
|
|
}
|
|
m := s.message(set, node, phases[node] == Lent, key, files)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
if old, ok := s.sent[node]; ok && proto.Equal(old, m) {
|
|
continue
|
|
}
|
|
if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil {
|
|
s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err)
|
|
continue
|
|
}
|
|
s.sent[node] = m
|
|
}
|
|
s.remember(ctx, set)
|
|
}
|
|
|
|
// recall reads what the last leader kept, once. Under s.mu.
|
|
func (s *Service) recall(ctx context.Context) {
|
|
if s.mem == nil || s.loaded {
|
|
return
|
|
}
|
|
k, err := s.mem.Recall(ctx)
|
|
if err != nil {
|
|
s.log.Warn("could not read Slurm's nodes from the store; the controller's files wait for every node",
|
|
"err", err)
|
|
return
|
|
}
|
|
s.known, s.loaded = k, true
|
|
}
|
|
|
|
// remember keeps slurm.conf's nodes and the controller's address when they
|
|
// changed. Under s.mu.
|
|
func (s *Service) remember(ctx context.Context, set config.SlurmSettings) {
|
|
if s.mem == nil || !s.loaded || !set.Enabled {
|
|
return
|
|
}
|
|
c, _ := s.cluster(set)
|
|
now := Kept{Controller: set.Controller, ControllerAddr: c.ControllerAddr, Nodes: map[string]slurm.Node{}}
|
|
for _, n := range c.Nodes {
|
|
now.Nodes[n.Name] = n
|
|
}
|
|
if now.equal(s.known) {
|
|
return
|
|
}
|
|
if err := s.mem.Keep(ctx, now); err != nil {
|
|
s.log.Warn("could not keep Slurm's nodes in the store", "err", err)
|
|
return
|
|
}
|
|
s.known = now
|
|
}
|
|
|
|
// controllerAddr is where the nodes reach slurmctld: the controller's
|
|
// management address, else the one kept. Under s.mu.
|
|
func (s *Service) controllerAddr(set config.SlurmSettings) string {
|
|
if n, ok := s.inv.Get(set.Controller); ok && n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if s.known.Controller == set.Controller {
|
|
return s.known.ControllerAddr
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// handOver moves each node of slurm.nodes a step between llmgw and Slurm,
|
|
// answering where each is. Under s.mu.
|
|
func (s *Service) handOver(set config.SlurmSettings) map[string]Phase {
|
|
out := map[string]Phase{}
|
|
if !set.Enabled {
|
|
// Off: every node is llmgw's again. What Slurm still runs is left
|
|
// to finish where it is; turning Slurm off with work lent is the
|
|
// operator's choice.
|
|
for node := range s.drained {
|
|
s.ready(node)
|
|
}
|
|
return out
|
|
}
|
|
for _, node := range set.Nodes {
|
|
out[node] = s.phase(set, node)
|
|
}
|
|
// A node no longer in slurm.nodes is llmgw's.
|
|
for node := range s.drained {
|
|
if !slices.Contains(set.Nodes, node) {
|
|
s.ready(node)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Service) phase(set config.SlurmSettings, node string) Phase {
|
|
n, ok := s.inv.Get(node)
|
|
st := s.status[node]
|
|
if set.IsLent(node) {
|
|
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
|
|
s.inv.SetState(node, inventory.Draining)
|
|
s.drained[node] = true
|
|
s.lendingAt[node] = time.Now()
|
|
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node,
|
|
"stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly))
|
|
}
|
|
if !ok || len(s.inv.Allocations(node)) > 0 {
|
|
return Lending
|
|
}
|
|
delete(s.lendingAt, node)
|
|
return Lent
|
|
}
|
|
// Not lent. Until the agent says Slurm has drained it with no job left,
|
|
// llmgw places nothing new on it: a new leader starts with every node
|
|
// ready. Cordoned, not drained: llmgw has nothing there to move.
|
|
if st == nil || slurmBusy(st) {
|
|
if ok && n.State == inventory.Ready {
|
|
s.inv.SetState(node, inventory.Cordoned)
|
|
s.drained[node] = true
|
|
if st != nil {
|
|
s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end",
|
|
"node", node, "jobs", st.Jobs)
|
|
}
|
|
}
|
|
if st == nil {
|
|
return Unknown
|
|
}
|
|
return Reclaiming
|
|
}
|
|
delete(s.lendingAt, node)
|
|
if s.drained[node] {
|
|
s.ready(node)
|
|
}
|
|
return Llmgw
|
|
}
|
|
|
|
// drainGrace is how long a drained replica waits for its replacement
|
|
// before it is stopped anyway: the reconciler's, which is not a setting.
|
|
var drainGrace = reconciler.DefaultConfig().DrainGrace
|
|
|
|
// slurmBusy is whether Slurm may still run work on a node: a job step left
|
|
// on it (or no telling), or a controller that would still give it one. A
|
|
// node the controller does not know of, or cannot be asked about, gets no
|
|
// job from it, so a Slurm that is broken never keeps a node from llmgw.
|
|
func slurmBusy(st *Reported) bool {
|
|
if st.Jobs != 0 {
|
|
return true
|
|
}
|
|
if st.State == "" {
|
|
return false
|
|
}
|
|
return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN")
|
|
}
|
|
|
|
func (s *Service) ready(node string) {
|
|
if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) {
|
|
s.inv.SetState(node, inventory.Ready)
|
|
s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node)
|
|
}
|
|
delete(s.drained, node)
|
|
}
|
|
|
|
// files are the controller's files, and the nodes left out of them for not
|
|
// having reported their hardware. Under s.mu.
|
|
func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) {
|
|
c, waiting := s.cluster(set)
|
|
if c.ControllerAddr == "" {
|
|
return nil, waiting
|
|
}
|
|
f := c.Files()
|
|
if s.containersReady(set) {
|
|
f["plugstack.conf"] = slurm.PlugstackConf
|
|
}
|
|
return f, waiting
|
|
}
|
|
|
|
// containersReady is whether jobs may ask for containers: every machine of
|
|
// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
|
|
// machine without the plugin it was told to load. Under s.mu.
|
|
func (s *Service) containersReady(set config.SlurmSettings) bool {
|
|
if !set.Containers {
|
|
return false
|
|
}
|
|
for _, n := range s.machines(set) {
|
|
if st := s.status[n]; st == nil || st.Containers == "" {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// cluster is what slurm.conf says: each node as it last reported, else as
|
|
// kept from before, and the nodes left out for having neither. Under s.mu.
|
|
func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
|
|
c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName()}
|
|
if !set.Enabled {
|
|
return c, nil
|
|
}
|
|
c.ControllerAddr = s.controllerAddr(set)
|
|
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
|
|
var waiting []string
|
|
for _, node := range set.Nodes {
|
|
sn, ok := s.reported(node)
|
|
if !ok {
|
|
sn, ok = s.known.Nodes[node]
|
|
}
|
|
if !ok {
|
|
waiting = append(waiting, node)
|
|
continue
|
|
}
|
|
sn.Lent = set.IsLent(node)
|
|
c.Nodes = append(c.Nodes, sn)
|
|
}
|
|
return c, waiting
|
|
}
|
|
|
|
// reported is a node as its agent and the inventory say now. Under s.mu.
|
|
func (s *Service) reported(node string) (slurm.Node, bool) {
|
|
n, ok := s.inv.Get(node)
|
|
st := s.status[node]
|
|
if !ok || n.MgmtAddr == "" || st == nil || st.Hardware == "" {
|
|
return slurm.Node{}, false
|
|
}
|
|
hw, err := slurm.ParseHardware(st.Hardware)
|
|
if err != nil {
|
|
return slurm.Node{}, false
|
|
}
|
|
sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw}
|
|
for _, g := range n.GPUs {
|
|
sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name})
|
|
}
|
|
return sn, true
|
|
}
|
|
|
|
// message is what a node is told. Under s.mu.
|
|
func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte,
|
|
files map[string]string) *llmgwv1.Slurm {
|
|
m := &llmgwv1.Slurm{}
|
|
if !set.Enabled {
|
|
return m
|
|
}
|
|
isNode := slices.Contains(set.Nodes, node)
|
|
isCtl := set.Controller == node && files != nil
|
|
if !isNode && !isCtl {
|
|
return m
|
|
}
|
|
m.Name, m.MungeKey, m.Lent = node, key, lent
|
|
m.Containers = set.Containers
|
|
m.Node = isNode
|
|
if addr := s.controllerAddr(set); addr != "" {
|
|
m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
|
|
}
|
|
if isNode && m.ConfServer == "" {
|
|
// Without its controller a node cannot start; it is told when the
|
|
// controller registers. Told nothing until then, not "no part": a
|
|
// node still running Slurm's jobs would stop them.
|
|
return nil
|
|
}
|
|
if isCtl {
|
|
m.Controller = true
|
|
m.Files = maps.Clone(files)
|
|
}
|
|
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
|
|
return nil
|
|
}
|
|
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
|
|
if !s.accountsOK {
|
|
return nil
|
|
}
|
|
m.ManageAccounts = true
|
|
for _, a := range s.accounts {
|
|
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// sharedPart puts the shared files in m, false when where they come from
|
|
// is not known yet: told nothing rather than no shared files, which would
|
|
// unmount them under running jobs. Under s.mu.
|
|
func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
|
|
sh := set.Shared
|
|
if sh.Path == "" {
|
|
return true
|
|
}
|
|
m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS}
|
|
switch {
|
|
case sh.Server == "":
|
|
case sh.Server == node:
|
|
m.Shared.Serve = true
|
|
for _, n := range s.machines(set) {
|
|
if n == node {
|
|
continue
|
|
}
|
|
if a := s.addr(set, n); a != "" {
|
|
m.Shared.Clients = append(m.Shared.Clients, a)
|
|
}
|
|
}
|
|
slices.Sort(m.Shared.Clients)
|
|
default:
|
|
a := s.addr(set, sh.Server)
|
|
if a == "" {
|
|
return false
|
|
}
|
|
if strings.Contains(a, ":") {
|
|
a = "[" + a + "]"
|
|
}
|
|
m.Shared.Nfs = a + ":" + sh.Path
|
|
}
|
|
return true
|
|
}
|
|
|
|
// usersPart puts the users in m, false while they have not been read.
|
|
// Under s.mu.
|
|
func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
|
|
if set.Users != config.UsersLlmgw {
|
|
return true
|
|
}
|
|
if !s.usersOK {
|
|
return false
|
|
}
|
|
m.ManageUsers = true
|
|
switch {
|
|
case set.Shared.Path == "":
|
|
m.MakeHomes = true // each machine's own /home
|
|
case set.Shared.Server != "":
|
|
m.MakeHomes = set.Shared.Server == node
|
|
default:
|
|
m.MakeHomes = set.Controller == node
|
|
}
|
|
for _, u := range s.users {
|
|
m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up
|
|
Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)})
|
|
}
|
|
return true
|
|
}
|
|
|
|
// machines are the controller and the nodes.
|
|
func (s *Service) machines(set config.SlurmSettings) []string {
|
|
out := slices.Clone(set.Nodes)
|
|
if !slices.Contains(out, set.Controller) {
|
|
out = append(out, set.Controller)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// addr is a machine's management address, from the inventory, else kept
|
|
// from before. Under s.mu.
|
|
func (s *Service) addr(set config.SlurmSettings, node string) string {
|
|
if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" {
|
|
return n.MgmtAddr
|
|
}
|
|
if node == set.Controller {
|
|
return s.controllerAddr(set)
|
|
}
|
|
return s.known.Nodes[node].Addr
|
|
}
|
|
|
|
// Status is the cluster as the service sees it now.
|
|
func (s *Service) Status() Status {
|
|
set := s.current()
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
|
|
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
|
|
if set.Users == config.UsersLlmgw {
|
|
out.UserCount = len(s.users)
|
|
for _, a := range s.accounts {
|
|
if a.Project != "" {
|
|
out.Accounts++
|
|
}
|
|
}
|
|
}
|
|
if !set.Enabled {
|
|
return out
|
|
}
|
|
if set.Containers {
|
|
out.Containers = "installing"
|
|
if s.containersReady(set) {
|
|
out.Containers = "on"
|
|
}
|
|
}
|
|
_, out.Waiting = s.files(set)
|
|
conn := s.agents.ConnectedNodes()
|
|
for _, node := range set.Nodes {
|
|
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
|
|
Connected: slices.Contains(conn, node)}
|
|
ns.Phase = s.peek(set, node)
|
|
if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending {
|
|
ns.StopsBy = at.Add(drainGrace)
|
|
}
|
|
out.Nodes = append(out.Nodes, ns)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// peek is phase without changing anything. Under s.mu.
|
|
func (s *Service) peek(set config.SlurmSettings, node string) Phase {
|
|
st := s.status[node]
|
|
if set.IsLent(node) {
|
|
if len(s.inv.Allocations(node)) > 0 {
|
|
return Lending
|
|
}
|
|
return Lent
|
|
}
|
|
switch {
|
|
case st == nil:
|
|
return Unknown
|
|
case slurmBusy(st):
|
|
return Reclaiming
|
|
}
|
|
return Llmgw
|
|
}
|
|
|
|
// Describe is one line for a log or a screen.
|
|
func (n NodeStatus) Describe() string {
|
|
switch n.Phase {
|
|
case Lending:
|
|
if !n.StopsBy.IsZero() {
|
|
// A replica moves once its replacement serves; one with nowhere
|
|
// else to go waits out the grace, and that wait is not a hang.
|
|
return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s",
|
|
n.Replicas, n.StopsBy.Local().Format(time.TimeOnly))
|
|
}
|
|
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
|
|
case Lent:
|
|
return "lent to Slurm"
|
|
case Reclaiming:
|
|
if n.Status != nil {
|
|
return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs)
|
|
}
|
|
return "being taken back"
|
|
case Llmgw:
|
|
return "llmgw's"
|
|
}
|
|
return "not heard from yet"
|
|
}
|