Files
llmgw/internal/control/slurmsvc/service.go
T
Ilker ManapandClaude Opus 5.5 fcda4d3f85
CI / test (push) Failing after 7m24s
Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 11:54:01 +01:00

869 lines
26 KiB
Go

// Package slurmsvc is the leader's side of the Slurm cluster run beside
// llmgw (BACKLOG P): it tells each node's agent what it is, writes the
// controller's files from what the nodes report, and hands nodes between
// llmgw and Slurm as the cluster's settings say.
//
// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does,
// and lent to Slurm once none is left. A node taken out of slurm.lent is
// drained in Slurm, and is llmgw's again when its last job has ended. Both
// are kept in force every tick, from the settings and what the agents say,
// so a leader that takes over carries on where the last one was.
package slurmsvc
import (
"context"
"fmt"
"log/slog"
"maps"
"net"
"slices"
"strconv"
"strings"
"sync"
"time"
"google.golang.org/protobuf/proto"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// Agents is how the service reaches the nodes.
type Agents interface {
Send(node string, msg *llmgwv1.ControlMessage) error
ConnectedNodes() []string
}
// Inventory is what the service reads and changes of llmgw's nodes.
type Inventory interface {
Get(id string) (inventory.Node, bool)
SetState(id string, st inventory.NodeState) bool
Allocations(nodeID string) []inventory.Allocation
Nodes() []inventory.Node
}
// Phase is where a node of slurm.nodes is between llmgw and Slurm.
type Phase string
const (
// Llmgw: drained in Slurm with no job left, llmgw's to place on.
Llmgw Phase = "llmgw"
// Lending: in slurm.lent, its replicas being moved off.
Lending Phase = "lending"
// Lent: Slurm's; llmgw places nothing there.
Lent Phase = "lent"
// Reclaiming: out of slurm.lent, drained in Slurm, jobs still running.
Reclaiming Phase = "reclaiming"
// Unknown: its agent has not said how it is in Slurm yet.
Unknown Phase = "unknown"
)
// NodeStatus is one node as the service sees it.
type NodeStatus struct {
Node string `json:"node"`
Phase Phase `json:"phase"`
Replicas int `json:"replicas"`
// StopsBy is, while it is being lent, when a replica still on it is
// stopped even with nowhere else to go: the drain's grace after the
// lend began. Zero when not known (a leader that took over meanwhile).
StopsBy time.Time `json:"stops_by,omitempty"`
Status *Reported `json:"slurm,omitempty"`
Connected bool `json:"connected"`
}
// Reported is the agent's SlurmStatus, for the admin API.
type Reported struct {
Version string `json:"version,omitempty"`
Hardware string `json:"hardware,omitempty"`
SlurmdRunning bool `json:"slurmd_running"`
SlurmctldRunning bool `json:"slurmctld_running"`
SlurmdbdRunning bool `json:"slurmdbd_running"`
// Shared is the shared path here: "mounted", "exported", "missing".
Shared string `json:"shared,omitempty"`
// Containers is what the machine has for jobs in containers, empty
// while not both of pyxis and enroot.
Containers string `json:"containers,omitempty"`
// Users is how many of llmgw's users the machine has.
Users int `json:"users"`
State string `json:"state,omitempty"`
Reason string `json:"reason,omitempty"`
Jobs int `json:"jobs"`
Error string `json:"error,omitempty"`
FilesSHA256 string `json:"files_sha256,omitempty"`
At time.Time `json:"at"`
}
// Status is the whole Slurm cluster, for `ctl slurm status`.
type Status struct {
Enabled bool `json:"enabled"`
Cluster string `json:"cluster"`
Controller string `json:"controller"`
Partition string `json:"partition"`
Ctld *Reported `json:"controller_status,omitempty"`
Nodes []NodeStatus `json:"nodes"`
// Waiting are nodes left out of slurm.conf until they report their
// hardware.
Waiting []string `json:"waiting,omitempty"`
// Shared is where the shared files are and come from; Users who keeps
// the users, and how many llmgw has.
Shared config.SharedStorage `json:"shared"`
Users string `json:"users,omitempty"`
UserCount int `json:"user_count,omitempty"`
// Containers is whether jobs may run in containers: "on" once every
// machine has pyxis and enroot, "installing" until then, empty when
// slurm.containers is off.
Containers string `json:"containers,omitempty"`
// Accounts is how many of Slurm's accounts are llmgw's projects'.
Accounts int `json:"accounts,omitempty"`
}
// User is a Linux user made from an llmgw account.
type User struct {
Login string `json:"login"`
UID int `json:"uid"`
Name string `json:"name"`
Email string `json:"email"`
SSHKeys []string `json:"ssh_keys,omitempty"`
}
// Service keeps the Slurm cluster.
type Service struct {
settings func() config.SlurmSettings
inv Inventory
agents Agents
key func(ctx context.Context) ([]byte, error)
log *slog.Logger
mu sync.Mutex
// status is each node's last SlurmStatus.
status map[string]*Reported
// drained are nodes this service put in Draining, which it alone makes
// ready again.
drained map[string]bool
// lendingAt is when this leader began moving a node's replicas off.
lendingAt map[string]time.Time
// sent is what each node was last told, so it is told again only when
// that changes or its stream is new.
sent map[string]*llmgwv1.Slurm
mungeKy []byte
kick chan struct{}
// mem keeps slurm.conf's nodes and the controller's address for the
// next leader; known is what it held when read, and what was kept
// since. loaded is false until it was read: until then the controller
// is sent no files that leave out a node, which would drop that node
// and its jobs.
mem Memory
known Kept
loaded bool
// started is the first pass, from which a node neither heard from nor
// kept is waited for, for HoldFor.
started time.Time
// userList reads the users from the accounts; users is what it last
// said, read at usersAt. Until it has been read once no node is told
// anything of users: an empty list would take every user away.
userList func(ctx context.Context) ([]User, error)
users []User
usersAt time.Time
usersOK bool
// accountList reads Slurm's accounts from the projects, with the users;
// accountsOK is false until it has been read, and the controller is
// told nothing of accounts until then: none would take every one away.
accountList func(ctx context.Context) ([]slurm.Account, error)
accounts []slurm.Account
accountsOK bool
// jobSink records the jobs that ended; recorded are those it took,
// recording whether a recording is under way.
jobSink func(ctx context.Context, jobs []JobRecord) error
recorded map[string]time.Time
recording bool
}
// JobRecord is a job that ended, for llmgw's accounting.
type JobRecord struct {
ID, Login, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SetAccounts has Slurm's accounts read with list, beside the users.
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
s.mu.Lock()
s.accountList = list
s.mu.Unlock()
}
// SetJobs has the jobs that ended recorded with record; each is given to it
// once it took it without an error.
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
s.mu.Lock()
s.jobSink = record
s.mu.Unlock()
}
// AccountsOf are the accounts a login may run jobs on.
func (s *Service) AccountsOf(login string) []string {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for _, a := range s.accounts {
if slices.Contains(a.Users, login) {
out = append(out, a.Name)
}
}
return out
}
// UsersEvery is how often the users are read again without being asked.
var UsersEvery = 30 * time.Second
// SetUsers has the users read with list, for slurm.users "llmgw".
func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) {
s.mu.Lock()
s.userList = list
s.mu.Unlock()
}
// UsersChanged has the users read again now: an account or its keys
// changed.
func (s *Service) UsersChanged() {
s.mu.Lock()
s.usersAt = time.Time{}
s.mu.Unlock()
s.Kick()
}
// readUsers reads the users when they are due. Under s.mu.
func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery {
return
}
us, err := s.userList(ctx)
s.usersAt = time.Now()
if err != nil {
s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err)
return
}
s.users, s.usersOK = us, true
if s.accountList == nil {
return
}
as, err := s.accountList(ctx)
if err != nil {
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
return
}
s.accounts, s.accountsOK = as, true
}
// takeJobs records the jobs a controller says ended that are not recorded
// yet, away from the agent's stream.
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
s.mu.Lock()
if s.jobSink == nil || s.recording {
s.mu.Unlock()
return
}
if s.recorded == nil {
s.recorded = map[string]time.Time{}
}
for id, at := range s.recorded {
if time.Since(at) > 3*24*time.Hour {
delete(s.recorded, id)
}
}
var jobs []JobRecord
for _, j := range ended {
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
continue
}
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
}
if len(jobs) == 0 {
s.mu.Unlock()
return
}
sink := s.jobSink
s.recording = true
s.mu.Unlock()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
err := sink(ctx, jobs)
s.mu.Lock()
defer s.mu.Unlock()
s.recording = false
if err != nil {
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
return
}
for _, j := range jobs {
s.recorded[j.ID] = time.Now()
}
}()
}
// HoldFor is how long a new leader waits for a node it neither has heard
// from nor finds kept before it writes the controller's files without it:
// on a cluster that kept nothing yet (one upgraded to this), the nodes
// report within seconds of a new leader, and the controller keeps the
// node and its jobs meanwhile.
var HoldFor = 2 * time.Minute
// New is a Service; key gives the munge key, made once from a cluster
// secret.
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service {
if log == nil {
log = slog.Default()
}
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{},
kick: make(chan struct{}, 1)}
}
// SetMemory has slurm.conf's nodes kept in mem, and read from it first.
func (s *Service) SetMemory(mem Memory) {
s.mu.Lock()
s.mem = mem
s.mu.Unlock()
}
// current is the settings, slurm.all_nodes resolved to the nodes the
// inventory has.
func (s *Service) current() config.SlurmSettings {
set := s.settings()
if !set.AllNodes {
return set
}
var ids []string
for _, n := range s.inv.Nodes() {
ids = append(ids, n.ID)
}
return set.WithNodes(ids)
}
// Every is how often the service looks again without being asked.
var Every = 10 * time.Second
// Run keeps the cluster until ctx ends.
func (s *Service) Run(ctx context.Context) {
t := time.NewTicker(Every)
defer t.Stop()
for {
s.Tick(ctx)
select {
case <-ctx.Done():
return
case <-t.C:
case <-s.kick:
}
}
}
// Kick has the service look again now: the settings changed.
func (s *Service) Kick() {
select {
case s.kick <- struct{}{}:
default:
}
}
// OnSlurmStatus records what a node's agent says.
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
s.mu.Lock()
prev := s.status[node]
s.status[node] = r
isCtl := s.settings().Controller == node
s.mu.Unlock()
if isCtl && len(st.GetEnded()) > 0 {
s.takeJobs(st.GetEnded())
}
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
}
if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs {
s.Kick()
}
}
// OnRegister is a node's new stream: it is told again what it is.
func (s *Service) OnRegister(node string) {
s.mu.Lock()
delete(s.sent, node)
s.mu.Unlock()
s.Kick()
}
// Tick does one pass: the nodes handed over as far as they can be, and
// each told what it is.
func (s *Service) Tick(ctx context.Context) {
set := s.current()
s.mu.Lock()
defer s.mu.Unlock()
s.recall(ctx)
s.readUsers(ctx, set)
if s.started.IsZero() {
s.started = time.Now()
}
phases := s.handOver(set)
var key []byte
if set.Enabled {
if s.mungeKy == nil {
k, err := s.key(ctx)
if err != nil {
s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err)
return
}
s.mungeKy = k
}
key = s.mungeKy
}
files, waiting := s.files(set)
// Not yet knowing what the last leader knew, a slurm.conf without some
// node would take it, and its jobs, from the controller: kept back.
hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
for _, node := range s.agents.ConnectedNodes() {
if hold && node == set.Controller {
continue
}
m := s.message(set, node, phases[node] == Lent, key, files)
if m == nil {
continue
}
if old, ok := s.sent[node]; ok && proto.Equal(old, m) {
continue
}
if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil {
s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err)
continue
}
s.sent[node] = m
}
s.remember(ctx, set)
}
// recall reads what the last leader kept, once. Under s.mu.
func (s *Service) recall(ctx context.Context) {
if s.mem == nil || s.loaded {
return
}
k, err := s.mem.Recall(ctx)
if err != nil {
s.log.Warn("could not read Slurm's nodes from the store; the controller's files wait for every node",
"err", err)
return
}
s.known, s.loaded = k, true
}
// remember keeps slurm.conf's nodes and the controller's address when they
// changed. Under s.mu.
func (s *Service) remember(ctx context.Context, set config.SlurmSettings) {
if s.mem == nil || !s.loaded || !set.Enabled {
return
}
c, _ := s.cluster(set)
now := Kept{Controller: set.Controller, ControllerAddr: c.ControllerAddr, Nodes: map[string]slurm.Node{}}
for _, n := range c.Nodes {
now.Nodes[n.Name] = n
}
if now.equal(s.known) {
return
}
if err := s.mem.Keep(ctx, now); err != nil {
s.log.Warn("could not keep Slurm's nodes in the store", "err", err)
return
}
s.known = now
}
// controllerAddr is where the nodes reach slurmctld: the controller's
// management address, else the one kept. Under s.mu.
func (s *Service) controllerAddr(set config.SlurmSettings) string {
if n, ok := s.inv.Get(set.Controller); ok && n.MgmtAddr != "" {
return n.MgmtAddr
}
if s.known.Controller == set.Controller {
return s.known.ControllerAddr
}
return ""
}
// handOver moves each node of slurm.nodes a step between llmgw and Slurm,
// answering where each is. Under s.mu.
func (s *Service) handOver(set config.SlurmSettings) map[string]Phase {
out := map[string]Phase{}
if !set.Enabled {
// Off: every node is llmgw's again. What Slurm still runs is left
// to finish where it is; turning Slurm off with work lent is the
// operator's choice.
for node := range s.drained {
s.ready(node)
}
return out
}
for _, node := range set.Nodes {
out[node] = s.phase(set, node)
}
// A node no longer in slurm.nodes is llmgw's.
for node := range s.drained {
if !slices.Contains(set.Nodes, node) {
s.ready(node)
}
}
return out
}
func (s *Service) phase(set config.SlurmSettings, node string) Phase {
n, ok := s.inv.Get(node)
st := s.status[node]
if set.IsLent(node) {
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
s.inv.SetState(node, inventory.Draining)
s.drained[node] = true
s.lendingAt[node] = time.Now()
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node,
"stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly))
}
if !ok || len(s.inv.Allocations(node)) > 0 {
return Lending
}
delete(s.lendingAt, node)
return Lent
}
// Not lent. Until the agent says Slurm has drained it with no job left,
// llmgw places nothing new on it: a new leader starts with every node
// ready. Cordoned, not drained: llmgw has nothing there to move.
if st == nil || slurmBusy(st) {
if ok && n.State == inventory.Ready {
s.inv.SetState(node, inventory.Cordoned)
s.drained[node] = true
if st != nil {
s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end",
"node", node, "jobs", st.Jobs)
}
}
if st == nil {
return Unknown
}
return Reclaiming
}
delete(s.lendingAt, node)
if s.drained[node] {
s.ready(node)
}
return Llmgw
}
// drainGrace is how long a drained replica waits for its replacement
// before it is stopped anyway: the reconciler's, which is not a setting.
var drainGrace = reconciler.DefaultConfig().DrainGrace
// slurmBusy is whether Slurm may still run work on a node: a job step left
// on it (or no telling), or a controller that would still give it one. A
// node the controller does not know of, or cannot be asked about, gets no
// job from it, so a Slurm that is broken never keeps a node from llmgw.
func slurmBusy(st *Reported) bool {
if st.Jobs != 0 {
return true
}
if st.State == "" {
return false
}
return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN")
}
func (s *Service) ready(node string) {
if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) {
s.inv.SetState(node, inventory.Ready)
s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node)
}
delete(s.drained, node)
}
// files are the controller's files, and the nodes left out of them for not
// having reported their hardware. Under s.mu.
func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) {
c, waiting := s.cluster(set)
if c.ControllerAddr == "" {
return nil, waiting
}
f := c.Files()
if s.containersReady(set) {
f["plugstack.conf"] = slurm.PlugstackConf
}
return f, waiting
}
// containersReady is whether jobs may ask for containers: every machine of
// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
// machine without the plugin it was told to load. Under s.mu.
func (s *Service) containersReady(set config.SlurmSettings) bool {
if !set.Containers {
return false
}
for _, n := range s.machines(set) {
if st := s.status[n]; st == nil || st.Containers == "" {
return false
}
}
return true
}
// cluster is what slurm.conf says: each node as it last reported, else as
// kept from before, and the nodes left out for having neither. Under s.mu.
func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName()}
if !set.Enabled {
return c, nil
}
c.ControllerAddr = s.controllerAddr(set)
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
var waiting []string
for _, node := range set.Nodes {
sn, ok := s.reported(node)
if !ok {
sn, ok = s.known.Nodes[node]
}
if !ok {
waiting = append(waiting, node)
continue
}
sn.Lent = set.IsLent(node)
c.Nodes = append(c.Nodes, sn)
}
return c, waiting
}
// reported is a node as its agent and the inventory say now. Under s.mu.
func (s *Service) reported(node string) (slurm.Node, bool) {
n, ok := s.inv.Get(node)
st := s.status[node]
if !ok || n.MgmtAddr == "" || st == nil || st.Hardware == "" {
return slurm.Node{}, false
}
hw, err := slurm.ParseHardware(st.Hardware)
if err != nil {
return slurm.Node{}, false
}
sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw}
for _, g := range n.GPUs {
sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name})
}
return sn, true
}
// message is what a node is told. Under s.mu.
func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte,
files map[string]string) *llmgwv1.Slurm {
m := &llmgwv1.Slurm{}
if !set.Enabled {
return m
}
isNode := slices.Contains(set.Nodes, node)
isCtl := set.Controller == node && files != nil
if !isNode && !isCtl {
return m
}
m.Name, m.MungeKey, m.Lent = node, key, lent
m.Containers = set.Containers
m.Node = isNode
if addr := s.controllerAddr(set); addr != "" {
m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
}
if isNode && m.ConfServer == "" {
// Without its controller a node cannot start; it is told when the
// controller registers. Told nothing until then, not "no part": a
// node still running Slurm's jobs would stop them.
return nil
}
if isCtl {
m.Controller = true
m.Files = maps.Clone(files)
}
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
return nil
}
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
if !s.accountsOK {
return nil
}
m.ManageAccounts = true
for _, a := range s.accounts {
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
}
}
return m
}
// sharedPart puts the shared files in m, false when where they come from
// is not known yet: told nothing rather than no shared files, which would
// unmount them under running jobs. Under s.mu.
func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
sh := set.Shared
if sh.Path == "" {
return true
}
m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS}
switch {
case sh.Server == "":
case sh.Server == node:
m.Shared.Serve = true
for _, n := range s.machines(set) {
if n == node {
continue
}
if a := s.addr(set, n); a != "" {
m.Shared.Clients = append(m.Shared.Clients, a)
}
}
slices.Sort(m.Shared.Clients)
default:
a := s.addr(set, sh.Server)
if a == "" {
return false
}
if strings.Contains(a, ":") {
a = "[" + a + "]"
}
m.Shared.Nfs = a + ":" + sh.Path
}
return true
}
// usersPart puts the users in m, false while they have not been read.
// Under s.mu.
func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
if set.Users != config.UsersLlmgw {
return true
}
if !s.usersOK {
return false
}
m.ManageUsers = true
switch {
case set.Shared.Path == "":
m.MakeHomes = true // each machine's own /home
case set.Shared.Server != "":
m.MakeHomes = set.Shared.Server == node
default:
m.MakeHomes = set.Controller == node
}
for _, u := range s.users {
m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up
Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)})
}
return true
}
// machines are the controller and the nodes.
func (s *Service) machines(set config.SlurmSettings) []string {
out := slices.Clone(set.Nodes)
if !slices.Contains(out, set.Controller) {
out = append(out, set.Controller)
}
return out
}
// addr is a machine's management address, from the inventory, else kept
// from before. Under s.mu.
func (s *Service) addr(set config.SlurmSettings, node string) string {
if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" {
return n.MgmtAddr
}
if node == set.Controller {
return s.controllerAddr(set)
}
return s.known.Nodes[node].Addr
}
// Status is the cluster as the service sees it now.
func (s *Service) Status() Status {
set := s.current()
s.mu.Lock()
defer s.mu.Unlock()
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
if set.Users == config.UsersLlmgw {
out.UserCount = len(s.users)
for _, a := range s.accounts {
if a.Project != "" {
out.Accounts++
}
}
}
if !set.Enabled {
return out
}
if set.Containers {
out.Containers = "installing"
if s.containersReady(set) {
out.Containers = "on"
}
}
_, out.Waiting = s.files(set)
conn := s.agents.ConnectedNodes()
for _, node := range set.Nodes {
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
Connected: slices.Contains(conn, node)}
ns.Phase = s.peek(set, node)
if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending {
ns.StopsBy = at.Add(drainGrace)
}
out.Nodes = append(out.Nodes, ns)
}
return out
}
// peek is phase without changing anything. Under s.mu.
func (s *Service) peek(set config.SlurmSettings, node string) Phase {
st := s.status[node]
if set.IsLent(node) {
if len(s.inv.Allocations(node)) > 0 {
return Lending
}
return Lent
}
switch {
case st == nil:
return Unknown
case slurmBusy(st):
return Reclaiming
}
return Llmgw
}
// Describe is one line for a log or a screen.
func (n NodeStatus) Describe() string {
switch n.Phase {
case Lending:
if !n.StopsBy.IsZero() {
// A replica moves once its replacement serves; one with nowhere
// else to go waits out the grace, and that wait is not a hang.
return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s",
n.Replicas, n.StopsBy.Local().Format(time.TimeOnly))
}
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
case Lent:
return "lent to Slurm"
case Reclaiming:
if n.Status != nil {
return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs)
}
return "being taken back"
case Llmgw:
return "llmgw's"
}
return "not heard from yet"
}