// Package slurmsvc is the leader's side of the Slurm cluster run beside // llmgw (BACKLOG P): it tells each node's agent what it is, writes the // controller's files from what the nodes report, and hands nodes between // llmgw and Slurm as the cluster's settings say. // // A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does, // and lent to Slurm once none is left. A node taken out of slurm.lent is // drained in Slurm, and is llmgw's again when its last job has ended. Both // are kept in force every tick, from the settings and what the agents say, // so a leader that takes over carries on where the last one was. package slurmsvc import ( "context" "fmt" "log/slog" "maps" "net" "slices" "strconv" "strings" "sync" "time" "google.golang.org/protobuf/proto" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) // Agents is how the service reaches the nodes. type Agents interface { Send(node string, msg *llmgwv1.ControlMessage) error ConnectedNodes() []string } // Inventory is what the service reads and changes of llmgw's nodes. type Inventory interface { Get(id string) (inventory.Node, bool) SetState(id string, st inventory.NodeState) bool Allocations(nodeID string) []inventory.Allocation Nodes() []inventory.Node } // Phase is where a node of slurm.nodes is between llmgw and Slurm. type Phase string const ( // Llmgw: drained in Slurm with no job left, llmgw's to place on. Llmgw Phase = "llmgw" // Lending: in slurm.lent, its replicas being moved off. Lending Phase = "lending" // Lent: Slurm's; llmgw places nothing there. Lent Phase = "lent" // Reclaiming: out of slurm.lent, drained in Slurm, jobs still running. Reclaiming Phase = "reclaiming" // Unknown: its agent has not said how it is in Slurm yet. Unknown Phase = "unknown" ) // NodeStatus is one node as the service sees it. type NodeStatus struct { Node string `json:"node"` Phase Phase `json:"phase"` Replicas int `json:"replicas"` // StopsBy is, while it is being lent, when a replica still on it is // stopped even with nowhere else to go: the drain's grace after the // lend began. Zero when not known (a leader that took over meanwhile). StopsBy time.Time `json:"stops_by,omitempty"` Status *Reported `json:"slurm,omitempty"` Connected bool `json:"connected"` } // Reported is the agent's SlurmStatus, for the admin API. type Reported struct { Version string `json:"version,omitempty"` Hardware string `json:"hardware,omitempty"` SlurmdRunning bool `json:"slurmd_running"` SlurmctldRunning bool `json:"slurmctld_running"` SlurmdbdRunning bool `json:"slurmdbd_running"` // Shared is the shared path here: "mounted", "exported", "missing". Shared string `json:"shared,omitempty"` // Containers is what the machine has for jobs in containers, empty // while not both of pyxis and enroot. Containers string `json:"containers,omitempty"` // Users is how many of llmgw's users the machine has. Users int `json:"users"` State string `json:"state,omitempty"` Reason string `json:"reason,omitempty"` Jobs int `json:"jobs"` Error string `json:"error,omitempty"` FilesSHA256 string `json:"files_sha256,omitempty"` At time.Time `json:"at"` } // Status is the whole Slurm cluster, for `ctl slurm status`. type Status struct { Enabled bool `json:"enabled"` Cluster string `json:"cluster"` Controller string `json:"controller"` Partition string `json:"partition"` Ctld *Reported `json:"controller_status,omitempty"` Nodes []NodeStatus `json:"nodes"` // Waiting are nodes left out of slurm.conf until they report their // hardware. Waiting []string `json:"waiting,omitempty"` // Shared is where the shared files are and come from; Users who keeps // the users, and how many llmgw has. Shared config.SharedStorage `json:"shared"` Users string `json:"users,omitempty"` UserCount int `json:"user_count,omitempty"` // Containers is whether jobs may run in containers: "on" once every // machine has pyxis and enroot, "installing" until then, empty when // slurm.containers is off. Containers string `json:"containers,omitempty"` // Accounts is how many of Slurm's accounts are llmgw's projects'. Accounts int `json:"accounts,omitempty"` } // User is a Linux user made from an llmgw account. type User struct { Login string `json:"login"` UID int `json:"uid"` Name string `json:"name"` Email string `json:"email"` SSHKeys []string `json:"ssh_keys,omitempty"` } // Service keeps the Slurm cluster. type Service struct { settings func() config.SlurmSettings inv Inventory agents Agents key func(ctx context.Context) ([]byte, error) log *slog.Logger mu sync.Mutex // status is each node's last SlurmStatus. status map[string]*Reported // drained are nodes this service put in Draining, which it alone makes // ready again. drained map[string]bool // lendingAt is when this leader began moving a node's replicas off. lendingAt map[string]time.Time // sent is what each node was last told, so it is told again only when // that changes or its stream is new. sent map[string]*llmgwv1.Slurm mungeKy []byte kick chan struct{} // mem keeps slurm.conf's nodes and the controller's address for the // next leader; known is what it held when read, and what was kept // since. loaded is false until it was read: until then the controller // is sent no files that leave out a node, which would drop that node // and its jobs. mem Memory known Kept loaded bool // started is the first pass, from which a node neither heard from nor // kept is waited for, for HoldFor. started time.Time // userList reads the users from the accounts; users is what it last // said, read at usersAt. Until it has been read once no node is told // anything of users: an empty list would take every user away. userList func(ctx context.Context) ([]User, error) users []User usersAt time.Time usersOK bool // accountList reads Slurm's accounts from the projects, with the users; // accountsOK is false until it has been read, and the controller is // told nothing of accounts until then: none would take every one away. accountList func(ctx context.Context) ([]slurm.Account, error) accounts []slurm.Account accountsOK bool // jobSink records the jobs that ended; recorded are those it took, // recording whether a recording is under way. jobSink func(ctx context.Context, jobs []JobRecord) error recorded map[string]time.Time recording bool } // JobRecord is a job that ended, for llmgw's accounting. type JobRecord struct { ID, Login, Account, Partition, State string Start, End time.Time Elapsed time.Duration CPUs, GPUs int } // SetAccounts has Slurm's accounts read with list, beside the users. func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) { s.mu.Lock() s.accountList = list s.mu.Unlock() } // SetJobs has the jobs that ended recorded with record; each is given to it // once it took it without an error. func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) { s.mu.Lock() s.jobSink = record s.mu.Unlock() } // AccountsOf are the accounts a login may run jobs on. func (s *Service) AccountsOf(login string) []string { s.mu.Lock() defer s.mu.Unlock() var out []string for _, a := range s.accounts { if slices.Contains(a.Users, login) { out = append(out, a.Name) } } return out } // UsersEvery is how often the users are read again without being asked. var UsersEvery = 30 * time.Second // SetUsers has the users read with list, for slurm.users "llmgw". func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) { s.mu.Lock() s.userList = list s.mu.Unlock() } // UsersChanged has the users read again now: an account or its keys // changed. func (s *Service) UsersChanged() { s.mu.Lock() s.usersAt = time.Time{} s.mu.Unlock() s.Kick() } // readUsers reads the users when they are due. Under s.mu. func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) { if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery { return } us, err := s.userList(ctx) s.usersAt = time.Now() if err != nil { s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err) return } s.users, s.usersOK = us, true if s.accountList == nil { return } as, err := s.accountList(ctx) if err != nil { s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err) return } s.accounts, s.accountsOK = as, true } // takeJobs records the jobs a controller says ended that are not recorded // yet, away from the agent's stream. func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) { s.mu.Lock() if s.jobSink == nil || s.recording { s.mu.Unlock() return } if s.recorded == nil { s.recorded = map[string]time.Time{} } for id, at := range s.recorded { if time.Since(at) > 3*24*time.Hour { delete(s.recorded, id) } } var jobs []JobRecord for _, j := range ended { if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" { continue } jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(), State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0), Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())}) } if len(jobs) == 0 { s.mu.Unlock() return } sink := s.jobSink s.recording = true s.mu.Unlock() go func() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := sink(ctx, jobs) s.mu.Lock() defer s.mu.Unlock() s.recording = false if err != nil { s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err) return } for _, j := range jobs { s.recorded[j.ID] = time.Now() } }() } // HoldFor is how long a new leader waits for a node it neither has heard // from nor finds kept before it writes the controller's files without it: // on a cluster that kept nothing yet (one upgraded to this), the nodes // report within seconds of a new leader, and the controller keeps the // node and its jobs meanwhile. var HoldFor = 2 * time.Minute // New is a Service; key gives the munge key, made once from a cluster // secret. func New(settings func() config.SlurmSettings, inv Inventory, agents Agents, key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service { if log == nil { log = slog.Default() } return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log, status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{}, kick: make(chan struct{}, 1)} } // SetMemory has slurm.conf's nodes kept in mem, and read from it first. func (s *Service) SetMemory(mem Memory) { s.mu.Lock() s.mem = mem s.mu.Unlock() } // current is the settings, slurm.all_nodes resolved to the nodes the // inventory has. func (s *Service) current() config.SlurmSettings { set := s.settings() if !set.AllNodes { return set } var ids []string for _, n := range s.inv.Nodes() { ids = append(ids, n.ID) } return set.WithNodes(ids) } // Every is how often the service looks again without being asked. var Every = 10 * time.Second // Run keeps the cluster until ctx ends. func (s *Service) Run(ctx context.Context) { t := time.NewTicker(Every) defer t.Stop() for { s.Tick(ctx) select { case <-ctx.Done(): return case <-t.C: case <-s.kick: } } } // Kick has the service look again now: the settings changed. func (s *Service) Kick() { select { case s.kick <- struct{}{}: default: } } // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(), SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() prev := s.status[node] s.status[node] = r isCtl := s.settings().Controller == node s.mu.Unlock() if isCtl && len(st.GetEnded()) > 0 { s.takeJobs(st.GetEnded()) } if r.Error != "" && (prev == nil || prev.Error != r.Error) { s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error) } if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs { s.Kick() } } // OnRegister is a node's new stream: it is told again what it is. func (s *Service) OnRegister(node string) { s.mu.Lock() delete(s.sent, node) s.mu.Unlock() s.Kick() } // Tick does one pass: the nodes handed over as far as they can be, and // each told what it is. func (s *Service) Tick(ctx context.Context) { set := s.current() s.mu.Lock() defer s.mu.Unlock() s.recall(ctx) s.readUsers(ctx, set) if s.started.IsZero() { s.started = time.Now() } phases := s.handOver(set) var key []byte if set.Enabled { if s.mungeKy == nil { k, err := s.key(ctx) if err != nil { s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err) return } s.mungeKy = k } key = s.mungeKy } files, waiting := s.files(set) // Not yet knowing what the last leader knew, a slurm.conf without some // node would take it, and its jobs, from the controller: kept back. hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor) for _, node := range s.agents.ConnectedNodes() { if hold && node == set.Controller { continue } m := s.message(set, node, phases[node] == Lent, key, files) if m == nil { continue } if old, ok := s.sent[node]; ok && proto.Equal(old, m) { continue } if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil { s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err) continue } s.sent[node] = m } s.remember(ctx, set) } // recall reads what the last leader kept, once. Under s.mu. func (s *Service) recall(ctx context.Context) { if s.mem == nil || s.loaded { return } k, err := s.mem.Recall(ctx) if err != nil { s.log.Warn("could not read Slurm's nodes from the store; the controller's files wait for every node", "err", err) return } s.known, s.loaded = k, true } // remember keeps slurm.conf's nodes and the controller's address when they // changed. Under s.mu. func (s *Service) remember(ctx context.Context, set config.SlurmSettings) { if s.mem == nil || !s.loaded || !set.Enabled { return } c, _ := s.cluster(set) now := Kept{Controller: set.Controller, ControllerAddr: c.ControllerAddr, Nodes: map[string]slurm.Node{}} for _, n := range c.Nodes { now.Nodes[n.Name] = n } if now.equal(s.known) { return } if err := s.mem.Keep(ctx, now); err != nil { s.log.Warn("could not keep Slurm's nodes in the store", "err", err) return } s.known = now } // controllerAddr is where the nodes reach slurmctld: the controller's // management address, else the one kept. Under s.mu. func (s *Service) controllerAddr(set config.SlurmSettings) string { if n, ok := s.inv.Get(set.Controller); ok && n.MgmtAddr != "" { return n.MgmtAddr } if s.known.Controller == set.Controller { return s.known.ControllerAddr } return "" } // handOver moves each node of slurm.nodes a step between llmgw and Slurm, // answering where each is. Under s.mu. func (s *Service) handOver(set config.SlurmSettings) map[string]Phase { out := map[string]Phase{} if !set.Enabled { // Off: every node is llmgw's again. What Slurm still runs is left // to finish where it is; turning Slurm off with work lent is the // operator's choice. for node := range s.drained { s.ready(node) } return out } for _, node := range set.Nodes { out[node] = s.phase(set, node) } // A node no longer in slurm.nodes is llmgw's. for node := range s.drained { if !slices.Contains(set.Nodes, node) { s.ready(node) } } return out } func (s *Service) phase(set config.SlurmSettings, node string) Phase { n, ok := s.inv.Get(node) st := s.status[node] if set.IsLent(node) { if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) { s.inv.SetState(node, inventory.Draining) s.drained[node] = true s.lendingAt[node] = time.Now() s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node, "stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly)) } if !ok || len(s.inv.Allocations(node)) > 0 { return Lending } delete(s.lendingAt, node) return Lent } // Not lent. Until the agent says Slurm has drained it with no job left, // llmgw places nothing new on it: a new leader starts with every node // ready. Cordoned, not drained: llmgw has nothing there to move. if st == nil || slurmBusy(st) { if ok && n.State == inventory.Ready { s.inv.SetState(node, inventory.Cordoned) s.drained[node] = true if st != nil { s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end", "node", node, "jobs", st.Jobs) } } if st == nil { return Unknown } return Reclaiming } delete(s.lendingAt, node) if s.drained[node] { s.ready(node) } return Llmgw } // drainGrace is how long a drained replica waits for its replacement // before it is stopped anyway: the reconciler's, which is not a setting. var drainGrace = reconciler.DefaultConfig().DrainGrace // slurmBusy is whether Slurm may still run work on a node: a job step left // on it (or no telling), or a controller that would still give it one. A // node the controller does not know of, or cannot be asked about, gets no // job from it, so a Slurm that is broken never keeps a node from llmgw. func slurmBusy(st *Reported) bool { if st.Jobs != 0 { return true } if st.State == "" { return false } return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN") } func (s *Service) ready(node string) { if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) { s.inv.SetState(node, inventory.Ready) s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node) } delete(s.drained, node) } // files are the controller's files, and the nodes left out of them for not // having reported their hardware. Under s.mu. func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) { c, waiting := s.cluster(set) if c.ControllerAddr == "" { return nil, waiting } f := c.Files() if s.containersReady(set) { f["plugstack.conf"] = slurm.PlugstackConf } return f, waiting } // containersReady is whether jobs may ask for containers: every machine of // the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a // machine without the plugin it was told to load. Under s.mu. func (s *Service) containersReady(set config.SlurmSettings) bool { if !set.Containers { return false } for _, n := range s.machines(set) { if st := s.status[n]; st == nil || st.Containers == "" { return false } } return true } // cluster is what slurm.conf says: each node as it last reported, else as // kept from before, and the nodes left out for having neither. Under s.mu. func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) { c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName()} if !set.Enabled { return c, nil } c.ControllerAddr = s.controllerAddr(set) c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil var waiting []string for _, node := range set.Nodes { sn, ok := s.reported(node) if !ok { sn, ok = s.known.Nodes[node] } if !ok { waiting = append(waiting, node) continue } sn.Lent = set.IsLent(node) c.Nodes = append(c.Nodes, sn) } return c, waiting } // reported is a node as its agent and the inventory say now. Under s.mu. func (s *Service) reported(node string) (slurm.Node, bool) { n, ok := s.inv.Get(node) st := s.status[node] if !ok || n.MgmtAddr == "" || st == nil || st.Hardware == "" { return slurm.Node{}, false } hw, err := slurm.ParseHardware(st.Hardware) if err != nil { return slurm.Node{}, false } sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw} for _, g := range n.GPUs { sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name}) } return sn, true } // message is what a node is told. Under s.mu. func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte, files map[string]string) *llmgwv1.Slurm { m := &llmgwv1.Slurm{} if !set.Enabled { return m } isNode := slices.Contains(set.Nodes, node) isCtl := set.Controller == node && files != nil if !isNode && !isCtl { return m } m.Name, m.MungeKey, m.Lent = node, key, lent m.Containers = set.Containers m.Node = isNode if addr := s.controllerAddr(set); addr != "" { m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort)) } if isNode && m.ConfServer == "" { // Without its controller a node cannot start; it is told when the // controller registers. Told nothing until then, not "no part": a // node still running Slurm's jobs would stop them. return nil } if isCtl { m.Controller = true m.Files = maps.Clone(files) } if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) { return nil } if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil { if !s.accountsOK { return nil } m.ManageAccounts = true for _, a := range s.accounts { m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)}) } } return m } // sharedPart puts the shared files in m, false when where they come from // is not known yet: told nothing rather than no shared files, which would // unmount them under running jobs. Under s.mu. func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { sh := set.Shared if sh.Path == "" { return true } m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS} switch { case sh.Server == "": case sh.Server == node: m.Shared.Serve = true for _, n := range s.machines(set) { if n == node { continue } if a := s.addr(set, n); a != "" { m.Shared.Clients = append(m.Shared.Clients, a) } } slices.Sort(m.Shared.Clients) default: a := s.addr(set, sh.Server) if a == "" { return false } if strings.Contains(a, ":") { a = "[" + a + "]" } m.Shared.Nfs = a + ":" + sh.Path } return true } // usersPart puts the users in m, false while they have not been read. // Under s.mu. func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { if set.Users != config.UsersLlmgw { return true } if !s.usersOK { return false } m.ManageUsers = true switch { case set.Shared.Path == "": m.MakeHomes = true // each machine's own /home case set.Shared.Server != "": m.MakeHomes = set.Shared.Server == node default: m.MakeHomes = set.Controller == node } for _, u := range s.users { m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)}) } return true } // machines are the controller and the nodes. func (s *Service) machines(set config.SlurmSettings) []string { out := slices.Clone(set.Nodes) if !slices.Contains(out, set.Controller) { out = append(out, set.Controller) } return out } // addr is a machine's management address, from the inventory, else kept // from before. Under s.mu. func (s *Service) addr(set config.SlurmSettings, node string) string { if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" { return n.MgmtAddr } if node == set.Controller { return s.controllerAddr(set) } return s.known.Nodes[node].Addr } // Status is the cluster as the service sees it now. func (s *Service) Status() Status { set := s.current() s.mu.Lock() defer s.mu.Unlock() out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users} if set.Users == config.UsersLlmgw { out.UserCount = len(s.users) for _, a := range s.accounts { if a.Project != "" { out.Accounts++ } } } if !set.Enabled { return out } if set.Containers { out.Containers = "installing" if s.containersReady(set) { out.Containers = "on" } } _, out.Waiting = s.files(set) conn := s.agents.ConnectedNodes() for _, node := range set.Nodes { ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)), Connected: slices.Contains(conn, node)} ns.Phase = s.peek(set, node) if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending { ns.StopsBy = at.Add(drainGrace) } out.Nodes = append(out.Nodes, ns) } return out } // peek is phase without changing anything. Under s.mu. func (s *Service) peek(set config.SlurmSettings, node string) Phase { st := s.status[node] if set.IsLent(node) { if len(s.inv.Allocations(node)) > 0 { return Lending } return Lent } switch { case st == nil: return Unknown case slurmBusy(st): return Reclaiming } return Llmgw } // Describe is one line for a log or a screen. func (n NodeStatus) Describe() string { switch n.Phase { case Lending: if !n.StopsBy.IsZero() { // A replica moves once its replacement serves; one with nowhere // else to go waits out the grace, and that wait is not a hang. return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s", n.Replicas, n.StopsBy.Local().Format(time.TimeOnly)) } return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas) case Lent: return "lent to Slurm" case Reclaiming: if n.Status != nil { return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs) } return "being taken back" case Llmgw: return "llmgw's" } return "not heard from yet" }