llmgw/internal/control/slurmsvc/service.go

// Package slurmsvc is the leader's side of the Slurm cluster run beside
// llmgw (BACKLOG P): it tells each node's agent what it is, writes the
// controller's files from what the nodes report, and hands nodes between
// llmgw and Slurm as the cluster's settings say.
//
// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does,
// and lent to Slurm once none is left. A node taken out of slurm.lent is
// drained in Slurm, and is llmgw's again when its last job has ended. Both
// are kept in force every tick, from the settings and what the agents say,
// so a leader that takes over carries on where the last one was.
package slurmsvc

import (
	"context"
	"fmt"
	"log/slog"
	"maps"
	"net"
	"slices"
	"strconv"
	"strings"
	"sync"
	"time"

	"google.golang.org/protobuf/proto"

	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)

// Agents is how the service reaches the nodes.
type Agents interface {
	Send(node string, msg *llmgwv1.ControlMessage) error
	ConnectedNodes() []string
}

// Inventory is what the service reads and changes of llmgw's nodes.
type Inventory interface {
	Get(id string) (inventory.Node, bool)
	SetState(id string, st inventory.NodeState) bool
	Allocations(nodeID string) []inventory.Allocation
	Nodes() []inventory.Node
}

// Phase is where a node of slurm.nodes is between llmgw and Slurm.
type Phase string

const (
	// Llmgw: drained in Slurm with no job left, llmgw's to place on.
	Llmgw Phase = "llmgw"
	// Lending: in slurm.lent, its replicas being moved off.
	Lending Phase = "lending"
	// Lent: Slurm's; llmgw places nothing there.
	Lent Phase = "lent"
	// Reclaiming: out of slurm.lent, drained in Slurm, jobs still running.
	Reclaiming Phase = "reclaiming"
	// Unknown: its agent has not said how it is in Slurm yet.
	Unknown Phase = "unknown"
)

// NodeStatus is one node as the service sees it.
type NodeStatus struct {
	Node     string `json:"node"`
	Phase    Phase  `json:"phase"`
	Replicas int    `json:"replicas"`
	// StopsBy is, while it is being lent, when a replica still on it is
	// stopped even with nowhere else to go: the drain's grace after the
	// lend began. Zero when not known (a leader that took over meanwhile).
	StopsBy   time.Time `json:"stops_by,omitempty"`
	Status    *Reported `json:"slurm,omitempty"`
	Connected bool      `json:"connected"`
}

// Reported is the agent's SlurmStatus, for the admin API.
type Reported struct {
	Version          string `json:"version,omitempty"`
	Hardware         string `json:"hardware,omitempty"`
	SlurmdRunning    bool   `json:"slurmd_running"`
	SlurmctldRunning bool   `json:"slurmctld_running"`
	SlurmdbdRunning  bool   `json:"slurmdbd_running"`
	// Shared is the shared path here: "mounted", "exported", "missing".
	Shared string `json:"shared,omitempty"`
	// Containers is what the machine has for jobs in containers, empty
	// while not both of pyxis and enroot.
	Containers string `json:"containers,omitempty"`
	// Users is how many of llmgw's users the machine has.
	Users       int       `json:"users"`
	State       string    `json:"state,omitempty"`
	Reason      string    `json:"reason,omitempty"`
	Jobs        int       `json:"jobs"`
	Error       string    `json:"error,omitempty"`
	FilesSHA256 string    `json:"files_sha256,omitempty"`
	At          time.Time `json:"at"`
}

// Status is the whole Slurm cluster, for `ctl slurm status`.
type Status struct {
	Enabled    bool         `json:"enabled"`
	Cluster    string       `json:"cluster"`
	Controller string       `json:"controller"`
	Partition  string       `json:"partition"`
	Ctld       *Reported    `json:"controller_status,omitempty"`
	Nodes      []NodeStatus `json:"nodes"`
	// Waiting are nodes left out of slurm.conf until they report their
	// hardware.
	Waiting []string `json:"waiting,omitempty"`
	// Shared is where the shared files are and come from; Users who keeps
	// the users, and how many llmgw has.
	Shared    config.SharedStorage `json:"shared"`
	Users     string               `json:"users,omitempty"`
	UserCount int                  `json:"user_count,omitempty"`
	// Containers is whether jobs may run in containers: "on" once every
	// machine has pyxis and enroot, "installing" until then, empty when
	// slurm.containers is off.
	Containers string `json:"containers,omitempty"`
}

// User is a Linux user made from an llmgw account.
type User struct {
	Login   string   `json:"login"`
	UID     int      `json:"uid"`
	Name    string   `json:"name"`
	Email   string   `json:"email"`
	SSHKeys []string `json:"ssh_keys,omitempty"`
}

// Service keeps the Slurm cluster.
type Service struct {
	settings func() config.SlurmSettings
	inv      Inventory
	agents   Agents
	key      func(ctx context.Context) ([]byte, error)
	log      *slog.Logger

	mu sync.Mutex
	// status is each node's last SlurmStatus.
	status map[string]*Reported
	// drained are nodes this service put in Draining, which it alone makes
	// ready again.
	drained map[string]bool
	// lendingAt is when this leader began moving a node's replicas off.
	lendingAt map[string]time.Time
	// sent is what each node was last told, so it is told again only when
	// that changes or its stream is new.
	sent    map[string]*llmgwv1.Slurm
	mungeKy []byte
	kick    chan struct{}
	// mem keeps slurm.conf's nodes and the controller's address for the
	// next leader; known is what it held when read, and what was kept
	// since. loaded is false until it was read: until then the controller
	// is sent no files that leave out a node, which would drop that node
	// and its jobs.
	mem    Memory
	known  Kept
	loaded bool
	// started is the first pass, from which a node neither heard from nor
	// kept is waited for, for HoldFor.
	started time.Time
	// userList reads the users from the accounts; users is what it last
	// said, read at usersAt. Until it has been read once no node is told
	// anything of users: an empty list would take every user away.
	userList func(ctx context.Context) ([]User, error)
	users    []User
	usersAt  time.Time
	usersOK  bool
}

// UsersEvery is how often the users are read again without being asked.
var UsersEvery = 30 * time.Second

// SetUsers has the users read with list, for slurm.users "llmgw".
func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) {
	s.mu.Lock()
	s.userList = list
	s.mu.Unlock()
}

// UsersChanged has the users read again now: an account or its keys
// changed.
func (s *Service) UsersChanged() {
	s.mu.Lock()
	s.usersAt = time.Time{}
	s.mu.Unlock()
	s.Kick()
}

// readUsers reads the users when they are due. Under s.mu.
func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
	if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery {
		return
	}
	us, err := s.userList(ctx)
	s.usersAt = time.Now()
	if err != nil {
		s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err)
		return
	}
	s.users, s.usersOK = us, true
}

// HoldFor is how long a new leader waits for a node it neither has heard
// from nor finds kept before it writes the controller's files without it:
// on a cluster that kept nothing yet (one upgraded to this), the nodes
// report within seconds of a new leader, and the controller keeps the
// node and its jobs meanwhile.
var HoldFor = 2 * time.Minute

// New is a Service; key gives the munge key, made once from a cluster
// secret.
func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
	key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service {
	if log == nil {
		log = slog.Default()
	}
	return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
		status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{},
		kick: make(chan struct{}, 1)}
}

// SetMemory has slurm.conf's nodes kept in mem, and read from it first.
func (s *Service) SetMemory(mem Memory) {
	s.mu.Lock()
	s.mem = mem
	s.mu.Unlock()
}

// current is the settings, slurm.all_nodes resolved to the nodes the
// inventory has.
func (s *Service) current() config.SlurmSettings {
	set := s.settings()
	if !set.AllNodes {
		return set
	}
	var ids []string
	for _, n := range s.inv.Nodes() {
		ids = append(ids, n.ID)
	}
	return set.WithNodes(ids)
}

// Every is how often the service looks again without being asked.
var Every = 10 * time.Second

// Run keeps the cluster until ctx ends.
func (s *Service) Run(ctx context.Context) {
	t := time.NewTicker(Every)
	defer t.Stop()
	for {
		s.Tick(ctx)
		select {
		case <-ctx.Done():
			return
		case <-t.C:
		case <-s.kick:
		}
	}
}

// Kick has the service look again now: the settings changed.
func (s *Service) Kick() {
	select {
	case s.kick <- struct{}{}:
	default:
	}
}

// OnSlurmStatus records what a node's agent says.
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
	r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
		Shared: st.GetShared(), Users: int(st.GetUsers()), Containers: st.GetContainers(),
		SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
		Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
	s.mu.Lock()
	prev := s.status[node]
	s.status[node] = r
	s.mu.Unlock()
	if r.Error != "" && (prev == nil || prev.Error != r.Error) {
		s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
	}
	if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs {
		s.Kick()
	}
}

// OnRegister is a node's new stream: it is told again what it is.
func (s *Service) OnRegister(node string) {
	s.mu.Lock()
	delete(s.sent, node)
	s.mu.Unlock()
	s.Kick()
}

// Tick does one pass: the nodes handed over as far as they can be, and
// each told what it is.
func (s *Service) Tick(ctx context.Context) {
	set := s.current()
	s.mu.Lock()
	defer s.mu.Unlock()
	s.recall(ctx)
	s.readUsers(ctx, set)
	if s.started.IsZero() {
		s.started = time.Now()
	}
	phases := s.handOver(set)
	var key []byte
	if set.Enabled {
		if s.mungeKy == nil {
			k, err := s.key(ctx)
			if err != nil {
				s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err)
				return
			}
			s.mungeKy = k
		}
		key = s.mungeKy
	}
	files, waiting := s.files(set)
	// Not yet knowing what the last leader knew, a slurm.conf without some
	// node would take it, and its jobs, from the controller: kept back.
	hold := s.mem != nil && len(waiting) > 0 && (!s.loaded || time.Since(s.started) < HoldFor)
	for _, node := range s.agents.ConnectedNodes() {
		if hold && node == set.Controller {
			continue
		}
		m := s.message(set, node, phases[node] == Lent, key, files)
		if m == nil {
			continue
		}
		if old, ok := s.sent[node]; ok && proto.Equal(old, m) {
			continue
		}
		if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil {
			s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err)
			continue
		}
		s.sent[node] = m
	}
	s.remember(ctx, set)
}

// recall reads what the last leader kept, once. Under s.mu.
func (s *Service) recall(ctx context.Context) {
	if s.mem == nil || s.loaded {
		return
	}
	k, err := s.mem.Recall(ctx)
	if err != nil {
		s.log.Warn("could not read Slurm's nodes from the store; the controller's files wait for every node",
			"err", err)
		return
	}
	s.known, s.loaded = k, true
}

// remember keeps slurm.conf's nodes and the controller's address when they
// changed. Under s.mu.
func (s *Service) remember(ctx context.Context, set config.SlurmSettings) {
	if s.mem == nil || !s.loaded || !set.Enabled {
		return
	}
	c, _ := s.cluster(set)
	now := Kept{Controller: set.Controller, ControllerAddr: c.ControllerAddr, Nodes: map[string]slurm.Node{}}
	for _, n := range c.Nodes {
		now.Nodes[n.Name] = n
	}
	if now.equal(s.known) {
		return
	}
	if err := s.mem.Keep(ctx, now); err != nil {
		s.log.Warn("could not keep Slurm's nodes in the store", "err", err)
		return
	}
	s.known = now
}

// controllerAddr is where the nodes reach slurmctld: the controller's
// management address, else the one kept. Under s.mu.
func (s *Service) controllerAddr(set config.SlurmSettings) string {
	if n, ok := s.inv.Get(set.Controller); ok && n.MgmtAddr != "" {
		return n.MgmtAddr
	}
	if s.known.Controller == set.Controller {
		return s.known.ControllerAddr
	}
	return ""
}

// handOver moves each node of slurm.nodes a step between llmgw and Slurm,
// answering where each is. Under s.mu.
func (s *Service) handOver(set config.SlurmSettings) map[string]Phase {
	out := map[string]Phase{}
	if !set.Enabled {
		// Off: every node is llmgw's again. What Slurm still runs is left
		// to finish where it is; turning Slurm off with work lent is the
		// operator's choice.
		for node := range s.drained {
			s.ready(node)
		}
		return out
	}
	for _, node := range set.Nodes {
		out[node] = s.phase(set, node)
	}
	// A node no longer in slurm.nodes is llmgw's.
	for node := range s.drained {
		if !slices.Contains(set.Nodes, node) {
			s.ready(node)
		}
	}
	return out
}

func (s *Service) phase(set config.SlurmSettings, node string) Phase {
	n, ok := s.inv.Get(node)
	st := s.status[node]
	if set.IsLent(node) {
		if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
			s.inv.SetState(node, inventory.Draining)
			s.drained[node] = true
			s.lendingAt[node] = time.Now()
			s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node,
				"stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly))
		}
		if !ok || len(s.inv.Allocations(node)) > 0 {
			return Lending
		}
		delete(s.lendingAt, node)
		return Lent
	}
	// Not lent. Until the agent says Slurm has drained it with no job left,
	// llmgw places nothing new on it: a new leader starts with every node
	// ready. Cordoned, not drained: llmgw has nothing there to move.
	if st == nil || slurmBusy(st) {
		if ok && n.State == inventory.Ready {
			s.inv.SetState(node, inventory.Cordoned)
			s.drained[node] = true
			if st != nil {
				s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end",
					"node", node, "jobs", st.Jobs)
			}
		}
		if st == nil {
			return Unknown
		}
		return Reclaiming
	}
	delete(s.lendingAt, node)
	if s.drained[node] {
		s.ready(node)
	}
	return Llmgw
}

// drainGrace is how long a drained replica waits for its replacement
// before it is stopped anyway: the reconciler's, which is not a setting.
var drainGrace = reconciler.DefaultConfig().DrainGrace

// slurmBusy is whether Slurm may still run work on a node: a job step left
// on it (or no telling), or a controller that would still give it one. A
// node the controller does not know of, or cannot be asked about, gets no
// job from it, so a Slurm that is broken never keeps a node from llmgw.
func slurmBusy(st *Reported) bool {
	if st.Jobs != 0 {
		return true
	}
	if st.State == "" {
		return false
	}
	return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN")
}

func (s *Service) ready(node string) {
	if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) {
		s.inv.SetState(node, inventory.Ready)
		s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node)
	}
	delete(s.drained, node)
}

// files are the controller's files, and the nodes left out of them for not
// having reported their hardware. Under s.mu.
func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) {
	c, waiting := s.cluster(set)
	if c.ControllerAddr == "" {
		return nil, waiting
	}
	f := c.Files()
	if s.containersReady(set) {
		f["plugstack.conf"] = slurm.PlugstackConf
	}
	return f, waiting
}

// containersReady is whether jobs may ask for containers: every machine of
// the Slurm cluster says it has pyxis and enroot. Slurm fails a job on a
// machine without the plugin it was told to load. Under s.mu.
func (s *Service) containersReady(set config.SlurmSettings) bool {
	if !set.Containers {
		return false
	}
	for _, n := range s.machines(set) {
		if st := s.status[n]; st == nil || st.Containers == "" {
			return false
		}
	}
	return true
}

// cluster is what slurm.conf says: each node as it last reported, else as
// kept from before, and the nodes left out for having neither. Under s.mu.
func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
	c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, Partition: set.PartitionName()}
	if !set.Enabled {
		return c, nil
	}
	c.ControllerAddr = s.controllerAddr(set)
	var waiting []string
	for _, node := range set.Nodes {
		sn, ok := s.reported(node)
		if !ok {
			sn, ok = s.known.Nodes[node]
		}
		if !ok {
			waiting = append(waiting, node)
			continue
		}
		sn.Lent = set.IsLent(node)
		c.Nodes = append(c.Nodes, sn)
	}
	return c, waiting
}

// reported is a node as its agent and the inventory say now. Under s.mu.
func (s *Service) reported(node string) (slurm.Node, bool) {
	n, ok := s.inv.Get(node)
	st := s.status[node]
	if !ok || n.MgmtAddr == "" || st == nil || st.Hardware == "" {
		return slurm.Node{}, false
	}
	hw, err := slurm.ParseHardware(st.Hardware)
	if err != nil {
		return slurm.Node{}, false
	}
	sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw}
	for _, g := range n.GPUs {
		sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name})
	}
	return sn, true
}

// message is what a node is told. Under s.mu.
func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte,
	files map[string]string) *llmgwv1.Slurm {
	m := &llmgwv1.Slurm{}
	if !set.Enabled {
		return m
	}
	isNode := slices.Contains(set.Nodes, node)
	isCtl := set.Controller == node && files != nil
	if !isNode && !isCtl {
		return m
	}
	m.Name, m.MungeKey, m.Lent = node, key, lent
	m.Containers = set.Containers
	m.Node = isNode
	if addr := s.controllerAddr(set); addr != "" {
		m.ConfServer = net.JoinHostPort(addr, strconv.Itoa(slurm.CtldPort))
	}
	if isNode && m.ConfServer == "" {
		// Without its controller a node cannot start; it is told when the
		// controller registers. Told nothing until then, not "no part": a
		// node still running Slurm's jobs would stop them.
		return nil
	}
	if isCtl {
		m.Controller = true
		m.Files = maps.Clone(files)
	}
	if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
		return nil
	}
	return m
}

// sharedPart puts the shared files in m, false when where they come from
// is not known yet: told nothing rather than no shared files, which would
// unmount them under running jobs. Under s.mu.
func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
	sh := set.Shared
	if sh.Path == "" {
		return true
	}
	m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS}
	switch {
	case sh.Server == "":
	case sh.Server == node:
		m.Shared.Serve = true
		for _, n := range s.machines(set) {
			if n == node {
				continue
			}
			if a := s.addr(set, n); a != "" {
				m.Shared.Clients = append(m.Shared.Clients, a)
			}
		}
		slices.Sort(m.Shared.Clients)
	default:
		a := s.addr(set, sh.Server)
		if a == "" {
			return false
		}
		if strings.Contains(a, ":") {
			a = "[" + a + "]"
		}
		m.Shared.Nfs = a + ":" + sh.Path
	}
	return true
}

// usersPart puts the users in m, false while they have not been read.
// Under s.mu.
func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
	if set.Users != config.UsersLlmgw {
		return true
	}
	if !s.usersOK {
		return false
	}
	m.ManageUsers = true
	switch {
	case set.Shared.Path == "":
		m.MakeHomes = true // each machine's own /home
	case set.Shared.Server != "":
		m.MakeHomes = set.Shared.Server == node
	default:
		m.MakeHomes = set.Controller == node
	}
	for _, u := range s.users {
		m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up
			Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)})
	}
	return true
}

// machines are the controller and the nodes.
func (s *Service) machines(set config.SlurmSettings) []string {
	out := slices.Clone(set.Nodes)
	if !slices.Contains(out, set.Controller) {
		out = append(out, set.Controller)
	}
	return out
}

// addr is a machine's management address, from the inventory, else kept
// from before. Under s.mu.
func (s *Service) addr(set config.SlurmSettings, node string) string {
	if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" {
		return n.MgmtAddr
	}
	if node == set.Controller {
		return s.controllerAddr(set)
	}
	return s.known.Nodes[node].Addr
}

// Status is the cluster as the service sees it now.
func (s *Service) Status() Status {
	set := s.current()
	s.mu.Lock()
	defer s.mu.Unlock()
	out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
		Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
	if set.Users == config.UsersLlmgw {
		out.UserCount = len(s.users)
	}
	if !set.Enabled {
		return out
	}
	if set.Containers {
		out.Containers = "installing"
		if s.containersReady(set) {
			out.Containers = "on"
		}
	}
	_, out.Waiting = s.files(set)
	conn := s.agents.ConnectedNodes()
	for _, node := range set.Nodes {
		ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
			Connected: slices.Contains(conn, node)}
		ns.Phase = s.peek(set, node)
		if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending {
			ns.StopsBy = at.Add(drainGrace)
		}
		out.Nodes = append(out.Nodes, ns)
	}
	return out
}

// peek is phase without changing anything. Under s.mu.
func (s *Service) peek(set config.SlurmSettings, node string) Phase {
	st := s.status[node]
	if set.IsLent(node) {
		if len(s.inv.Allocations(node)) > 0 {
			return Lending
		}
		return Lent
	}
	switch {
	case st == nil:
		return Unknown
	case slurmBusy(st):
		return Reclaiming
	}
	return Llmgw
}

// Describe is one line for a log or a screen.
func (n NodeStatus) Describe() string {
	switch n.Phase {
	case Lending:
		if !n.StopsBy.IsZero() {
			// A replica moves once its replacement serves; one with nowhere
			// else to go waits out the grace, and that wait is not a hang.
			return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s",
				n.Replicas, n.StopsBy.Local().Format(time.TimeOnly))
		}
		return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
	case Lent:
		return "lent to Slurm"
	case Reclaiming:
		if n.Status != nil {
			return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs)
		}
		return "being taken back"
	case Llmgw:
		return "llmgw's"
	}
	return "not heard from yet"
}