Files
llmgw/internal/agent/slurmnode/slurmnode.go
T
Ilker ManapandClaude Opus 5.5 849760c67d Slurm: no second restart; backfill every 10 s, MailProg set
The InvalidAccount seen on the PoC was not the reconfigure: 23.11.4 without
accounting storage has its main scheduler mark a batch job so, and backfill
starts it on its next pass (job 4 waited 15 s). bf_interval=10 keeps that
short; the extra restart is taken out again. MailProg=/bin/true quiets the
warning on every start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 13:18:35 +01:00

603 lines
18 KiB
Go

// Package slurmnode keeps this machine's part in the Slurm cluster run beside
// llmgw (BACKLOG P), as the leader says: slurmctld on the controller, slurmd
// on a node, munge on both, installed from the machine's own package sources.
//
// What the leader sends is wanted state, kept by Keeper's loop: sent again,
// or checked again a minute later, it changes nothing that is already so.
package slurmnode
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"log/slog"
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"time"
"google.golang.org/protobuf/proto"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// Host is what Keeper does to the machine; Local in the agent, a fake in
// tests.
type Host interface {
Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error)
LookPath(name string) (string, error)
ReadFile(path string) ([]byte, error)
// WriteFile writes b at path with mode, owned by owner ("" for root),
// making its directory.
WriteFile(path string, b []byte, mode fs.FileMode, owner string) error
// MkdirAll makes path, owned by owner.
MkdirAll(path string, mode fs.FileMode, owner string) error
Remove(path string) error
}
// Installer installs packages by family, "deb" and "rpm".
type Installer interface {
InstallPackages(ctx context.Context, byFamily map[string][]string) error
}
// Reasons llmgw drains a node with; one drained for another reason, by a
// Slurm administrator, is never resumed by llmgw.
const (
ReasonServing = "llmgw: serving models"
reasonPrefix = "llmgw"
)
// DropIn is the unit drop-in that starts slurmd configless.
const DropIn = "/etc/systemd/system/slurmd.service.d/llmgw.conf"
// Keeper keeps the machine as the last Slurm message says.
type Keeper struct {
host Host
inst Installer
log *slog.Logger
marker string
mu sync.Mutex
want *llmgwv1.Slurm
kick chan struct{}
// resend has the next status sent even if unchanged: a new stream's
// control plane may never have had it.
resend bool
// installed is true once the packages were found or put there, so a
// minute's check does not look again.
installed map[string]bool
files string
}
// New is a Keeper; marker is a file it keeps while it runs Slurm here, so
// an agent started again knows to stop what it started when told it has no
// part any more.
func New(host Host, inst Installer, marker string, log *slog.Logger) *Keeper {
if log == nil {
log = slog.Default()
}
return &Keeper{host: host, inst: inst, log: log, marker: marker,
kick: make(chan struct{}, 1), installed: map[string]bool{}}
}
// Apply is a new message from the leader: kept from now on.
func (k *Keeper) Apply(m *llmgwv1.Slurm) {
k.mu.Lock()
k.want = proto.CloneOf(m)
k.mu.Unlock()
select {
case k.kick <- struct{}{}:
default:
}
}
// Resend has the status sent again on the next pass, which it starts now.
func (k *Keeper) Resend() {
k.mu.Lock()
k.resend = true
k.mu.Unlock()
select {
case k.kick <- struct{}{}:
default:
}
}
// Every is how often the machine is checked again without a new message.
var Every = time.Minute
// resendAfter is how often an unchanged status is sent anyway.
const resendAfter = 5 * time.Minute
// Run keeps the machine until ctx ends, sending its status when it changes
// and every few minutes while it has a part.
func (k *Keeper) Run(ctx context.Context, send func(*llmgwv1.SlurmStatus)) {
t := time.NewTicker(Every)
defer t.Stop()
var last *llmgwv1.SlurmStatus
var sent time.Time
for {
select {
case <-ctx.Done():
return
case <-k.kick:
case <-t.C:
}
k.mu.Lock()
want, again := k.want, k.resend
k.resend = false
k.mu.Unlock()
if want == nil {
continue
}
st := k.Reconcile(ctx, want)
if st == nil {
last = nil
continue
}
if again || !proto.Equal(st, last) || time.Since(sent) > resendAfter {
send(st)
last, sent = st, time.Now()
}
}
}
// Reconcile makes the machine what want says once, and answers its status;
// nil when it has no part in Slurm.
func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.SlurmStatus {
ctx, cancel := context.WithTimeout(ctx, 20*time.Minute)
defer cancel()
if !want.GetController() && !want.GetNode() {
k.stop(ctx)
return nil
}
st := &llmgwv1.SlurmStatus{}
fail := func(what string, err error) *llmgwv1.SlurmStatus {
st.Error = what + ": " + err.Error()
k.log.Warn("keeping Slurm on this node: "+what, "err", err)
return st
}
if err := k.ensurePackages(ctx, want); err != nil {
return fail("installing Slurm", err)
}
// From the daemon, not scontrol: a client command looks for its files
// even to say its version, and a configless node has none before slurmd
// has fetched them.
daemon := "slurmd"
if !want.GetNode() {
daemon = "slurmctld"
}
out, err := k.host.Run(ctx, nil, daemon, "-V")
if err != nil {
return fail("reading Slurm's version", fmt.Errorf("%v: %s", err, lastLine(out)))
}
v, err := slurm.ParseVersion(string(out))
if err != nil {
return fail("reading Slurm's version", err)
}
st.Version = strings.TrimSpace(string(out))
if i := strings.LastIndexByte(st.Version, ' '); i >= 0 {
st.Version = st.Version[i+1:]
}
if slurm.TooOld(v) {
return fail("Slurm", fmt.Errorf("%d.%02d is older than %d.%02d, the first to run jobs in cgroup v2; "+
"a newer release of the distribution, or Slurm's own packages, has one", v[0], v[1],
slurm.MinVersion[0], slurm.MinVersion[1]))
}
if err := k.host.WriteFile(k.marker, []byte("llmgw runs Slurm here (BACKLOG P)\n"), 0o644, ""); err != nil {
return fail("keeping its marker", err)
}
if err := k.munge(ctx, want.GetMungeKey()); err != nil {
return fail("munge", err)
}
if want.GetController() {
sum, err := k.controller(ctx, want.GetFiles())
st.FilesSha256 = sum
if err != nil {
return fail("slurmctld", err)
}
st.SlurmctldRunning = k.active(ctx, "slurmctld")
}
if want.GetNode() {
if err := k.node(ctx, want, st); err != nil {
return fail("slurmd", err)
}
}
return st
}
func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error {
need := map[string][2]string{
"munged": {"munge", "munge"},
"scontrol": {"slurm-client", "slurm"},
}
if want.GetController() {
need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"}
}
if want.GetNode() {
need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"}
}
pkgs := map[string][]string{}
for bin, p := range need {
if k.installed[bin] {
continue
}
if _, err := k.host.LookPath(bin); err == nil {
k.installed[bin] = true
continue
}
pkgs["deb"] = append(pkgs["deb"], p[0])
pkgs["rpm"] = append(pkgs["rpm"], p[1])
}
if len(pkgs) == 0 {
return nil
}
slices.Sort(pkgs["deb"])
slices.Sort(pkgs["rpm"])
if k.inst == nil {
return fmt.Errorf("%s missing, and this agent installs nothing", strings.Join(pkgs["deb"], ", "))
}
k.log.Info("installing Slurm, as the cluster's settings say", "packages", pkgs["deb"])
if err := k.inst.InstallPackages(ctx, pkgs); err != nil {
return err
}
for bin := range need {
if _, err := k.host.LookPath(bin); err != nil {
return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", "))
}
k.installed[bin] = true
}
return nil
}
// munge writes the cluster's key and keeps munged running on it.
func (k *Keeper) munge(ctx context.Context, key []byte) error {
if len(key) < 32 {
return errors.New("the leader sent no key")
}
changed, err := k.write(slurm.MungeKeyPath, key, 0o400, "munge")
if err != nil {
return err
}
if changed {
k.log.Info("munge key written; munge starts again on it")
return k.systemctl(ctx, "enable", "--now", "munge", "&&restart")
}
if !k.active(ctx, "munge") {
return k.systemctl(ctx, "enable", "--now", "munge")
}
return nil
}
// controller writes the files and keeps slurmctld running on them,
// answering their digest.
func (k *Keeper) controller(ctx context.Context, files map[string]string) (string, error) {
if files["slurm.conf"] == "" {
return "", errors.New("the leader sent no slurm.conf")
}
for _, d := range []string{slurm.StateDir, slurm.LogDir} {
if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil {
return "", err
}
}
names := make([]string, 0, len(files))
for n := range files {
names = append(names, n)
}
slices.Sort(names)
h := sha256.New()
changed := false
for _, n := range names {
if strings.ContainsAny(n, "/\\") || n == "" || n == "." || n == ".." {
return "", fmt.Errorf("the leader sent a file named %q", n)
}
fmt.Fprintf(h, "%s\x00%s\x00", n, files[n])
c, err := k.write(filepath.Join(slurm.ConfDir, n), []byte(files[n]), 0o644, "")
if err != nil {
return "", err
}
changed = changed || c
}
sum := hex.EncodeToString(h.Sum(nil))
if changed || sum != k.files {
if err := k.systemctl(ctx, "enable", "slurmctld"); err != nil {
return sum, err
}
// Nodes added or taken away need a restart before 23.11 and a
// reconfigure after it; a restart does both, and running jobs live
// through it.
k.log.Info("slurmctld starts again on new files")
if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil {
return sum, err
}
if err := k.waitPing(ctx); err != nil {
return sum, err
}
// Hands the configless slurmd the new files.
if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil {
return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out))
}
k.files = sum
return sum, nil
}
if !k.active(ctx, "slurmctld") {
return sum, k.systemctl(ctx, "enable", "--now", "slurmctld")
}
return sum, nil
}
func (k *Keeper) waitPing(ctx context.Context) error {
var last []byte
for range 30 {
out, err := k.host.Run(ctx, nil, "scontrol", "ping")
if err == nil && bytes.Contains(out, []byte("UP")) {
return nil
}
last = out
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(pingWait):
}
}
return fmt.Errorf("slurmctld did not answer after starting: %s", lastLine(last))
}
var pingWait = time.Second
// node keeps slurmd running configless and puts the node in or out of the
// controller's hands as lent says.
func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.SlurmStatus) error {
name, server := want.GetName(), want.GetConfServer()
if name == "" || server == "" {
return errors.New("the leader sent no name or controller")
}
if strings.ContainsAny(name+server, " \n\t\"'\\$`") {
return fmt.Errorf("the leader sent a name or controller with spaces or quotes in it")
}
for _, d := range []string{slurm.SpoolDir, slurm.LogDir} {
if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil {
return err
}
}
bin, err := k.host.LookPath("slurmd")
if err != nil {
return err
}
unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" +
"[Unit]\nConditionPathExists=\n" +
// --systemd as the packaged unit has it (Type=notify, since 22.05):
// without it systemd waits for a readiness that never comes.
"[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n"
changed, err := k.write(DropIn, []byte(unit), 0o644, "")
if err != nil {
return err
}
if out, err := k.host.Run(ctx, nil, "slurmd", "-C"); err == nil {
st.Hardware = strings.TrimSpace(string(out))
}
if changed {
if err := k.systemctl(ctx, "daemon-reload"); err != nil {
return err
}
if err := k.systemctl(ctx, "enable", "slurmd"); err != nil {
return err
}
if err := k.systemctl(ctx, "restart", "slurmd"); err != nil {
return err
}
} else if !k.active(ctx, "slurmd") {
if err := k.systemctl(ctx, "enable", "--now", "slurmd"); err != nil {
return err
}
}
st.SlurmdRunning = k.active(ctx, "slurmd")
st.Jobs = k.localJobs(ctx)
env := k.clientEnv()
out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name)
if err != nil {
// Not in slurm.conf yet: the leader writes it once this node's
// hardware is in.
return nil
}
ns, err := slurm.ParseNodeState(string(out))
if err != nil {
return nil
}
ours := ns.Reason == "" || strings.HasPrefix(ns.Reason, reasonPrefix)
switch {
case want.GetLent() && ns.Drained() && ours:
k.log.Info("this node is lent to Slurm: resumed there")
if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=RESUME"); err != nil {
return fmt.Errorf("resuming the node: %v: %s", err, lastLine(out))
}
case !want.GetLent() && !strings.Contains(ns.State, "DRAIN"):
k.log.Info("this node is llmgw's: drained in Slurm, its jobs left to finish")
if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=DRAIN",
"Reason="+ReasonServing); err != nil {
return fmt.Errorf("draining the node: %v: %s", err, lastLine(out))
}
}
if out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name); err == nil {
if ns2, err := slurm.ParseNodeState(string(out)); err == nil {
ns = ns2
}
}
st.State, st.Reason = ns.State, ns.Reason
return nil
}
// localJobs counts the job steps running here, from their slurmstepd, not
// from the controller: a job outlives a controller that is down, and is
// what decides whether llmgw may have the cards back. The one slurmstepd
// that is no job's, "slurmstepd infinity" holding slurmd's cgroup v2 scope,
// is not counted. -1 when it cannot tell.
func (k *Keeper) localJobs(ctx context.Context) int32 {
// pgrep exits 1 with no output when there is none.
out, err := k.host.Run(ctx, nil, "pgrep", "-a", "-x", "slurmstepd")
if err != nil && len(bytes.TrimSpace(out)) > 0 {
return -1
}
n := int32(0)
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
f := strings.Fields(line)
if len(f) < 2 || f[len(f)-1] == "infinity" {
continue
}
n++
}
return n
}
// clientEnv points scontrol at slurmd's copy of the files on a node that is
// not the controller, which has none of its own.
func (k *Keeper) clientEnv() []string {
if _, err := k.host.ReadFile(filepath.Join(slurm.ConfDir, "slurm.conf")); err == nil {
return nil
}
return []string{"SLURM_CONF=" + filepath.Join(slurm.SpoolDir, "conf-cache", "slurm.conf")}
}
// stop stops what llmgw started here, if it did.
func (k *Keeper) stop(ctx context.Context) {
if _, err := k.host.ReadFile(k.marker); err != nil {
return
}
k.log.Info("this node has no part in Slurm any more: slurmd and slurmctld stopped; the packages stay")
for _, u := range []string{"slurmd", "slurmctld"} {
_ = k.systemctl(ctx, "disable", "--now", u)
}
_ = k.host.Remove(DropIn)
_ = k.systemctl(ctx, "daemon-reload")
_ = k.host.Remove(k.marker)
k.files = ""
}
// write writes a file when it differs, saying whether it did.
func (k *Keeper) write(path string, b []byte, mode fs.FileMode, owner string) (bool, error) {
if old, err := k.host.ReadFile(path); err == nil && bytes.Equal(old, b) {
return false, nil
}
return true, k.host.WriteFile(path, b, mode, owner)
}
// systemctl runs systemctl; a last argument "&&restart" restarts the unit
// after.
func (k *Keeper) systemctl(ctx context.Context, args ...string) error {
restart := false
if n := len(args); n > 0 && args[n-1] == "&&restart" {
args, restart = args[:n-1], true
}
if out, err := k.host.Run(ctx, nil, "systemctl", args...); err != nil {
return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
}
if restart {
u := args[len(args)-1]
if out, err := k.host.Run(ctx, nil, "systemctl", "restart", u); err != nil {
return fmt.Errorf("systemctl restart %s: %v: %s", u, err, lastLine(out))
}
}
return nil
}
func (k *Keeper) active(ctx context.Context, unit string) bool {
out, err := k.host.Run(ctx, nil, "systemctl", "is-active", unit)
return err == nil && strings.TrimSpace(string(out)) == "active"
}
func lastLine(b []byte) string {
s := strings.TrimSpace(string(b))
if i := strings.LastIndexByte(s, '\n'); i >= 0 {
s = s[i+1:]
}
return s
}
// Local is the machine the agent runs on.
type Local struct{}
// Run runs a command, with env added to the agent's.
func (Local) Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
cmd := exec.CommandContext(ctx, name, args...)
if len(env) > 0 {
cmd.Env = append(os.Environ(), env...)
}
return cmd.CombinedOutput()
}
// LookPath finds a command, in the sbin directories too: an agent started
// by systemd has them, one started by hand may not.
func (Local) LookPath(name string) (string, error) {
if p, err := exec.LookPath(name); err == nil {
return p, nil
}
for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} {
p := filepath.Join(d, name)
if fi, err := os.Stat(p); err == nil && !fi.IsDir() {
return p, nil
}
}
return "", fmt.Errorf("%s is not installed", name)
}
// ReadFile reads a file.
func (Local) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } //nolint:gosec // our paths
// WriteFile writes a file whole, through a temporary one beside it.
func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories
return err
}
tmp := path + ".llmgw-new"
if err := os.WriteFile(tmp, b, mode); err != nil {
return err
}
if err := os.Chmod(tmp, mode); err != nil {
return err
}
if err := chown(tmp, owner); err != nil {
_ = os.Remove(tmp)
return err
}
return os.Rename(tmp, path)
}
// MkdirAll makes a directory owned by owner.
func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error {
if err := os.MkdirAll(path, mode); err != nil {
return err
}
return chown(path, owner)
}
// Remove removes a file, and is content when there is none.
func (Local) Remove(path string) error {
if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
return nil
}
func chown(path, owner string) error {
if owner == "" {
return nil
}
u, err := user.Lookup(owner)
if err != nil {
return fmt.Errorf("user %s: %w (installed with its package)", owner, err)
}
uid, _ := strconv.Atoi(u.Uid)
gid, _ := strconv.Atoi(u.Gid)
return os.Chown(path, uid, gid)
}