// Package slurmnode keeps this machine's part in the Slurm cluster run beside // llmgw (BACKLOG P), as the leader says: slurmctld on the controller, slurmd // on a node, munge on both, installed from the machine's own package sources. // // What the leader sends is wanted state, kept by Keeper's loop: sent again, // or checked again a minute later, it changes nothing that is already so. package slurmnode import ( "bytes" "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io/fs" "log/slog" "os" "os/exec" "os/user" "path/filepath" "slices" "strconv" "strings" "sync" "time" "google.golang.org/protobuf/proto" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) // Host is what Keeper does to the machine; Local in the agent, a fake in // tests. type Host interface { Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) LookPath(name string) (string, error) ReadFile(path string) ([]byte, error) // WriteFile writes b at path with mode, owned by owner ("" for root), // making its directory. WriteFile(path string, b []byte, mode fs.FileMode, owner string) error // MkdirAll makes path, owned by owner. MkdirAll(path string, mode fs.FileMode, owner string) error Remove(path string) error } // Installer installs packages by family, "deb" and "rpm". type Installer interface { InstallPackages(ctx context.Context, byFamily map[string][]string) error } // Reasons llmgw drains a node with; one drained for another reason, by a // Slurm administrator, is never resumed by llmgw. const ( ReasonServing = "llmgw: serving models" reasonPrefix = "llmgw" ) // DropIn is the unit drop-in that starts slurmd configless. const DropIn = "/etc/systemd/system/slurmd.service.d/llmgw.conf" // Keeper keeps the machine as the last Slurm message says. type Keeper struct { host Host inst Installer log *slog.Logger marker string mu sync.Mutex want *llmgwv1.Slurm kick chan struct{} // resend has the next status sent even if unchanged: a new stream's // control plane may never have had it. resend bool // installed is true once the packages were found or put there, so a // minute's check does not look again. installed map[string]bool files string } // New is a Keeper; marker is a file it keeps while it runs Slurm here, so // an agent started again knows to stop what it started when told it has no // part any more. func New(host Host, inst Installer, marker string, log *slog.Logger) *Keeper { if log == nil { log = slog.Default() } return &Keeper{host: host, inst: inst, log: log, marker: marker, kick: make(chan struct{}, 1), installed: map[string]bool{}} } // Apply is a new message from the leader: kept from now on. func (k *Keeper) Apply(m *llmgwv1.Slurm) { k.mu.Lock() k.want = proto.CloneOf(m) k.mu.Unlock() select { case k.kick <- struct{}{}: default: } } // Resend has the status sent again on the next pass, which it starts now. func (k *Keeper) Resend() { k.mu.Lock() k.resend = true k.mu.Unlock() select { case k.kick <- struct{}{}: default: } } // Every is how often the machine is checked again without a new message. var Every = time.Minute // resendAfter is how often an unchanged status is sent anyway. const resendAfter = 5 * time.Minute // Run keeps the machine until ctx ends, sending its status when it changes // and every few minutes while it has a part. func (k *Keeper) Run(ctx context.Context, send func(*llmgwv1.SlurmStatus)) { t := time.NewTicker(Every) defer t.Stop() var last *llmgwv1.SlurmStatus var sent time.Time for { select { case <-ctx.Done(): return case <-k.kick: case <-t.C: } k.mu.Lock() want, again := k.want, k.resend k.resend = false k.mu.Unlock() if want == nil { continue } st := k.Reconcile(ctx, want) if st == nil { last = nil continue } if again || !proto.Equal(st, last) || time.Since(sent) > resendAfter { send(st) last, sent = st, time.Now() } } } // Reconcile makes the machine what want says once, and answers its status; // nil when it has no part in Slurm. func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.SlurmStatus { ctx, cancel := context.WithTimeout(ctx, 20*time.Minute) defer cancel() if !want.GetController() && !want.GetNode() { k.stop(ctx) return nil } st := &llmgwv1.SlurmStatus{} fail := func(what string, err error) *llmgwv1.SlurmStatus { st.Error = what + ": " + err.Error() k.log.Warn("keeping Slurm on this node: "+what, "err", err) return st } if err := k.ensurePackages(ctx, want); err != nil { return fail("installing Slurm", err) } // From the daemon, not scontrol: a client command looks for its files // even to say its version, and a configless node has none before slurmd // has fetched them. daemon := "slurmd" if !want.GetNode() { daemon = "slurmctld" } out, err := k.host.Run(ctx, nil, daemon, "-V") if err != nil { return fail("reading Slurm's version", fmt.Errorf("%v: %s", err, lastLine(out))) } v, err := slurm.ParseVersion(string(out)) if err != nil { return fail("reading Slurm's version", err) } st.Version = strings.TrimSpace(string(out)) if i := strings.LastIndexByte(st.Version, ' '); i >= 0 { st.Version = st.Version[i+1:] } if slurm.TooOld(v) { return fail("Slurm", fmt.Errorf("%d.%02d is older than %d.%02d, the first to run jobs in cgroup v2; "+ "a newer release of the distribution, or Slurm's own packages, has one", v[0], v[1], slurm.MinVersion[0], slurm.MinVersion[1])) } if err := k.host.WriteFile(k.marker, []byte("llmgw runs Slurm here (BACKLOG P)\n"), 0o644, ""); err != nil { return fail("keeping its marker", err) } if err := k.munge(ctx, want.GetMungeKey()); err != nil { return fail("munge", err) } if want.GetController() { sum, err := k.controller(ctx, want.GetFiles()) st.FilesSha256 = sum if err != nil { return fail("slurmctld", err) } st.SlurmctldRunning = k.active(ctx, "slurmctld") } if want.GetNode() { if err := k.node(ctx, want, st); err != nil { return fail("slurmd", err) } } return st } func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error { need := map[string][2]string{ "munged": {"munge", "munge"}, "scontrol": {"slurm-client", "slurm"}, } if want.GetController() { need["slurmctld"] = [2]string{"slurmctld", "slurm-slurmctld"} } if want.GetNode() { need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"} } pkgs := map[string][]string{} for bin, p := range need { if k.installed[bin] { continue } if _, err := k.host.LookPath(bin); err == nil { k.installed[bin] = true continue } pkgs["deb"] = append(pkgs["deb"], p[0]) pkgs["rpm"] = append(pkgs["rpm"], p[1]) } if len(pkgs) == 0 { return nil } slices.Sort(pkgs["deb"]) slices.Sort(pkgs["rpm"]) if k.inst == nil { return fmt.Errorf("%s missing, and this agent installs nothing", strings.Join(pkgs["deb"], ", ")) } k.log.Info("installing Slurm, as the cluster's settings say", "packages", pkgs["deb"]) if err := k.inst.InstallPackages(ctx, pkgs); err != nil { return err } for bin := range need { if _, err := k.host.LookPath(bin); err != nil { return fmt.Errorf("%s is still missing after installing %s", bin, strings.Join(pkgs["deb"], ", ")) } k.installed[bin] = true } return nil } // munge writes the cluster's key and keeps munged running on it. func (k *Keeper) munge(ctx context.Context, key []byte) error { if len(key) < 32 { return errors.New("the leader sent no key") } changed, err := k.write(slurm.MungeKeyPath, key, 0o400, "munge") if err != nil { return err } if changed { k.log.Info("munge key written; munge starts again on it") return k.systemctl(ctx, "enable", "--now", "munge", "&&restart") } if !k.active(ctx, "munge") { return k.systemctl(ctx, "enable", "--now", "munge") } return nil } // controller writes the files and keeps slurmctld running on them, // answering their digest. func (k *Keeper) controller(ctx context.Context, files map[string]string) (string, error) { if files["slurm.conf"] == "" { return "", errors.New("the leader sent no slurm.conf") } for _, d := range []string{slurm.StateDir, slurm.LogDir} { if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil { return "", err } } names := make([]string, 0, len(files)) for n := range files { names = append(names, n) } slices.Sort(names) h := sha256.New() changed := false for _, n := range names { if strings.ContainsAny(n, "/\\") || n == "" || n == "." || n == ".." { return "", fmt.Errorf("the leader sent a file named %q", n) } fmt.Fprintf(h, "%s\x00%s\x00", n, files[n]) c, err := k.write(filepath.Join(slurm.ConfDir, n), []byte(files[n]), 0o644, "") if err != nil { return "", err } changed = changed || c } sum := hex.EncodeToString(h.Sum(nil)) if changed || sum != k.files { if err := k.systemctl(ctx, "enable", "slurmctld"); err != nil { return sum, err } // Nodes added or taken away need a restart before 23.11 and a // reconfigure after it; a restart does both, and running jobs live // through it. k.log.Info("slurmctld starts again on new files") if err := k.systemctl(ctx, "restart", "slurmctld"); err != nil { return sum, err } if err := k.waitPing(ctx); err != nil { return sum, err } // Hands the configless slurmd the new files. if out, err := k.host.Run(ctx, nil, "scontrol", "reconfigure"); err != nil { return sum, fmt.Errorf("scontrol reconfigure: %v: %s", err, lastLine(out)) } k.files = sum return sum, nil } if !k.active(ctx, "slurmctld") { return sum, k.systemctl(ctx, "enable", "--now", "slurmctld") } return sum, nil } func (k *Keeper) waitPing(ctx context.Context) error { var last []byte for range 30 { out, err := k.host.Run(ctx, nil, "scontrol", "ping") if err == nil && bytes.Contains(out, []byte("UP")) { return nil } last = out select { case <-ctx.Done(): return ctx.Err() case <-time.After(pingWait): } } return fmt.Errorf("slurmctld did not answer after starting: %s", lastLine(last)) } var pingWait = time.Second // node keeps slurmd running configless and puts the node in or out of the // controller's hands as lent says. func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.SlurmStatus) error { name, server := want.GetName(), want.GetConfServer() if name == "" || server == "" { return errors.New("the leader sent no name or controller") } if strings.ContainsAny(name+server, " \n\t\"'\\$`") { return fmt.Errorf("the leader sent a name or controller with spaces or quotes in it") } for _, d := range []string{slurm.SpoolDir, slurm.LogDir} { if err := k.host.MkdirAll(d, 0o755, "slurm"); err != nil { return err } } bin, err := k.host.LookPath("slurmd") if err != nil { return err } unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" + "[Unit]\nConditionPathExists=\n" + // --systemd as the packaged unit has it (Type=notify, since 22.05): // without it systemd waits for a readiness that never comes. "[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" changed, err := k.write(DropIn, []byte(unit), 0o644, "") if err != nil { return err } if out, err := k.host.Run(ctx, nil, "slurmd", "-C"); err == nil { st.Hardware = strings.TrimSpace(string(out)) } if changed { if err := k.systemctl(ctx, "daemon-reload"); err != nil { return err } if err := k.systemctl(ctx, "enable", "slurmd"); err != nil { return err } if err := k.systemctl(ctx, "restart", "slurmd"); err != nil { return err } } else if !k.active(ctx, "slurmd") { if err := k.systemctl(ctx, "enable", "--now", "slurmd"); err != nil { return err } } st.SlurmdRunning = k.active(ctx, "slurmd") st.Jobs = k.localJobs(ctx) env := k.clientEnv() out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name) if err != nil { // Not in slurm.conf yet: the leader writes it once this node's // hardware is in. return nil } ns, err := slurm.ParseNodeState(string(out)) if err != nil { return nil } ours := ns.Reason == "" || strings.HasPrefix(ns.Reason, reasonPrefix) switch { case want.GetLent() && ns.Drained() && ours: k.log.Info("this node is lent to Slurm: resumed there") if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=RESUME"); err != nil { return fmt.Errorf("resuming the node: %v: %s", err, lastLine(out)) } case !want.GetLent() && !strings.Contains(ns.State, "DRAIN"): k.log.Info("this node is llmgw's: drained in Slurm, its jobs left to finish") if out, err := k.host.Run(ctx, env, "scontrol", "update", "NodeName="+name, "State=DRAIN", "Reason="+ReasonServing); err != nil { return fmt.Errorf("draining the node: %v: %s", err, lastLine(out)) } } if out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name); err == nil { if ns2, err := slurm.ParseNodeState(string(out)); err == nil { ns = ns2 } } st.State, st.Reason = ns.State, ns.Reason return nil } // localJobs counts the job steps running here, from their slurmstepd, not // from the controller: a job outlives a controller that is down, and is // what decides whether llmgw may have the cards back. The one slurmstepd // that is no job's, "slurmstepd infinity" holding slurmd's cgroup v2 scope, // is not counted. -1 when it cannot tell. func (k *Keeper) localJobs(ctx context.Context) int32 { // pgrep exits 1 with no output when there is none. out, err := k.host.Run(ctx, nil, "pgrep", "-a", "-x", "slurmstepd") if err != nil && len(bytes.TrimSpace(out)) > 0 { return -1 } n := int32(0) for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { f := strings.Fields(line) if len(f) < 2 || f[len(f)-1] == "infinity" { continue } n++ } return n } // clientEnv points scontrol at slurmd's copy of the files on a node that is // not the controller, which has none of its own. func (k *Keeper) clientEnv() []string { if _, err := k.host.ReadFile(filepath.Join(slurm.ConfDir, "slurm.conf")); err == nil { return nil } return []string{"SLURM_CONF=" + filepath.Join(slurm.SpoolDir, "conf-cache", "slurm.conf")} } // stop stops what llmgw started here, if it did. func (k *Keeper) stop(ctx context.Context) { if _, err := k.host.ReadFile(k.marker); err != nil { return } k.log.Info("this node has no part in Slurm any more: slurmd and slurmctld stopped; the packages stay") for _, u := range []string{"slurmd", "slurmctld"} { _ = k.systemctl(ctx, "disable", "--now", u) } _ = k.host.Remove(DropIn) _ = k.systemctl(ctx, "daemon-reload") _ = k.host.Remove(k.marker) k.files = "" } // write writes a file when it differs, saying whether it did. func (k *Keeper) write(path string, b []byte, mode fs.FileMode, owner string) (bool, error) { if old, err := k.host.ReadFile(path); err == nil && bytes.Equal(old, b) { return false, nil } return true, k.host.WriteFile(path, b, mode, owner) } // systemctl runs systemctl; a last argument "&&restart" restarts the unit // after. func (k *Keeper) systemctl(ctx context.Context, args ...string) error { restart := false if n := len(args); n > 0 && args[n-1] == "&&restart" { args, restart = args[:n-1], true } if out, err := k.host.Run(ctx, nil, "systemctl", args...); err != nil { return fmt.Errorf("systemctl %s: %v: %s", strings.Join(args, " "), err, lastLine(out)) } if restart { u := args[len(args)-1] if out, err := k.host.Run(ctx, nil, "systemctl", "restart", u); err != nil { return fmt.Errorf("systemctl restart %s: %v: %s", u, err, lastLine(out)) } } return nil } func (k *Keeper) active(ctx context.Context, unit string) bool { out, err := k.host.Run(ctx, nil, "systemctl", "is-active", unit) return err == nil && strings.TrimSpace(string(out)) == "active" } func lastLine(b []byte) string { s := strings.TrimSpace(string(b)) if i := strings.LastIndexByte(s, '\n'); i >= 0 { s = s[i+1:] } return s } // Local is the machine the agent runs on. type Local struct{} // Run runs a command, with env added to the agent's. func (Local) Run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) { cmd := exec.CommandContext(ctx, name, args...) if len(env) > 0 { cmd.Env = append(os.Environ(), env...) } return cmd.CombinedOutput() } // LookPath finds a command, in the sbin directories too: an agent started // by systemd has them, one started by hand may not. func (Local) LookPath(name string) (string, error) { if p, err := exec.LookPath(name); err == nil { return p, nil } for _, d := range []string{"/usr/sbin", "/sbin", "/usr/local/sbin"} { p := filepath.Join(d, name) if fi, err := os.Stat(p); err == nil && !fi.IsDir() { return p, nil } } return "", fmt.Errorf("%s is not installed", name) } // ReadFile reads a file. func (Local) ReadFile(path string) ([]byte, error) { return os.ReadFile(path) } //nolint:gosec // our paths // WriteFile writes a file whole, through a temporary one beside it. func (Local) WriteFile(path string, b []byte, mode fs.FileMode, owner string) error { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { //nolint:gosec // system directories return err } tmp := path + ".llmgw-new" if err := os.WriteFile(tmp, b, mode); err != nil { return err } if err := os.Chmod(tmp, mode); err != nil { return err } if err := chown(tmp, owner); err != nil { _ = os.Remove(tmp) return err } return os.Rename(tmp, path) } // MkdirAll makes a directory owned by owner. func (Local) MkdirAll(path string, mode fs.FileMode, owner string) error { if err := os.MkdirAll(path, mode); err != nil { return err } return chown(path, owner) } // Remove removes a file, and is content when there is none. func (Local) Remove(path string) error { if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { return err } return nil } func chown(path, owner string) error { if owner == "" { return nil } u, err := user.Lookup(owner) if err != nil { return fmt.Errorf("user %s: %w (installed with its package)", owner, err) } uid, _ := strconv.Atoi(u.Uid) gid, _ := strconv.Atoi(u.Gid) return os.Chown(path, uid, gid) }