Slurm: slurmd started as its unit expects, and slurmd's own stepd not a job

Found on the PoC (Ubuntu 24.04, Slurm 23.11.4): the packaged unit is
Type=notify, so slurmd needs --systemd or systemd times it out; and slurmd
keeps a 'slurmstepd infinity' for its cgroup v2 scope, which was counted as
a job and kept the node from llmgw.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/agent/slurmnode/slurmnode.go

@@ -375,7 +375,9 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur
 	}
 	unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" +
 		"[Unit]\nConditionPathExists=\n" +
-		"[Service]\nExecStart=\nExecStart=" + bin + " -D -s --conf-server " + server + " -N " + name + "\n"
+		// --systemd as the packaged unit has it (Type=notify, since 22.05):
+		// without it systemd waits for a readiness that never comes.
+		"[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n"
 	changed, err := k.write(DropIn, []byte(unit), 0o644, "")
 	if err != nil {
 		return err
@@ -437,16 +439,24 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur
 
 // localJobs counts the job steps running here, from their slurmstepd, not
 // from the controller: a job outlives a controller that is down, and is
-// what decides whether llmgw may have the cards back. -1 when it cannot
-// tell.
+// what decides whether llmgw may have the cards back. The one slurmstepd
+// that is no job's, "slurmstepd infinity" holding slurmd's cgroup v2 scope,
+// is not counted. -1 when it cannot tell.
 func (k *Keeper) localJobs(ctx context.Context) int32 {
-	// pgrep -c prints 0 and exits 1 when there is none.
-	out, _ := k.host.Run(ctx, nil, "pgrep", "-c", "-x", "slurmstepd")
-	n, err := strconv.Atoi(strings.TrimSpace(string(out)))
-	if err != nil {
+	// pgrep exits 1 with no output when there is none.
+	out, err := k.host.Run(ctx, nil, "pgrep", "-a", "-x", "slurmstepd")
+	if err != nil && len(bytes.TrimSpace(out)) > 0 {
 		return -1
 	}
-	return int32(n) //nolint:gosec // a count of processes
+	n := int32(0)
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		f := strings.Fields(line)
+		if len(f) < 2 || f[len(f)-1] == "infinity" {
+			continue
+		}
+		n++
+	}
+	return n
 }
 
 // clientEnv points scontrol at slurmd's copy of the files on a node that is

internal/agent/slurmnode/slurmnode_test.go

@@ -70,10 +70,8 @@ func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...str
 		}
 		return nil, nil
 	case strings.HasPrefix(c, "pgrep"):
-		if f.outputs["jobs"] == "" {
-			return []byte("0\n"), errors.New("exit status 1")
-		}
-		return []byte(f.outputs["jobs"]), nil
+		out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"]
+		return []byte(out), nil
 	}
 	return []byte(f.outputs[c]), nil
 }
@@ -148,7 +146,7 @@ func TestNodeInstallsAndStaysDrained(t *testing.T) {
 		t.Fatal("the munge key was not written for munge")
 	}
 	unit := string(h.files[DropIn])
-	if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd -D -s --conf-server 10.0.0.146:6817 -N cuda01\n") ||
+	if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") ||
 		!strings.Contains(unit, "ConditionPathExists=\n") {
 		t.Fatalf("drop-in:\n%s", unit)
 	}
@@ -179,7 +177,7 @@ func TestLendAndReclaim(t *testing.T) {
 	if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" {
 		t.Fatalf("a lent node was not resumed: %v %v", h.ran, st)
 	}
-	h.outputs["jobs"] = "2\n"
+	h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n"
 	st = k.Reconcile(context.Background(), nodeMsg(false))
 	if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 {
 		t.Fatalf("a node taken back was not drained: %v", h.ran)