diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 4f060cb..3bb1ceb 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -375,7 +375,9 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur } unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" + "[Unit]\nConditionPathExists=\n" + - "[Service]\nExecStart=\nExecStart=" + bin + " -D -s --conf-server " + server + " -N " + name + "\n" + // --systemd as the packaged unit has it (Type=notify, since 22.05): + // without it systemd waits for a readiness that never comes. + "[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" changed, err := k.write(DropIn, []byte(unit), 0o644, "") if err != nil { return err @@ -437,16 +439,24 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur // localJobs counts the job steps running here, from their slurmstepd, not // from the controller: a job outlives a controller that is down, and is -// what decides whether llmgw may have the cards back. -1 when it cannot -// tell. +// what decides whether llmgw may have the cards back. The one slurmstepd +// that is no job's, "slurmstepd infinity" holding slurmd's cgroup v2 scope, +// is not counted. -1 when it cannot tell. func (k *Keeper) localJobs(ctx context.Context) int32 { - // pgrep -c prints 0 and exits 1 when there is none. - out, _ := k.host.Run(ctx, nil, "pgrep", "-c", "-x", "slurmstepd") - n, err := strconv.Atoi(strings.TrimSpace(string(out))) - if err != nil { + // pgrep exits 1 with no output when there is none. + out, err := k.host.Run(ctx, nil, "pgrep", "-a", "-x", "slurmstepd") + if err != nil && len(bytes.TrimSpace(out)) > 0 { return -1 } - return int32(n) //nolint:gosec // a count of processes + n := int32(0) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + f := strings.Fields(line) + if len(f) < 2 || f[len(f)-1] == "infinity" { + continue + } + n++ + } + return n } // clientEnv points scontrol at slurmd's copy of the files on a node that is diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index 36daae0..c1dc80c 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -70,10 +70,8 @@ func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...str } return nil, nil case strings.HasPrefix(c, "pgrep"): - if f.outputs["jobs"] == "" { - return []byte("0\n"), errors.New("exit status 1") - } - return []byte(f.outputs["jobs"]), nil + out := "1800358 /usr/sbin/slurmstepd infinity\n" + f.outputs["jobs"] + return []byte(out), nil } return []byte(f.outputs[c]), nil } @@ -148,7 +146,7 @@ func TestNodeInstallsAndStaysDrained(t *testing.T) { t.Fatal("the munge key was not written for munge") } unit := string(h.files[DropIn]) - if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd -D -s --conf-server 10.0.0.146:6817 -N cuda01\n") || + if !strings.Contains(unit, "ExecStart=/usr/sbin/slurmd --systemd --conf-server 10.0.0.146:6817 -N cuda01\n") || !strings.Contains(unit, "ConditionPathExists=\n") { t.Fatalf("drop-in:\n%s", unit) } @@ -179,7 +177,7 @@ func TestLendAndReclaim(t *testing.T) { if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" { t.Fatalf("a lent node was not resumed: %v %v", h.ran, st) } - h.outputs["jobs"] = "2\n" + h.outputs["jobs"] = "2001 slurmstepd: [101.batch]\n2002 slurmstepd: [102.0]\n" st = k.Reconcile(context.Background(), nodeMsg(false)) if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 { t.Fatalf("a node taken back was not drained: %v", h.ran)