Slurm: the version from the daemon, not scontrol

Found on the PoC: scontrol looks for its files even for --version, and a
configless node has none before slurmd has fetched them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/agent/slurmnode/slurmnode.go

@@ -169,15 +169,25 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
 	if err := k.ensurePackages(ctx, want); err != nil {
 		return fail("installing Slurm", err)
 	}
-	out, err := k.host.Run(ctx, nil, "scontrol", "--version")
+	// From the daemon, not scontrol: a client command looks for its files
+	// even to say its version, and a configless node has none before slurmd
+	// has fetched them.
+	daemon := "slurmd"
+	if !want.GetNode() {
+		daemon = "slurmctld"
+	}
+	out, err := k.host.Run(ctx, nil, daemon, "-V")
 	if err != nil {
-		return fail("reading Slurm's version", err)
+		return fail("reading Slurm's version", fmt.Errorf("%v: %s", err, lastLine(out)))
 	}
 	v, err := slurm.ParseVersion(string(out))
 	if err != nil {
 		return fail("reading Slurm's version", err)
 	}
-	st.Version = strings.TrimPrefix(strings.TrimSpace(string(out)), "slurm ")
+	st.Version = strings.TrimSpace(string(out))
+	if i := strings.LastIndexByte(st.Version, ' '); i >= 0 {
+		st.Version = st.Version[i+1:]
+	}
 	if slurm.TooOld(v) {
 		return fail("Slurm", fmt.Errorf("%d.%02d is older than %d.%02d, the first to run jobs in cgroup v2; "+
 			"a newer release of the distribution, or Slurm's own packages, has one", v[0], v[1],

internal/agent/slurmnode/slurmnode_test.go

@@ -28,9 +28,10 @@ type fakeHost struct {
 func newFake() *fakeHost {
 	return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{},
 		active: map[string]bool{}, outputs: map[string]string{
-			"scontrol --version": "slurm 23.11.4\n",
-			"scontrol ping":      "Slurmctld(primary) at node06 is UP\n",
-			"slurmd -C":          "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
+			"slurmd -V":     "slurm-wlm 23.11.4\n",
+			"slurmctld -V":  "slurm-wlm 23.11.4\n",
+			"scontrol ping": "Slurmctld(primary) at node06 is UP\n",
+			"slurmd -C":     "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n",
 		}, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew}
 }
 
@@ -261,7 +262,7 @@ func TestController(t *testing.T) {
 
 func TestTooOld(t *testing.T) {
 	h := newFake()
-	h.outputs["scontrol --version"] = "slurm-wlm 21.08.5\n"
+	h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n"
 	k := New(h, &fakeInst{host: h}, "/m", nil)
 	st := k.Reconcile(context.Background(), nodeMsg(false))
 	if !strings.Contains(st.GetError(), "older than 22.05") {