diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 975a262..4f060cb 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -169,15 +169,25 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl if err := k.ensurePackages(ctx, want); err != nil { return fail("installing Slurm", err) } - out, err := k.host.Run(ctx, nil, "scontrol", "--version") + // From the daemon, not scontrol: a client command looks for its files + // even to say its version, and a configless node has none before slurmd + // has fetched them. + daemon := "slurmd" + if !want.GetNode() { + daemon = "slurmctld" + } + out, err := k.host.Run(ctx, nil, daemon, "-V") if err != nil { - return fail("reading Slurm's version", err) + return fail("reading Slurm's version", fmt.Errorf("%v: %s", err, lastLine(out))) } v, err := slurm.ParseVersion(string(out)) if err != nil { return fail("reading Slurm's version", err) } - st.Version = strings.TrimPrefix(strings.TrimSpace(string(out)), "slurm ") + st.Version = strings.TrimSpace(string(out)) + if i := strings.LastIndexByte(st.Version, ' '); i >= 0 { + st.Version = st.Version[i+1:] + } if slurm.TooOld(v) { return fail("Slurm", fmt.Errorf("%d.%02d is older than %d.%02d, the first to run jobs in cgroup v2; "+ "a newer release of the distribution, or Slurm's own packages, has one", v[0], v[1], diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index fb62b97..36daae0 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -28,9 +28,10 @@ type fakeHost struct { func newFake() *fakeHost { return &fakeHost{files: map[string][]byte{}, owners: map[string]string{}, bins: map[string]bool{}, active: map[string]bool{}, outputs: map[string]string{ - "scontrol --version": "slurm 23.11.4\n", - "scontrol ping": "Slurmctld(primary) at node06 is UP\n", - "slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n", + "slurmd -V": "slurm-wlm 23.11.4\n", + "slurmctld -V": "slurm-wlm 23.11.4\n", + "scontrol ping": "Slurmctld(primary) at node06 is UP\n", + "slurmd -C": "NodeName=cuda01 CPUs=16 RealMemory=64000\nUpTime=1-00:00:00\n", }, fail: map[string]bool{}, state: "IDLE+DRAIN", reason: slurm.ReasonNew} } @@ -261,7 +262,7 @@ func TestController(t *testing.T) { func TestTooOld(t *testing.T) { h := newFake() - h.outputs["scontrol --version"] = "slurm-wlm 21.08.5\n" + h.outputs["slurmd -V"] = "slurm-wlm 21.08.5\n" k := New(h, &fakeInst{host: h}, "/m", nil) st := k.Reconcile(context.Background(), nodeMsg(false)) if !strings.Contains(st.GetError(), "older than 22.05") {