A package install that is no engine's clears its progress; slurmd retried

Found on the PoC: InstallPackages reported 'installing munge, slurm-client,
slurmd' under no engine and never cleared it, so the node's page kept
saying so (infiniband-diags and perftest did the same). slurmd started
before the leader has written its node into slurm.conf is refused; the
drop-in now has systemd try it again every 15 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/agent/provision.go

@@ -164,6 +164,13 @@ func (a *Agent) provisionLoop(ctx context.Context) {
 
 // provisionStatus records a change in one engine's installation.
 func (a *Agent) provisionStatus(st provision.Status) {
+	if st == (provision.Status{}) {
+		// A package install that was no engine's has ended.
+		a.mu.Lock()
+		delete(a.provStatus, "")
+		a.mu.Unlock()
+		return
+	}
 	a.mu.Lock()
 	prev, had := a.provStatus[st.Engine]
 	a.provStatus[st.Engine] = st

internal/agent/provision/install.go

@@ -497,7 +497,13 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string,
 // byFamily is keyed by discovery.Host.Family: "deb" or "rpm".
 func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error {
 	var st Status
-	return in.packages(ctx, byFamily, &st)
+	err := in.packages(ctx, byFamily, &st)
+	if st.State == Installing {
+		// Its progress was no engine's: an empty status takes it off the
+		// node's page again, done or failed (the caller says which).
+		in.report(Status{})
+	}
+	return err
 }
 
 // lockWait bounds how long a package command waits for another to finish.

internal/agent/provision/packages_status_test.go

@@ -0,0 +1,36 @@
+package provision
+
+import (
+	"context"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+)
+
+// A package install that is no engine's says it is done, so its progress
+// does not stay on the node's page (found on the PoC installing Slurm).
+func TestInstallPackagesClearsItsProgress(t *testing.T) {
+	var got []Status
+	in := &Installer{Host: discovery.Host{OSID: "ubuntu"}, OnStatus: func(s Status) { got = append(got, s) },
+		Run: func(_ context.Context, _ []string, name string, _ ...string) ([]byte, error) {
+			if name == "sh" { // the query: not installed
+				return nil, context.Canceled
+			}
+			return nil, nil
+		}}
+	if err := in.InstallPackages(context.Background(), map[string][]string{"deb": {"munge"}}); err != nil {
+		t.Fatal(err)
+	}
+	if len(got) < 2 || got[0].State != Installing || got[len(got)-1] != (Status{}) {
+		t.Fatalf("statuses %+v", got)
+	}
+
+	got = nil
+	in.Run = func(context.Context, []string, string, ...string) ([]byte, error) { return nil, nil }
+	if err := in.InstallPackages(context.Background(), map[string][]string{"deb": {"munge"}}); err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 0 {
+		t.Fatalf("installed already, yet said %+v", got)
+	}
+}

internal/agent/slurmnode/slurmnode.go

@@ -374,10 +374,13 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur
 		return err
 	}
 	unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" +
-		"[Unit]\nConditionPathExists=\n" +
+		"[Unit]\nConditionPathExists=\nStartLimitIntervalSec=0\n" +
 		// --systemd as the packaged unit has it (Type=notify, since 22.05):
 		// without it systemd waits for a readiness that never comes.
-		"[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n"
+		"[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" +
+		// A node new to the controller is refused until the leader has
+		// written it into slurm.conf; tried again until it is.
+		"Restart=on-failure\nRestartSec=15\n"
 	changed, err := k.write(DropIn, []byte(unit), 0o644, "")
 	if err != nil {
 		return err