diff --git a/internal/agent/provision.go b/internal/agent/provision.go index d49557f..d58856a 100644 --- a/internal/agent/provision.go +++ b/internal/agent/provision.go @@ -164,6 +164,13 @@ func (a *Agent) provisionLoop(ctx context.Context) { // provisionStatus records a change in one engine's installation. func (a *Agent) provisionStatus(st provision.Status) { + if st == (provision.Status{}) { + // A package install that was no engine's has ended. + a.mu.Lock() + delete(a.provStatus, "") + a.mu.Unlock() + return + } a.mu.Lock() prev, had := a.provStatus[st.Engine] a.provStatus[st.Engine] = st diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index d34a155..27311ba 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -497,7 +497,13 @@ func (in *Installer) packages(ctx context.Context, byFamily map[string][]string, // byFamily is keyed by discovery.Host.Family: "deb" or "rpm". func (in *Installer) InstallPackages(ctx context.Context, byFamily map[string][]string) error { var st Status - return in.packages(ctx, byFamily, &st) + err := in.packages(ctx, byFamily, &st) + if st.State == Installing { + // Its progress was no engine's: an empty status takes it off the + // node's page again, done or failed (the caller says which). + in.report(Status{}) + } + return err } // lockWait bounds how long a package command waits for another to finish. diff --git a/internal/agent/provision/packages_status_test.go b/internal/agent/provision/packages_status_test.go new file mode 100644 index 0000000..edd6fdc --- /dev/null +++ b/internal/agent/provision/packages_status_test.go @@ -0,0 +1,36 @@ +package provision + +import ( + "context" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" +) + +// A package install that is no engine's says it is done, so its progress +// does not stay on the node's page (found on the PoC installing Slurm). +func TestInstallPackagesClearsItsProgress(t *testing.T) { + var got []Status + in := &Installer{Host: discovery.Host{OSID: "ubuntu"}, OnStatus: func(s Status) { got = append(got, s) }, + Run: func(_ context.Context, _ []string, name string, _ ...string) ([]byte, error) { + if name == "sh" { // the query: not installed + return nil, context.Canceled + } + return nil, nil + }} + if err := in.InstallPackages(context.Background(), map[string][]string{"deb": {"munge"}}); err != nil { + t.Fatal(err) + } + if len(got) < 2 || got[0].State != Installing || got[len(got)-1] != (Status{}) { + t.Fatalf("statuses %+v", got) + } + + got = nil + in.Run = func(context.Context, []string, string, ...string) ([]byte, error) { return nil, nil } + if err := in.InstallPackages(context.Background(), map[string][]string{"deb": {"munge"}}); err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("installed already, yet said %+v", got) + } +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 3bb1ceb..7b11a29 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -374,10 +374,13 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur return err } unit := "# Written by llmgw (BACKLOG P): slurmd fetches its files from slurmctld.\n" + - "[Unit]\nConditionPathExists=\n" + + "[Unit]\nConditionPathExists=\nStartLimitIntervalSec=0\n" + // --systemd as the packaged unit has it (Type=notify, since 22.05): // without it systemd waits for a readiness that never comes. - "[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" + "[Service]\nExecStart=\nExecStart=" + bin + " --systemd --conf-server " + server + " -N " + name + "\n" + + // A node new to the controller is refused until the leader has + // written it into slurm.conf; tried again until it is. + "Restart=on-failure\nRestartSec=15\n" changed, err := k.write(DropIn, []byte(unit), 0o644, "") if err != nil { return err