llmgw/internal/agent/slurmnode/shared_test.go

package slurmnode

import (
	"context"
	"slices"
	"strings"
	"testing"

	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)

func ayse() *llmgwv1.SlurmUser {
	return &llmgwv1.SlurmUser{Login: "ayse", Uid: 200000, Name: "Ayşe Y.", Home: "/shared/home/ayse",
		SshKeys: []string{"ssh-ed25519 AAAA ayse@laptop"}}
}

func TestTheServerExportsAndMakesUsersWithHomes(t *testing.T) {
	h := newFake()
	h.active["ssh"] = true
	inst := &fakeInst{host: h}
	k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
	m := nodeMsg(false)
	m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Serve: true, Clients: []string{"10.0.0.22", "10.0.0.21"}}
	m.ManageUsers, m.MakeHomes, m.Users = true, true, []*llmgwv1.SlurmUser{ayse()}
	h.fail["test -d /shared/home/ayse"] = true
	h.files[slurm.PAMSSHD] = []byte(pamStack)
	st := k.Reconcile(context.Background(), m)
	if st.GetError() != "" {
		t.Fatal(st.GetError())
	}
	if st.GetShared() != SharedExported || st.GetUsers() != 1 {
		t.Fatalf("status %v", st)
	}
	if !slices.Contains(inst.got["deb"], "nfs-kernel-server") {
		t.Errorf("packages %v", inst.got)
	}
	if e := string(h.files[slurm.ExportsPath]); !strings.Contains(e, `"/shared" 10.0.0.21(`) {
		t.Errorf("exports:\n%s", e)
	}
	for _, want := range []string{"systemctl enable --now nfs-server", "exportfs -ra", "groupadd -g 200000 ayse",
		"useradd -u 200000 -g 200000 -M -d /shared/home/ayse -s /bin/bash -c Ayşe Y. ayse",
		"install -d -m 0700 -o 200000 -g 200000 /shared/home/ayse", "sshd -t", "systemctl reload ssh"} {
		if !slices.Contains(h.ran, want) {
			t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
		}
	}
	if string(h.files[slurm.AuthorizedKeysDir+"/ayse"]) != "ssh-ed25519 AAAA ayse@laptop\n" ||
		string(h.files[k.usersMarker()]) != "ayse\n" || h.files[slurm.SSHDropIn] == nil {
		t.Errorf("files: keys %q, marker %q", h.files[slurm.AuthorizedKeysDir+"/ayse"], h.files[k.usersMarker()])
	}

	// Again, made: nothing is run again.
	h.outputs["getent passwd ayse"] = "ayse:x:200000:200000:Ayşe Y.:/shared/home/ayse:/bin/bash\n"
	delete(h.fail, "test -d /shared/home/ayse")
	h.ran = nil
	k.Reconcile(context.Background(), m)
	for _, c := range h.ran {
		if strings.HasPrefix(c, "useradd") || strings.HasPrefix(c, "usermod") || strings.HasPrefix(c, "exportfs") ||
			strings.HasPrefix(c, "install") || c == "systemctl reload ssh" {
			t.Errorf("ran %q again", c)
		}
	}

	// Gone from the accounts: taken away, its home left.
	m.Users = nil
	k.Reconcile(context.Background(), m)
	if !slices.Contains(h.ran, "userdel ayse") || h.files[k.usersMarker()] != nil || h.files[slurm.SSHDropIn] != nil ||
		h.files[slurm.AuthorizedKeysDir+"/ayse"] != nil {
		t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n"))
	}
	if slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "rm") || strings.Contains(c, "userdel -r") }) {
		t.Error("a home was removed")
	}
}

func TestAClientMountsAndLeavesOthersUsersAlone(t *testing.T) {
	h := newFake()
	inst := &fakeInst{host: h}
	k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
	m := nodeMsg(false)
	m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"}
	bob := &llmgwv1.SlurmUser{Login: "bob", Uid: 200001, Home: "/shared/home/bob"}
	m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse(), bob}
	// The site's own ayse, with its own UID.
	h.outputs["getent passwd ayse"] = "ayse:x:1001:1001::/home/ayse:/bin/bash\n"
	st := k.Reconcile(context.Background(), m)
	if st.GetShared() != SharedMounted || st.GetUsers() != 1 {
		t.Fatalf("status %v", st)
	}
	if !strings.Contains(st.GetError(), "ayse is already a user here with UID 1001") {
		t.Errorf("error %q", st.GetError())
	}
	if u := string(h.files["/etc/systemd/system/shared.mount"]); !strings.Contains(u, "What=10.0.0.11:/shared") {
		t.Fatalf("unit:\n%s", u)
	}
	for _, want := range []string{"systemctl enable shared.mount", "systemctl restart shared.mount", "mountpoint -q /shared",
		"useradd -u 200001 -g 200001 -M -d /shared/home/bob -s /bin/bash -c  bob"} {
		if !slices.Contains(h.ran, want) {
			t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
		}
	}
	if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "ayse") && !strings.HasPrefix(c, "getent") }) ||
		slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "install -d") }) {
		t.Errorf("touched the site's ayse, or made a home:\n%s", strings.Join(h.ran, "\n"))
	}
	if string(h.files[k.usersMarker()]) != "bob\n" {
		t.Errorf("marker %q", h.files[k.usersMarker()])
	}

	// Not mounted: said.
	h.fail["mountpoint -q /shared"] = true
	if st := k.Reconcile(context.Background(), m); st.GetShared() != SharedMissing ||
		!strings.Contains(st.GetError(), "not mounted from 10.0.0.11:/shared") {
		t.Errorf("status %v", st)
	}

	// No shared path any more: the mount is taken away.
	h.outputs["systemctl list-units --all --plain --no-legend --type=mount"] = "shared.mount loaded active mounted llmgw\n"
	m.Shared = nil
	k.Reconcile(context.Background(), m)
	if !slices.Contains(h.ran, "systemctl disable --now shared.mount") || h.files["/etc/systemd/system/shared.mount"] != nil {
		t.Errorf("mount kept:\n%s", strings.Join(h.ran, "\n"))
	}
}

const pamStack = "@include common-auth\naccount required pam_nologin.so\n@include common-account\n@include common-session\n"

func TestUsersReachANodeOnlyThroughTheirJobs(t *testing.T) {
	h := newFake()
	inst := &fakeInst{host: h}
	k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
	h.files[slurm.PAMSSHD] = []byte(pamStack)
	// The module is not there until its package is installed.
	for _, d := range slurm.ModuleDirs {
		h.fail["test -f "+d+"/"+slurm.AdoptModule] = true
	}
	m := nodeMsg(true)
	m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse()}
	k.Reconcile(context.Background(), m)
	if !slices.Contains(inst.got["deb"], "libpam-slurm-adopt") {
		t.Fatalf("not installed: %v", inst.got)
	}
	if got := string(h.files[slurm.PAMSSHD]); got != pamStack {
		t.Fatalf("lines put in with the module still missing:\n%s", got)
	}
	delete(h.fail, "test -f "+slurm.ModuleDirs[0]+"/"+slurm.AdoptModule)
	st := k.Reconcile(context.Background(), m)
	got := string(h.files[slurm.PAMSSHD])
	if !strings.Contains(got, "pam_listfile.so item=user sense=allow file="+k.usersMarker()) ||
		!strings.Contains(got, "account required "+slurm.AdoptModule) {
		t.Fatalf("not held to their jobs (%s):\n%s", st.GetError(), got)
	}

	// The controller is where they log in: not held there.
	m.Controller = true
	m.Files = map[string]string{"slurm.conf": "x"}
	k.Reconcile(context.Background(), m)
	if string(h.files[slurm.PAMSSHD]) != pamStack {
		t.Errorf("held on the controller:\n%s", h.files[slurm.PAMSSHD])
	}

	// No users of llmgw's any more: the lines go.
	m.Controller, m.Files = false, nil
	k.Reconcile(context.Background(), m)
	m.Users = nil
	k.Reconcile(context.Background(), m)
	if string(h.files[slurm.PAMSSHD]) != pamStack {
		t.Errorf("lines left with no users:\n%s", h.files[slurm.PAMSSHD])
	}
}