Slurm users reach a node only through a job of theirs

llmgw's Slurm users log in to the controller freely and to another Slurm
machine only while a job of theirs runs there: the agent installs
pam_slurm_adopt and puts it in sshd's account stack for the users it made
alone, so root and the machine's own users are never locked out. Their
session is put in the job's extern step (PrologFlags=Contain), and
pam_systemd, which would move it out, is skipped for them by taking the
session file into sshd's stack with a guard in front of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

BACKLOG.md

@@ -2071,6 +2071,16 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
   from the portal's Profile or `ctl slurm users keys`. A user already on a machine with another UID
   is left alone and said. The site has AD (not yet live): users from it, through sssd, are the next
   step (`slurm.users` empty today leaves them to the site).
+- Logging in to a node, done 2026-11-14: llmgw's users reach the controller freely, another Slurm
+  machine only while a job of theirs runs there (pam_slurm_adopt, installed by the agent; PrologFlags=
+  Contain). Only the users listed in the agent's users file are held: root and the machine's own users
+  log in as before, so a wrong line never locks out who would fix it. pam_systemd would move the session
+  out of the job, so sshd's stack takes the session file in place of its include, with pam_systemd
+  skipped for the listed users alone; where that cannot be done safely (nested includes, jumps across
+  it) they are held to having a job, not to its cores and cards, and the agent says so. Tried on the
+  VMs: ayse without a job refused ("no active jobs on this node"), with one in
+  job_N/step_extern (nproc 1), cut off when it ended; mehmet refused; admin's session unchanged.
+  Left: sites whose users come from AD (sssd) are not held yet; a group of theirs would be the list.
 - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm:
   each project an account (`sacctmgr add account`, parent its team and organization), each member a user
   of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads

CHANGELOG.md

@@ -60,7 +60,11 @@ APIs change between minors without a deprecation period.
   llmgw` (`ctl slurm users on`) makes each active account a Linux user on
   every Slurm machine, the same login and UID (from 200000, never reused)
   everywhere, its home on the shared path; people add SSH keys in the
-  portal's Profile (`ctl slurm users keys` for an operator).
+  portal's Profile (`ctl slurm users keys` for an operator). They log in to
+  the controller; to another Slurm machine only while a job of theirs runs
+  there, their session put in the job (pam_slurm_adopt), so nobody uses a
+  node's cards outside Slurm, nor on a node llmgw serves from. Root and the
+  machines' own users are not held to it.
 
 ### Changed
 

docs/install.md

@@ -210,7 +210,10 @@ active llmgw account is a Linux user on every Slurm machine
 200000 up that is never given to anyone else, its home under
 `/shared/home`. People add their SSH public keys in the portal's
 **Profile** and log in to this machine to run `sbatch`; an account suspended
-or deleted loses its user, its files stay. A site whose users come from its
+or deleted loses its user, its files stay. On the other Slurm machines they
+are let in only while a job of theirs runs there, and their ssh session is
+put in that job, held to its cores and cards (pam_slurm_adopt, which the
+agents install); root and the machines' own users log in as before. A site whose users come from its
 directory (sssd against Active Directory or LDAP) passes `--slurm-users
 site`, or later `llmgw ctl slurm users off`.
 

docs/reference.md

@@ -3595,8 +3595,9 @@ Takes a [SharedStorage](#sharedstorage) block.
 Users is who keeps the Linux users that run jobs, the same on every
 machine of the Slurm cluster: "llmgw" makes one for each active llmgw
 account (its login from the e-mail address, a UID of llmgw's from
-200000 up, its SSH keys from the portal); empty leaves users to the
-site (a directory through sssd, or by hand).
+200000 up, its SSH keys from the portal), let in to the controller,
+and to a node only while a job of theirs runs there (pam_slurm_adopt);
+empty leaves users to the site (a directory through sssd, or by hand).
 
 ## StrataConfig
 

internal/agent/slurmnode/shared_test.go

@@ -24,6 +24,7 @@ func TestTheServerExportsAndMakesUsersWithHomes(t *testing.T) {
 	m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Serve: true, Clients: []string{"10.0.0.22", "10.0.0.21"}}
 	m.ManageUsers, m.MakeHomes, m.Users = true, true, []*llmgwv1.SlurmUser{ayse()}
 	h.fail["test -d /shared/home/ayse"] = true
+	h.files[slurm.PAMSSHD] = []byte(pamStack)
 	st := k.Reconcile(context.Background(), m)
 	if st.GetError() != "" {
 		t.Fatal(st.GetError())
@@ -122,3 +123,49 @@ func TestAClientMountsAndLeavesOthersUsersAlone(t *testing.T) {
 		t.Errorf("mount kept:\n%s", strings.Join(h.ran, "\n"))
 	}
 }
+
+const pamStack = "@include common-auth\naccount required pam_nologin.so\n@include common-account\n@include common-session\n"
+
+func TestUsersReachANodeOnlyThroughTheirJobs(t *testing.T) {
+	h := newFake()
+	inst := &fakeInst{host: h}
+	k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
+	h.files[slurm.PAMSSHD] = []byte(pamStack)
+	// The module is not there until its package is installed.
+	for _, d := range slurm.ModuleDirs {
+		h.fail["test -f "+d+"/"+slurm.AdoptModule] = true
+	}
+	m := nodeMsg(true)
+	m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse()}
+	k.Reconcile(context.Background(), m)
+	if !slices.Contains(inst.got["deb"], "libpam-slurm-adopt") {
+		t.Fatalf("not installed: %v", inst.got)
+	}
+	if got := string(h.files[slurm.PAMSSHD]); got != pamStack {
+		t.Fatalf("lines put in with the module still missing:\n%s", got)
+	}
+	delete(h.fail, "test -f "+slurm.ModuleDirs[0]+"/"+slurm.AdoptModule)
+	st := k.Reconcile(context.Background(), m)
+	got := string(h.files[slurm.PAMSSHD])
+	if !strings.Contains(got, "pam_listfile.so item=user sense=allow file="+k.usersMarker()) ||
+		!strings.Contains(got, "account required "+slurm.AdoptModule) {
+		t.Fatalf("not held to their jobs (%s):\n%s", st.GetError(), got)
+	}
+
+	// The controller is where they log in: not held there.
+	m.Controller = true
+	m.Files = map[string]string{"slurm.conf": "x"}
+	k.Reconcile(context.Background(), m)
+	if string(h.files[slurm.PAMSSHD]) != pamStack {
+		t.Errorf("held on the controller:\n%s", h.files[slurm.PAMSSHD])
+	}
+
+	// No users of llmgw's any more: the lines go.
+	m.Controller, m.Files = false, nil
+	k.Reconcile(context.Background(), m)
+	m.Users = nil
+	k.Reconcile(context.Background(), m)
+	if string(h.files[slurm.PAMSSHD]) != pamStack {
+		t.Errorf("lines left with no users:\n%s", h.files[slurm.PAMSSHD])
+	}
+}

internal/agent/slurmnode/users.go

@@ -52,9 +52,87 @@ func (k *Keeper) users(ctx context.Context, want *llmgwv1.Slurm) (int32, error)
 	if err := k.sshConf(ctx, len(keep) > 0); err != nil {
 		errs = append(errs, err)
 	}
+	// The controller is where they log in and submit from; a node only
+	// while a job of theirs runs there.
+	if err := k.nodeLogin(ctx, len(keep) > 0 && want.GetNode() && !want.GetController()); err != nil {
+		errs = append(errs, err)
+	}
 	return n, errors.Join(errs...)
 }
 
+// nodeLogin holds the users llmgw made here to their jobs when they log in
+// (pam_slurm_adopt), or no longer. Only they are: root and the site's
+// administrators log in as before, so a mistake here never locks out the
+// people who would put it right.
+func (k *Keeper) nodeLogin(ctx context.Context, on bool) error {
+	old, err := k.host.ReadFile(slurm.PAMSSHD)
+	if err != nil {
+		if !on {
+			return nil
+		}
+		return fmt.Errorf("reading %s: %w", slurm.PAMSSHD, err)
+	}
+	if on {
+		if err := k.adoptModule(ctx); err != nil {
+			return err
+		}
+	}
+	read := func(name string) (string, error) {
+		b, err := k.host.ReadFile(filepath.Join(filepath.Dir(slurm.PAMSSHD), name))
+		return string(b), err
+	}
+	stack, contained, err := slurm.PAMAdopt(string(old), k.usersMarker(), read, on)
+	if err != nil {
+		return err
+	}
+	if stack == string(old) {
+		return nil
+	}
+	if on {
+		k.log.Info("llmgw's Slurm users log in to this node only while a job of theirs runs here",
+			"in_the_job", contained)
+		if !contained {
+			k.log.Warn("pam_systemd could not be skipped for them safely here: their ssh sessions are not held " +
+				"to their job's cores and cards, only to having one")
+		}
+	} else {
+		k.log.Info("llmgw's Slurm users are no longer held to their jobs when they log in here")
+	}
+	// Read by sshd for each login: nothing to reload.
+	_, err = k.write(slurm.PAMSSHD, []byte(stack), 0o644, "")
+	return err
+}
+
+// adoptModule is there, or is installed.
+func (k *Keeper) adoptModule(ctx context.Context) error {
+	if k.installed[slurm.AdoptModule] {
+		return nil
+	}
+	find := func() bool {
+		for _, d := range slurm.ModuleDirs {
+			if _, err := k.host.Run(ctx, nil, "test", "-f", filepath.Join(d, slurm.AdoptModule)); err == nil {
+				return true
+			}
+		}
+		return false
+	}
+	if !find() {
+		if k.inst == nil {
+			return fmt.Errorf("%s missing, and this agent installs nothing", slurm.AdoptModule)
+		}
+		k.log.Info("installing pam_slurm_adopt, which holds llmgw's Slurm users to their jobs here")
+		if err := k.inst.InstallPackages(ctx, slurm.AdoptPackages); err != nil {
+			return err
+		}
+		if !find() {
+			return fmt.Errorf("%s is still missing after installing %s", slurm.AdoptModule,
+				strings.Join(slurm.AdoptPackages["deb"], ", "))
+		}
+	}
+	k.installed[slurm.AdoptModule] = true
+	return nil
+}
+
 // errNotMade is a user llmgw could not make here, nor took for its own.
 var errNotMade = errors.New("not made")
 
@@ -162,6 +240,9 @@ func (k *Keeper) removeUsers(ctx context.Context, made, keep []string) error {
 		if err := k.sshConf(ctx, false); err != nil {
 			errs = append(errs, err)
 		}
+		if err := k.nodeLogin(ctx, false); err != nil {
+			errs = append(errs, err)
+		}
 	}
 	return errors.Join(errs...)
 }

internal/config/slurm.go

@@ -50,8 +50,9 @@ type SlurmSettings struct {
 	// Users is who keeps the Linux users that run jobs, the same on every
 	// machine of the Slurm cluster: "llmgw" makes one for each active llmgw
 	// account (its login from the e-mail address, a UID of llmgw's from
-	// 200000 up, its SSH keys from the portal); empty leaves users to the
-	// site (a directory through sssd, or by hand).
+	// 200000 up, its SSH keys from the portal), let in to the controller,
+	// and to a node only while a job of theirs runs there (pam_slurm_adopt);
+	// empty leaves users to the site (a directory through sssd, or by hand).
 	Users string `koanf:"users"`
 }
 

internal/docgen/reference_gen.go

@@ -527,7 +527,7 @@ var Reference = []Block{
 		{Key: "lent", Type: "[]string", Doc: "Lent are the nodes of Nodes Slurm may run jobs on now. A node put here\nis drained of llmgw's replicas first, as `ctl drain` does, and resumed\nin Slurm once none is left; a node taken out is drained in Slurm and\ngiven back to llmgw when its last job ends.", Nested: ""},
 		{Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""},
 		{Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"},
-		{Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal); empty leaves users to the\nsite (a directory through sssd, or by hand).", Nested: ""},
+		{Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal), let in to the controller,\nand to a node only while a job of theirs runs there (pam_slurm_adopt);\nempty leaves users to the site (a directory through sssd, or by hand).", Nested: ""},
 	}},
 	{Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{
 		{Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""},

internal/slurm/pam.go

@@ -0,0 +1,229 @@
+package slurm
+
+import (
+	"errors"
+	"fmt"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Logging in to a node: llmgw's users reach the controller, the machine
+// they submit from, whenever they like, and a node only while a job of
+// theirs runs there, their session put in that job (pam_slurm_adopt).
+// Without it a user could ssh to a node and use its cards outside Slurm,
+// on a node llmgw serves models from too.
+const (
+	// PAMSSHD is sshd's PAM stack, where the lines go.
+	PAMSSHD = "/etc/pam.d/sshd"
+	// AdoptModule is the module's file name.
+	AdoptModule = "pam_slurm_adopt.so"
+
+	pamBegin   = "# llmgw (BACKLOG P): llmgw's Slurm users reach this node only while a job of theirs runs here."
+	pamSession = "# llmgw (BACKLOG P): pam_systemd skipped for llmgw's Slurm users, so their session stays in the job pam_slurm_adopt put it in."
+	pamWas     = "# llmgw, in place of: "
+	pamEnd     = "# llmgw end"
+)
+
+// AdoptPackages are the packages with AdoptModule, by family.
+var AdoptPackages = map[string][]string{"deb": {"libpam-slurm-adopt"}, "rpm": {"slurm-pam_slurm"}}
+
+// ModuleDirs are where PAM's modules are, on Debian's and Red Hat's.
+var ModuleDirs = []string{
+	"/usr/lib/x86_64-linux-gnu/security", "/lib/x86_64-linux-gnu/security",
+	"/usr/lib/aarch64-linux-gnu/security", "/lib/aarch64-linux-gnu/security",
+	"/usr/lib64/security", "/lib64/security", "/usr/lib/security", "/lib/security",
+}
+
+// PAMAdopt is sshd's PAM stack with llmgw's lines in it (on) or taken out
+// and the stack as it was. Only the users listed in the file list, one a
+// line, are held to their jobs; anyone else, root and the site's
+// administrators, logs in as before.
+//
+// The account lines go after the last account line, so a user the
+// machine's own rules refuse is still refused. pam_systemd, which would
+// move the session out of the job into a scope of its own, is skipped for
+// the listed users: the file sshd includes it from (read through read) is
+// put in sshd's stack in its place, with the one line in front of
+// pam_systemd. contained is whether that was done; when it cannot be done
+// safely, the users are still held to their jobs, their sessions not to
+// the job's cards and cores.
+func PAMAdopt(stack, list string, read func(name string) (string, error), on bool) (out string, contained bool, err error) {
+	lines, err := pamStrip(stack)
+	if err != nil {
+		return "", false, err
+	}
+	if !on {
+		return join(lines), false, nil
+	}
+	last := -1
+	for i, l := range lines {
+		f := strings.Fields(l)
+		if len(f) > 0 && (f[0] == "account" || (f[0] == "@include" && len(f) > 1 && strings.Contains(f[1], "account"))) {
+			last = i
+		}
+	}
+	if last < 0 {
+		return "", false, errors.New(PAMSSHD + " has no account lines to put llmgw's after")
+	}
+	block := []string{
+		pamBegin,
+		// Listed: on to the next line; not listed, or no list: past it.
+		"account [success=ignore default=1] pam_listfile.so item=user sense=allow file=" + list + " onerr=fail",
+		"account required " + AdoptModule,
+		pamEnd,
+	}
+	lines = append(lines[:last+1:last+1], append(block, lines[last+1:]...)...)
+	lines, contained = pamSkipSystemd(lines, list, read)
+	return join(lines), contained, nil
+}
+
+func join(lines []string) string { return strings.Join(lines, "\n") + "\n" }
+
+// pamStrip takes llmgw's blocks out, putting back the lines they stand in
+// place of.
+func pamStrip(stack string) ([]string, error) {
+	var lines []string
+	in := false
+	for _, l := range strings.Split(strings.TrimSuffix(stack, "\n"), "\n") {
+		switch {
+		case !in && (l == pamBegin || l == pamSession):
+			in = true
+		case in && l == pamEnd:
+			in = false
+		case in && strings.HasPrefix(l, pamWas):
+			lines = append(lines, strings.TrimPrefix(l, pamWas))
+		case !in:
+			lines = append(lines, l)
+		}
+	}
+	if in {
+		return nil, errors.New("llmgw's lines in " + PAMSSHD + " have no end; put them right by hand")
+	}
+	return lines, nil
+}
+
+// pamLine is a PAM line's type, control and module, the control with its
+// brackets.
+var pamLine = regexp.MustCompile(`^\s*(-?[a-z]+)\s+(\[[^\]]*\]|\S+)\s+(\S+)`)
+
+// pamJump is a jump of N in a bracketed control.
+var pamJump = regexp.MustCompile(`=(\d+)`)
+
+func isSession(l string) bool {
+	m := pamLine.FindStringSubmatch(l)
+	return m != nil && strings.TrimPrefix(m[1], "-") == "session"
+}
+
+// pamSkipSystemd puts the guard in front of pam_systemd, in the session
+// file sshd includes it from or in sshd's own lines; false, and lines as
+// they were, when it is in neither or cannot be done without changing
+// what any other jump in the stack lands on.
+func pamSkipSystemd(lines []string, list string, read func(string) (string, error)) ([]string, bool) {
+	guard := "session [success=1 default=ignore] pam_listfile.so item=user sense=allow file=" + list + " onerr=fail"
+	hasSystemd := func(l string) bool {
+		m := pamLine.FindStringSubmatch(l)
+		return m != nil && strings.TrimPrefix(m[1], "-") == "session" && path.Base(m[3]) == "pam_systemd.so"
+	}
+	for i, l := range lines {
+		f := strings.Fields(l)
+		var body []string
+		switch {
+		case hasSystemd(l):
+			body = []string{l}
+		case len(f) == 2 && f[0] == "@include" && strings.Contains(f[1], "session"),
+			len(f) == 3 && f[0] == "session" && f[1] == "include":
+			name := f[len(f)-1]
+			if strings.Contains(name, "/") {
+				continue
+			}
+			text, err := read(name)
+			if err != nil {
+				return lines, false
+			}
+			for _, il := range strings.Split(text, "\n") {
+				t := strings.TrimSpace(il)
+				if t == "" || strings.HasPrefix(t, "#") {
+					continue
+				}
+				if f[0] == "session" && !isSession(t) {
+					continue // include takes only the lines of its own type
+				}
+				body = append(body, t)
+			}
+		default:
+			continue
+		}
+		at := -1
+		for j, b := range body {
+			if hasSystemd(b) {
+				at = j
+			}
+			if g := strings.Fields(b); len(g) > 0 && (g[0] == "@include" || strings.Contains(b, "include") || strings.Contains(b, "substack")) {
+				return lines, false // nested: not followed
+			}
+		}
+		if at < 0 {
+			continue
+		}
+		// A jump in sshd's own session lines may cross what is put in.
+		for j, o := range lines {
+			if j != i && isSession(o) && pamJump.MatchString(pamControl(o)) {
+				return lines, false
+			}
+		}
+		body, ok := pamInsert(body, at, guard)
+		if !ok {
+			return lines, false
+		}
+		block := append([]string{pamSession, pamWas + l}, body...)
+		block = append(block, pamEnd)
+		return append(lines[:i:i], append(block, lines[i+1:]...)...), true
+	}
+	return lines, false
+}
+
+func pamControl(l string) string {
+	if m := pamLine.FindStringSubmatch(l); m != nil {
+		return m[2]
+	}
+	return ""
+}
+
+// pamInsert puts guard in front of body[at], a jump that crosses it made
+// one longer so it lands where it did.
+func pamInsert(body []string, at int, guard string) ([]string, bool) {
+	// Each session line's place among the session lines.
+	idx := make([]int, len(body))
+	n := 0
+	for j, b := range body {
+		idx[j] = n
+		if isSession(b) {
+			n++
+		}
+	}
+	out := make([]string, 0, len(body)+1)
+	for j, b := range body {
+		if j == at {
+			out = append(out, guard)
+		}
+		ctl := pamControl(b)
+		if isSession(b) && j < at && strings.HasPrefix(ctl, "[") {
+			from, target := idx[j], idx[at]
+			fixed := pamJump.ReplaceAllStringFunc(ctl, func(s string) string {
+				k, err := strconv.Atoi(s[1:])
+				if err != nil {
+					return s
+				}
+				if from < target && target <= from+k {
+					k++
+				}
+				return fmt.Sprintf("=%d", k)
+			})
+			b = strings.Replace(b, ctl, fixed, 1)
+		}
+		out = append(out, b)
+	}
+	return out, true
+}

internal/slurm/pam_test.go

@@ -0,0 +1,113 @@
+package slurm
+
+import (
+	"errors"
+	"strings"
+	"testing"
+)
+
+const ubuntuSSHD = `# PAM configuration for the Secure Shell service
+@include common-auth
+account    required     pam_nologin.so
+@include common-account
+session [success=ok ignore=ignore module_unknown=ignore default=bad]        pam_selinux.so close
+session    required     pam_loginuid.so
+@include common-session
+session    required     pam_limits.so
+@include common-password
+`
+
+const ubuntuSession = `# here are the per-package modules (the "Primary" block)
+session	[default=1]			pam_permit.so
+session	requisite			pam_deny.so
+session	required			pam_permit.so
+session optional			pam_umask.so
+session	required	pam_unix.so
+session	optional	pam_systemd.so
+# end of pam-auth-update config
+`
+
+const redhatSSHD = `#%PAM-1.0
+auth       substack     password-auth
+account    required     pam_sepermit.so
+account    required     pam_nologin.so
+account    include      password-auth
+password   include      password-auth
+session    required     pam_loginuid.so
+session    include      password-auth
+session    include      postlogin
+`
+
+const redhatPasswordAuth = `auth        required      pam_env.so
+account     required      pam_unix.so
+-session    optional      pam_keyinit.so revoke
+session     required      pam_limits.so
+-session    optional      pam_systemd.so
+session     [success=1 default=ignore] pam_succeed_if.so service in crond quiet use_uid
+session     required      pam_unix.so
+`
+
+func files(m map[string]string) func(string) (string, error) {
+	return func(n string) (string, error) {
+		if s, ok := m[n]; ok {
+			return s, nil
+		}
+		return "", errors.New("no such file")
+	}
+}
+
+func TestPAMAdopt(t *testing.T) {
+	read := files(map[string]string{"common-session": ubuntuSession, "password-auth": redhatPasswordAuth, "postlogin": "session optional pam_lastlog.so\n"})
+	for name, stack := range map[string]string{"ubuntu": ubuntuSSHD, "redhat": redhatSSHD} {
+		on, contained, err := PAMAdopt(stack, "/var/lib/x-users", read, true)
+		if err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		if !contained {
+			t.Errorf("%s: pam_systemd not skipped:\n%s", name, on)
+		}
+		lines := strings.Split(on, "\n")
+		if !strings.Contains(on, "account [success=ignore default=1] pam_listfile.so item=user sense=allow file=/var/lib/x-users onerr=fail") {
+			t.Fatalf("%s: lines not put in:\n%s", name, on)
+		}
+		for j, l := range lines {
+			if l == pamBegin && !strings.Contains(lines[j-1], "account") {
+				t.Errorf("%s: put after %q", name, lines[j-1])
+			}
+			if strings.Contains(l, "pam_systemd.so") &&
+				lines[j-1] != "session [success=1 default=ignore] pam_listfile.so item=user sense=allow file=/var/lib/x-users onerr=fail" {
+				t.Errorf("%s: pam_systemd not guarded, after %q", name, lines[j-1])
+			}
+		}
+		again, _, _ := PAMAdopt(on, "/var/lib/x-users", read, true)
+		if again != on {
+			t.Errorf("%s: put in twice:\n%s", name, again)
+		}
+		off, _, err := PAMAdopt(on, "", read, false)
+		if err != nil || off != stack {
+			t.Errorf("%s: taken out, not as it was (%v):\n%s", name, err, off)
+		}
+	}
+	// The jump over pam_deny lands where it did; the one past crond's line too.
+	on, _, _ := PAMAdopt(ubuntuSSHD, "/l", read, true)
+	if !strings.Contains(on, "[default=1]\t\t\tpam_permit.so") {
+		t.Errorf("a jump that does not cross the guard was changed:\n%s", on)
+	}
+	body := []string{"session [default=2] pam_permit.so", "session required pam_unix.so", "session optional pam_systemd.so", "session required pam_x.so"}
+	got, _ := pamInsert(body, 2, "G")
+	if got[0] != "session [default=3] pam_permit.so" || got[2] != "G" {
+		t.Errorf("a jump across the guard not made longer: %q", got)
+	}
+
+	if _, _, err := PAMAdopt("auth required pam_unix.so\n", "/x", read, true); err == nil {
+		t.Error("a stack with no account lines taken")
+	}
+	if _, _, err := PAMAdopt(pamBegin+"\naccount required x\n", "/x", read, false); err == nil {
+		t.Error("lines with no end taken")
+	}
+	// The session file unreadable: held to their jobs all the same.
+	on, contained, err := PAMAdopt(ubuntuSSHD, "/l", files(nil), true)
+	if err != nil || contained || !strings.Contains(on, AdoptModule) {
+		t.Errorf("contained %v, err %v:\n%s", contained, err, on)
+	}
+}

internal/slurm/slurm.go

@@ -214,6 +214,9 @@ SlurmdLogFile=` + LogDir + `/slurmd.log
 SlurmctldPidFile=/run/slurmctld.pid
 SlurmdPidFile=/run/slurmd.pid
 ProctrackType=proctrack/cgroup
+# Each job has a step from its start, where pam_slurm_adopt puts a user's
+# ssh session on the node.
+PrologFlags=Contain
 TaskPlugin=task/cgroup,task/affinity
 # What each job used, from its cgroup, for the records; the cards too.
 JobAcctGatherType=jobacct_gather/cgroup