Slurm users reach a node only through a job of theirs
CI / test (push) Successful in 9m41s

llmgw's Slurm users log in to the controller freely and to another Slurm
machine only while a job of theirs runs there: the agent installs
pam_slurm_adopt and puts it in sshd's account stack for the users it made
alone, so root and the machine's own users are never locked out. Their
session is put in the job's extern step (PrologFlags=Contain), and
pam_systemd, which would move it out, is skipped for them by taking the
session file into sshd's stack with a guard in front of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-14 20:30:49 +01:00
1 parent a00c92648e
commit 4b4306bb6a
11 files changed
+499 -7

No files matched your search

+47
View File
@@ -24,6 +24,7 @@ func TestTheServerExportsAndMakesUsersWithHomes(t *testing.T) {
m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Serve: true, Clients: []string{"10.0.0.22", "10.0.0.21"}}
m.ManageUsers, m.MakeHomes, m.Users = true, true, []*llmgwv1.SlurmUser{ayse()}
h.fail["test -d /shared/home/ayse"] = true
h.files[slurm.PAMSSHD] = []byte(pamStack)
st := k.Reconcile(context.Background(), m)
if st.GetError() != "" {
t.Fatal(st.GetError())
@@ -122,3 +123,49 @@ func TestAClientMountsAndLeavesOthersUsersAlone(t *testing.T) {
t.Errorf("mount kept:\n%s", strings.Join(h.ran, "\n"))
}
}
const pamStack = "@include common-auth\naccount required pam_nologin.so\n@include common-account\n@include common-session\n"
func TestUsersReachANodeOnlyThroughTheirJobs(t *testing.T) {
h := newFake()
inst := &fakeInst{host: h}
k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil)
h.files[slurm.PAMSSHD] = []byte(pamStack)
// The module is not there until its package is installed.
for _, d := range slurm.ModuleDirs {
h.fail["test -f "+d+"/"+slurm.AdoptModule] = true
}
m := nodeMsg(true)
m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse()}
k.Reconcile(context.Background(), m)
if !slices.Contains(inst.got["deb"], "libpam-slurm-adopt") {
t.Fatalf("not installed: %v", inst.got)
}
if got := string(h.files[slurm.PAMSSHD]); got != pamStack {
t.Fatalf("lines put in with the module still missing:\n%s", got)
}
delete(h.fail, "test -f "+slurm.ModuleDirs[0]+"/"+slurm.AdoptModule)
st := k.Reconcile(context.Background(), m)
got := string(h.files[slurm.PAMSSHD])
if !strings.Contains(got, "pam_listfile.so item=user sense=allow file="+k.usersMarker()) ||
!strings.Contains(got, "account required "+slurm.AdoptModule) {
t.Fatalf("not held to their jobs (%s):\n%s", st.GetError(), got)
}
// The controller is where they log in: not held there.
m.Controller = true
m.Files = map[string]string{"slurm.conf": "x"}
k.Reconcile(context.Background(), m)
if string(h.files[slurm.PAMSSHD]) != pamStack {
t.Errorf("held on the controller:\n%s", h.files[slurm.PAMSSHD])
}
// No users of llmgw's any more: the lines go.
m.Controller, m.Files = false, nil
k.Reconcile(context.Background(), m)
m.Users = nil
k.Reconcile(context.Background(), m)
if string(h.files[slurm.PAMSSHD]) != pamStack {
t.Errorf("lines left with no users:\n%s", h.files[slurm.PAMSSHD])
}
}
+81
View File
@@ -52,9 +52,87 @@ func (k *Keeper) users(ctx context.Context, want *llmgwv1.Slurm) (int32, error)
if err := k.sshConf(ctx, len(keep) > 0); err != nil {
errs = append(errs, err)
}
// The controller is where they log in and submit from; a node only
// while a job of theirs runs there.
if err := k.nodeLogin(ctx, len(keep) > 0 && want.GetNode() && !want.GetController()); err != nil {
errs = append(errs, err)
}
return n, errors.Join(errs...)
}
// nodeLogin holds the users llmgw made here to their jobs when they log in
// (pam_slurm_adopt), or no longer. Only they are: root and the site's
// administrators log in as before, so a mistake here never locks out the
// people who would put it right.
func (k *Keeper) nodeLogin(ctx context.Context, on bool) error {
old, err := k.host.ReadFile(slurm.PAMSSHD)
if err != nil {
if !on {
return nil
}
return fmt.Errorf("reading %s: %w", slurm.PAMSSHD, err)
}
if on {
if err := k.adoptModule(ctx); err != nil {
return err
}
}
read := func(name string) (string, error) {
b, err := k.host.ReadFile(filepath.Join(filepath.Dir(slurm.PAMSSHD), name))
return string(b), err
}
stack, contained, err := slurm.PAMAdopt(string(old), k.usersMarker(), read, on)
if err != nil {
return err
}
if stack == string(old) {
return nil
}
if on {
k.log.Info("llmgw's Slurm users log in to this node only while a job of theirs runs here",
"in_the_job", contained)
if !contained {
k.log.Warn("pam_systemd could not be skipped for them safely here: their ssh sessions are not held " +
"to their job's cores and cards, only to having one")
}
} else {
k.log.Info("llmgw's Slurm users are no longer held to their jobs when they log in here")
}
// Read by sshd for each login: nothing to reload.
_, err = k.write(slurm.PAMSSHD, []byte(stack), 0o644, "")
return err
}
// adoptModule is there, or is installed.
func (k *Keeper) adoptModule(ctx context.Context) error {
if k.installed[slurm.AdoptModule] {
return nil
}
find := func() bool {
for _, d := range slurm.ModuleDirs {
if _, err := k.host.Run(ctx, nil, "test", "-f", filepath.Join(d, slurm.AdoptModule)); err == nil {
return true
}
}
return false
}
if !find() {
if k.inst == nil {
return fmt.Errorf("%s missing, and this agent installs nothing", slurm.AdoptModule)
}
k.log.Info("installing pam_slurm_adopt, which holds llmgw's Slurm users to their jobs here")
if err := k.inst.InstallPackages(ctx, slurm.AdoptPackages); err != nil {
return err
}
if !find() {
return fmt.Errorf("%s is still missing after installing %s", slurm.AdoptModule,
strings.Join(slurm.AdoptPackages["deb"], ", "))
}
}
k.installed[slurm.AdoptModule] = true
return nil
}
// errNotMade is a user llmgw could not make here, nor took for its own.
var errNotMade = errors.New("not made")
@@ -162,6 +240,9 @@ func (k *Keeper) removeUsers(ctx context.Context, made, keep []string) error {
if err := k.sshConf(ctx, false); err != nil {
errs = append(errs, err)
}
if err := k.nodeLogin(ctx, false); err != nil {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}