Slurm's shared files and users: one path everywhere, accounts as Linux users
CI / test (push) Failing after 7m1s

slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 21:34:34 +01:00
1 parent b9d20e7bba
commit d8ce573bdb
45 files changed
+2505 -79

No files matched your search

+83 -1
View File
@@ -2,8 +2,10 @@ package config
import (
"fmt"
"path"
"regexp"
"slices"
"strings"
)
// SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes
@@ -38,6 +40,83 @@ type SlurmSettings struct {
// Partition is the one partition every node is in. Empty is "gpu".
Partition string `koanf:"partition"`
// Shared is the file system every machine of the Slurm cluster sees at
// the same path: home directories, datasets, the jobs' output and
// checkpoints. Empty path: none, and a batch job's output stays on the
// node it ran on.
Shared SharedStorage `koanf:"shared"`
// Users is who keeps the Linux users that run jobs, the same on every
// machine of the Slurm cluster: "llmgw" makes one for each active llmgw
// account (its login from the e-mail address, a UID of llmgw's from
// 200000 up, its SSH keys from the portal); empty leaves users to the
// site (a directory through sssd, or by hand).
Users string `koanf:"users"`
}
// SharedStorage is where the Slurm cluster's shared files come from: the
// site's NFS export (NFS), an NFS export llmgw keeps on one of its nodes
// (Server), or, with neither, a file system the site mounts itself
// (Lustre, BeeGFS, GPFS), which llmgw only looks for.
type SharedStorage struct {
// Path is where every machine sees it, as /shared.
Path string `koanf:"path"`
// NFS is the site's export, host:/path, mounted at Path on every
// machine of the Slurm cluster.
NFS string `koanf:"nfs"`
// Server is the llmgw node, the controller or one of the nodes, that
// exports Path over NFS to the others: for a site with no storage of
// its own. One machine: not highly available, and no faster than its
// disk and network.
Server string `koanf:"server"`
}
// UsersLlmgw has llmgw make the Slurm cluster's users from its accounts.
const UsersLlmgw = "llmgw"
// HomeDir is where a user's home directory is: under the shared path when
// there is one, else on each machine's own disk.
func (s SlurmSettings) HomeDir(login string) string {
if s.Shared.Path != "" {
return path.Join(s.Shared.Path, "home", login)
}
return path.Join("/home", login)
}
// nfsExport is host:/path.
var nfsExport = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:\[\]-]*:/[A-Za-z0-9_./-]*$`)
// sharedPath is an absolute path of plain names, which a mount unit's name
// is made from.
var sharedPath = regexp.MustCompile(`^(/[A-Za-z0-9_.-]+)+$`)
func (s SlurmSettings) checkShared() error {
sh := s.Shared
if sh.Path == "" {
if sh.NFS != "" || sh.Server != "" {
return fmt.Errorf("slurm.shared.path is empty: say where every machine sees the shared files, as /shared")
}
return nil
}
if !sharedPath.MatchString(sh.Path) || strings.Contains(sh.Path, "/.") {
return fmt.Errorf("slurm.shared.path %q is not an absolute path of plain names, as /shared", sh.Path)
}
for _, sys := range []string{"/", "/etc", "/usr", "/var", "/var/lib", "/home", "/root", "/boot", "/opt", "/srv", "/tmp", "/proc", "/sys", "/dev", "/run"} {
if path.Clean(sh.Path) == sys {
return fmt.Errorf("slurm.shared.path %s is the system's own directory: take one of its own, as /shared", sh.Path)
}
}
if sh.NFS != "" && sh.Server != "" {
return fmt.Errorf("slurm.shared names both the site's export (nfs) and an llmgw node to export it (server): one of them")
}
if sh.NFS != "" && !nfsExport.MatchString(sh.NFS) {
return fmt.Errorf("slurm.shared.nfs %q is not host:/path", sh.NFS)
}
if sh.Server != "" && sh.Server != s.Controller && !s.AllNodes && !slices.Contains(s.Nodes, sh.Server) {
return fmt.Errorf("slurm.shared.server %q is neither the controller nor one of slurm.nodes", sh.Server)
}
return nil
}
// slurmName is what Slurm takes as a node, cluster or partition name.
@@ -111,5 +190,8 @@ func (s SlurmSettings) Check() error {
return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n)
}
}
return nil
if s.Users != "" && s.Users != UsersLlmgw {
return fmt.Errorf("slurm.users %q: %q has llmgw make them, empty leaves them to the site", s.Users, UsersLlmgw)
}
return s.checkShared()
}
+43
View File
@@ -35,6 +35,49 @@ func TestSlurmCheck(t *testing.T) {
}
}
func TestSlurmShared(t *testing.T) {
base := SlurmSettings{Enabled: true, Controller: "c", Nodes: []string{"a"}}
for _, sh := range []SharedStorage{{}, {Path: "/shared"}, {Path: "/shared", NFS: "filer.site:/export/hpc"},
{Path: "/data/hpc", Server: "c"}, {Path: "/shared", Server: "a"}} {
s := base
s.Shared = sh
if err := s.Check(); err != nil {
t.Errorf("%+v refused: %v", sh, err)
}
}
for want, sh := range map[string]SharedStorage{
"path is empty": {NFS: "f:/x"},
"plain names": {Path: "shared"},
"system's own": {Path: "/home"},
"one of them": {Path: "/shared", NFS: "f:/x", Server: "c"},
"not host:/path": {Path: "/shared", NFS: "f/x"},
"neither": {Path: "/shared", Server: "z"},
} {
s := base
s.Shared = sh
if err := s.Check(); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%+v: got %v, want %q", sh, err, want)
}
}
s := base
s.Shared = SharedStorage{Path: "/a/../etc"}
if s.Check() == nil {
t.Error("a path with .. taken")
}
s.Shared, s.Users = SharedStorage{}, "ldap"
if s.Check() == nil {
t.Error("slurm.users ldap taken")
}
s.Users, s.Shared = UsersLlmgw, SharedStorage{Path: "/shared"}
if s.Check() != nil || s.HomeDir("ayse") != "/shared/home/ayse" {
t.Errorf("home %s", s.HomeDir("ayse"))
}
s.Shared = SharedStorage{}
if s.HomeDir("ayse") != "/home/ayse" {
t.Errorf("home %s", s.HomeDir("ayse"))
}
}
func TestSlurmAllNodes(t *testing.T) {
s := SlurmSettings{Enabled: true, Controller: "c", AllNodes: true, Lent: []string{"joined"}}
if err := s.Check(); err != nil {