Slurm's shared files and users: one path everywhere, accounts as Linux users
CI / test (push) Failing after 7m1s

slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 21:34:34 +01:00
1 parent b9d20e7bba
commit d8ce573bdb
45 files changed
+2505 -79

No files matched your search

+45 -4
View File
@@ -144,6 +144,13 @@ func (in *serverInit) checkChoices() error {
if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil {
return fmt.Errorf("slurm: this machine's hostname: %w", err)
}
set := config.SlurmSettings{Enabled: true, Controller: localNodeID(), Shared: slurmShared(in.slurmShared)}
if err := set.Check(); err != nil {
return fmt.Errorf("--slurm-shared %q: here, none, or the site's HOST:/EXPORT: %w", in.slurmShared, err)
}
if u := in.slurmUsers; u != "" && u != config.UsersLlmgw && u != "site" {
return fmt.Errorf("--slurm-users %q: llmgw or site", u)
}
}
if in.vip != "" {
ip, _, err := net.ParseCIDR(in.vip)
@@ -329,7 +336,7 @@ func (in *serverInit) tuneConfig(body string) string {
"engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n")
}
if in.slurm {
extra = append(extra, slurmSection(localNodeID()))
extra = append(extra, slurmSection(localNodeID(), slurmShared(in.slurmShared), in.slurmUsers != "site"))
}
if in.evictAfterIdle > 0 || in.minLifetime > 0 {
b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n"
@@ -347,15 +354,43 @@ func (in *serverInit) tuneConfig(body string) string {
return body
}
// slurmShared is --slurm-shared as settings: "here" is /shared exported
// by this machine's node, "none" nothing, anything else the site's export.
func slurmShared(v string) config.SharedStorage {
switch v {
case "", "none":
return config.SharedStorage{}
case "here":
return config.SharedStorage{Path: "/shared", Server: localNodeID()}
}
return config.SharedStorage{Path: "/shared", NFS: v}
}
// slurmSection is server.yaml's slurm block as an install chooses it:
// slurmctld on this machine's node, slurmd on every node.
func slurmSection(controller string) string {
return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
// slurmctld on this machine's node, slurmd on every node, the shared files
// and the users as chosen.
func slurmSection(controller string, shared config.SharedStorage, users bool) string {
b := "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
"# for the jobs' records, on this machine; slurmd on\n" +
"# every node, those that join later too. Each node is llmgw's, drained in\n" +
"# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" +
"# reclaim NODE' takes it back once its jobs end.\n" +
"slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n"
switch {
case shared.Server != "":
b += " # /shared on every Slurm machine, exported over NFS from this one\n" +
" # ('llmgw ctl slurm shared' points it at the site's storage instead).\n" +
" shared:\n path: " + yamlString(shared.Path) + "\n server: " + yamlString(shared.Server) + "\n"
case shared.NFS != "":
b += " # /shared on every Slurm machine, mounted from the site's export.\n" +
" shared:\n path: " + yamlString(shared.Path) + "\n nfs: " + yamlString(shared.NFS) + "\n"
}
if users {
b += " # A Linux user for each llmgw account, the same UID everywhere; SSH keys\n" +
" # in the portal's Profile. Empty leaves users to the site (sssd, a directory).\n" +
" users: llmgw\n"
}
return b
}
// localNodeID is the node id this machine's agent takes: its hostname.
@@ -501,6 +536,12 @@ func (in *serverInit) initFlags() []string {
}
if in.slurm {
add("--slurm")
if in.slurmShared != "" && in.slurmShared != "here" {
add("--slurm-shared", in.slurmShared)
}
if in.slurmUsers == "site" {
add("--slurm-users", "site")
}
}
if in.evictAfterIdle > 0 {
add("--evict-after-idle", durationYAML(in.evictAfterIdle))