Slurm's shared files and users: one path everywhere, accounts as Linux users
CI / test (push) Failing after 7m1s

slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 21:34:34 +01:00
1 parent b9d20e7bba
commit d8ce573bdb
45 files changed
+2505 -79

No files matched your search

+167 -4
View File
@@ -3,12 +3,16 @@ package cli
import (
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"slices"
"strings"
"text/tabwriter"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
@@ -27,7 +31,148 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client),
newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true),
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false))
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false),
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON))
return cmd
}
func newSlurmSharedCmd(client func() *ctlClient) *cobra.Command {
var path, server, nfs, note string
var off bool
c := &cobra.Command{
Use: "shared --path DIR (--server NODE | --nfs HOST:/EXPORT) | --off",
Short: "Files every machine of the Slurm cluster sees at one path",
Long: "Home directories, datasets, the jobs' output and checkpoints, at the same path on the controller\n" +
"and every node. Three ways:\n" +
" --nfs HOST:/EXPORT the site's own NFS export, mounted at --path on every machine;\n" +
" --server NODE an llmgw node (the controller or a node) exports --path over NFS to the\n" +
" others, for a site with no storage of its own: one machine, not highly\n" +
" available, no faster than its disk and network;\n" +
" neither the site mounts --path itself (Lustre, BeeGFS, GPFS); llmgw only looks\n" +
" that it is there.\n" +
"The export answers the Slurm machines alone, NFS's port the cluster's networks. A machine\n" +
"that gives up its mount does so when no job holds files on it.",
Example: " llmgw ctl slurm shared --path /shared --server node06\n" +
" llmgw ctl slurm shared --path /shared --nfs filer.site:/export/hpc\n" +
" llmgw ctl slurm shared --path /lustre/hpc\n" +
" llmgw ctl slurm shared --off",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if off == (path != "") {
return fmt.Errorf("give --path, or --off")
}
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if off {
if doc.Node([]string{"slurm", "shared"}) != nil {
doc.Unset([]string{"slurm", "shared"})
}
return nil
}
v := map[string]string{"path": path}
if server != "" {
v["server"] = server
}
if nfs != "" {
v["nfs"] = nfs
}
b, _ := json.Marshal(v)
return doc.SetYAML([]string{"slurm", "shared"}, string(b))
})
},
}
c.Flags().StringVar(&path, "path", "", "where every machine sees the files, as /shared")
c.Flags().StringVar(&server, "server", "", "the llmgw node that exports them over NFS")
c.Flags().StringVar(&nfs, "nfs", "", "the site's NFS export, HOST:/EXPORT")
c.Flags().BoolVar(&off, "off", false, "no shared files; mounts and exports llmgw made are taken away")
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
return c
}
func newSlurmUsersCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
cmd := &cobra.Command{
Use: "users",
Short: "The Linux users that run jobs, made from llmgw's accounts",
Long: "With slurm.users \"llmgw\" every active account is a Linux user on every machine of the Slurm\n" +
"cluster: its login from its e-mail address, a UID of llmgw's from 200000 up (never given to\n" +
"another, even when the account goes), its home under the shared path. It logs in over SSH\n" +
"with the keys given in the portal (Profile) or here. An account suspended or deleted loses\n" +
"its user; its home and files stay. Off, users are the site's: from its directory (sssd), or\n" +
"made by hand.",
}
list := &cobra.Command{
Use: "list",
Short: "Every user, with its login, UID and how many keys it has",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var us []slurmsvc.User
if err := client().get(cmd.Context(), "/api/v1/slurm/users", &us); err != nil {
return err
}
w := cmd.OutOrStdout()
if *asJSON {
return json.NewEncoder(w).Encode(us)
}
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, "LOGIN\tUID\tACCOUNT\tSSH KEYS")
for _, u := range us {
fmt.Fprintf(tw, "%s\t%d\t%s\t%d\n", u.Login, u.UID, u.Email, len(u.SSHKeys))
}
return tw.Flush()
},
}
var note string
toggle := func(use, short, value string) *cobra.Command {
c := &cobra.Command{
Use: use,
Short: short,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if value == "" {
if doc.Node([]string{"slurm", "users"}) != nil {
doc.Unset([]string{"slurm", "users"})
}
return nil
}
return doc.SetYAML([]string{"slurm", "users"}, yamlString(value))
})
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
return c
}
keys := &cobra.Command{
Use: "keys ACCOUNT FILE",
Short: "Replace an account's SSH public keys, from a file (- for standard input)",
Long: "FILE holds OpenSSH public keys, one a line, as ~/.ssh/id_ed25519.pub; an empty one takes them all away.",
Example: " llmgw ctl slurm users keys ayse@site.example ayse.pub\n" +
" ssh-add -L | llmgw ctl slurm users keys ayse@site.example -",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
var b []byte
var err error
if args[1] == "-" {
b, err = io.ReadAll(io.LimitReader(cmd.InOrStdin(), 64<<10))
} else {
b, err = os.ReadFile(args[1])
}
if err != nil {
return err
}
body, _ := json.Marshal(map[string]string{"keys": string(b)})
var u slurmsvc.User
if err := client().put(cmd.Context(), "/api/v1/slurm/users/"+url.PathEscape(args[0])+"/ssh-keys", body, &u); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s (UID %d) has %d SSH key(s); the machines have them within a minute\n",
u.Login, u.UID, len(u.SSHKeys))
return nil
},
}
cmd.AddCommand(list,
toggle("on", "Make a Linux user for every active account, on every machine of the Slurm cluster", "llmgw"),
toggle("off", "Leave users to the site; those llmgw made are taken away, their homes left", ""),
keys)
return cmd
}
@@ -62,14 +207,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
default:
fmt.Fprintln(w, ": slurmctld not running")
}
fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared))
if st.Users != "" {
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
}
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tNOTE")
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE")
for _, n := range st.Nodes {
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe()
if r := n.Status; r != nil {
state = or(r.State, "-")
jobs = fmt.Sprint(r.Jobs)
slurmd = map[bool]string{true: "running", false: "stopped"}[r.SlurmdRunning]
shared = or(r.Shared, "-")
if r.Error != "" {
note = r.Error
}
@@ -77,7 +227,7 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if !n.Connected {
note = "agent not connected; " + note
}
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, note)
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, shared, note)
}
if err := tw.Flush(); err != nil {
return err
@@ -172,6 +322,19 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command {
return c
}
// sharedLine says where the shared files are and come from.
func sharedLine(sh config.SharedStorage) string {
switch {
case sh.Path == "":
return "none (`llmgw ctl slurm shared` adds them; a batch job's output stays on its node)"
case sh.Server != "":
return fmt.Sprintf("%s, exported over NFS by %s", sh.Path, sh.Server)
case sh.NFS != "":
return fmt.Sprintf("%s, mounted from %s", sh.Path, sh.NFS)
}
return sh.Path + ", mounted by the site"
}
// newSlurmListCmd adds nodes to slurm.lent, or takes them out.
func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cobra.Command {
var note string