Slurm's shared files and users: one path everywhere, accounts as Linux users
CI / test (push) Failing after 7m1s

slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-12 21:34:34 +01:00
1 parent b9d20e7bba
commit d8ce573bdb
45 files changed
+2505 -79

No files matched your search

+167 -4
View File
@@ -3,12 +3,16 @@ package cli
import (
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"slices"
"strings"
"text/tabwriter"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
@@ -27,7 +31,148 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client),
newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true),
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false))
newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false),
newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON))
return cmd
}
func newSlurmSharedCmd(client func() *ctlClient) *cobra.Command {
var path, server, nfs, note string
var off bool
c := &cobra.Command{
Use: "shared --path DIR (--server NODE | --nfs HOST:/EXPORT) | --off",
Short: "Files every machine of the Slurm cluster sees at one path",
Long: "Home directories, datasets, the jobs' output and checkpoints, at the same path on the controller\n" +
"and every node. Three ways:\n" +
" --nfs HOST:/EXPORT the site's own NFS export, mounted at --path on every machine;\n" +
" --server NODE an llmgw node (the controller or a node) exports --path over NFS to the\n" +
" others, for a site with no storage of its own: one machine, not highly\n" +
" available, no faster than its disk and network;\n" +
" neither the site mounts --path itself (Lustre, BeeGFS, GPFS); llmgw only looks\n" +
" that it is there.\n" +
"The export answers the Slurm machines alone, NFS's port the cluster's networks. A machine\n" +
"that gives up its mount does so when no job holds files on it.",
Example: " llmgw ctl slurm shared --path /shared --server node06\n" +
" llmgw ctl slurm shared --path /shared --nfs filer.site:/export/hpc\n" +
" llmgw ctl slurm shared --path /lustre/hpc\n" +
" llmgw ctl slurm shared --off",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
if off == (path != "") {
return fmt.Errorf("give --path, or --off")
}
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if off {
if doc.Node([]string{"slurm", "shared"}) != nil {
doc.Unset([]string{"slurm", "shared"})
}
return nil
}
v := map[string]string{"path": path}
if server != "" {
v["server"] = server
}
if nfs != "" {
v["nfs"] = nfs
}
b, _ := json.Marshal(v)
return doc.SetYAML([]string{"slurm", "shared"}, string(b))
})
},
}
c.Flags().StringVar(&path, "path", "", "where every machine sees the files, as /shared")
c.Flags().StringVar(&server, "server", "", "the llmgw node that exports them over NFS")
c.Flags().StringVar(&nfs, "nfs", "", "the site's NFS export, HOST:/EXPORT")
c.Flags().BoolVar(&off, "off", false, "no shared files; mounts and exports llmgw made are taken away")
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
return c
}
func newSlurmUsersCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
cmd := &cobra.Command{
Use: "users",
Short: "The Linux users that run jobs, made from llmgw's accounts",
Long: "With slurm.users \"llmgw\" every active account is a Linux user on every machine of the Slurm\n" +
"cluster: its login from its e-mail address, a UID of llmgw's from 200000 up (never given to\n" +
"another, even when the account goes), its home under the shared path. It logs in over SSH\n" +
"with the keys given in the portal (Profile) or here. An account suspended or deleted loses\n" +
"its user; its home and files stay. Off, users are the site's: from its directory (sssd), or\n" +
"made by hand.",
}
list := &cobra.Command{
Use: "list",
Short: "Every user, with its login, UID and how many keys it has",
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
var us []slurmsvc.User
if err := client().get(cmd.Context(), "/api/v1/slurm/users", &us); err != nil {
return err
}
w := cmd.OutOrStdout()
if *asJSON {
return json.NewEncoder(w).Encode(us)
}
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, "LOGIN\tUID\tACCOUNT\tSSH KEYS")
for _, u := range us {
fmt.Fprintf(tw, "%s\t%d\t%s\t%d\n", u.Login, u.UID, u.Email, len(u.SSHKeys))
}
return tw.Flush()
},
}
var note string
toggle := func(use, short, value string) *cobra.Command {
c := &cobra.Command{
Use: use,
Short: short,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
if value == "" {
if doc.Node([]string{"slurm", "users"}) != nil {
doc.Unset([]string{"slurm", "users"})
}
return nil
}
return doc.SetYAML([]string{"slurm", "users"}, yamlString(value))
})
},
}
c.Flags().StringVar(&note, "note", "", "why, kept with the settings' version")
return c
}
keys := &cobra.Command{
Use: "keys ACCOUNT FILE",
Short: "Replace an account's SSH public keys, from a file (- for standard input)",
Long: "FILE holds OpenSSH public keys, one a line, as ~/.ssh/id_ed25519.pub; an empty one takes them all away.",
Example: " llmgw ctl slurm users keys ayse@site.example ayse.pub\n" +
" ssh-add -L | llmgw ctl slurm users keys ayse@site.example -",
Args: cobra.ExactArgs(2),
RunE: func(cmd *cobra.Command, args []string) error {
var b []byte
var err error
if args[1] == "-" {
b, err = io.ReadAll(io.LimitReader(cmd.InOrStdin(), 64<<10))
} else {
b, err = os.ReadFile(args[1])
}
if err != nil {
return err
}
body, _ := json.Marshal(map[string]string{"keys": string(b)})
var u slurmsvc.User
if err := client().put(cmd.Context(), "/api/v1/slurm/users/"+url.PathEscape(args[0])+"/ssh-keys", body, &u); err != nil {
return err
}
fmt.Fprintf(cmd.OutOrStdout(), "%s (UID %d) has %d SSH key(s); the machines have them within a minute\n",
u.Login, u.UID, len(u.SSHKeys))
return nil
},
}
cmd.AddCommand(list,
toggle("on", "Make a Linux user for every active account, on every machine of the Slurm cluster", "llmgw"),
toggle("off", "Leave users to the site; those llmgw made are taken away, their homes left", ""),
keys)
return cmd
}
@@ -62,14 +207,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
default:
fmt.Fprintln(w, ": slurmctld not running")
}
fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared))
if st.Users != "" {
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
}
tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tNOTE")
fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE")
for _, n := range st.Nodes {
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe()
if r := n.Status; r != nil {
state = or(r.State, "-")
jobs = fmt.Sprint(r.Jobs)
slurmd = map[bool]string{true: "running", false: "stopped"}[r.SlurmdRunning]
shared = or(r.Shared, "-")
if r.Error != "" {
note = r.Error
}
@@ -77,7 +227,7 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if !n.Connected {
note = "agent not connected; " + note
}
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, note)
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, shared, note)
}
if err := tw.Flush(); err != nil {
return err
@@ -172,6 +322,19 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command {
return c
}
// sharedLine says where the shared files are and come from.
func sharedLine(sh config.SharedStorage) string {
switch {
case sh.Path == "":
return "none (`llmgw ctl slurm shared` adds them; a batch job's output stays on its node)"
case sh.Server != "":
return fmt.Sprintf("%s, exported over NFS by %s", sh.Path, sh.Server)
case sh.NFS != "":
return fmt.Sprintf("%s, mounted from %s", sh.Path, sh.NFS)
}
return sh.Path + ", mounted by the site"
}
// newSlurmListCmd adds nodes to slurm.lent, or takes them out.
func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cobra.Command {
var note string
+11
View File
@@ -256,6 +256,7 @@ func newServerCmd() *cobra.Command {
// The settings in force, for what reads them at each use.
var current atomic.Pointer[config.Server]
current.Store(&cfg)
rt.SetSlurm(func() config.SlurmSettings { return current.Load().Slurm })
var agentSvc *agentsvc.Service
var agentHandler http.Handler
var agentPath string
@@ -452,6 +453,16 @@ func newServerCmd() *cobra.Command {
}, log)
// What slurm.conf held, for the leader that comes next.
slurmSvc.SetMemory(slurmsvc.StoreMemory(st))
// slurm.users "llmgw": a Linux user for each active account.
slurmSvc.SetUsers(func(ctx context.Context) ([]slurmsvc.User, error) {
us, err := st.SlurmUsers(ctx)
out := make([]slurmsvc.User, 0, len(us))
for _, u := range us {
out = append(out, adminapi.SlurmUserView(u))
}
return out, err
})
api.SetSlurmUsers(st)
agentSvc.SetSlurm(slurmSvc)
go slurmSvc.Run(ctx)
api.SetSlurm(slurmSvc)
+8
View File
@@ -127,6 +127,10 @@ type serverInit struct {
// slurm runs Slurm beside llmgw: slurmctld on this machine's node,
// slurmd on every node, each llmgw's until it is lent.
slurm bool
// slurmShared is where Slurm's shared files come from: "here" (this
// machine exports /shared), the site's export as HOST:/EXPORT, or
// "none". slurmUsers is "llmgw" (users from the accounts) or "site".
slurmShared, slurmUsers string
// evictAfterIdle and minLifetime are on_demand's; zero leaves its own.
evictAfterIdle, minLifetime time.Duration
// standbys makes each machine of the replicated store a control plane
@@ -263,6 +267,10 @@ func newServerInitCmd() *cobra.Command {
"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
f.BoolVar(&in.slurm, "slurm", false,
"Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
f.StringVar(&in.slurmShared, "slurm-shared", "here",
"with --slurm, the jobs' shared files at /shared: here (this machine exports them over NFS), the site's HOST:/EXPORT, or none")
f.StringVar(&in.slurmUsers, "slurm-users", config.UsersLlmgw,
"with --slurm, who makes the Linux users that run jobs: llmgw (one for each account, SSH keys in the portal) or site")
f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
f.DurationVar(&in.minLifetime, "min-lifetime", 0,
+45 -4
View File
@@ -144,6 +144,13 @@ func (in *serverInit) checkChoices() error {
if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil {
return fmt.Errorf("slurm: this machine's hostname: %w", err)
}
set := config.SlurmSettings{Enabled: true, Controller: localNodeID(), Shared: slurmShared(in.slurmShared)}
if err := set.Check(); err != nil {
return fmt.Errorf("--slurm-shared %q: here, none, or the site's HOST:/EXPORT: %w", in.slurmShared, err)
}
if u := in.slurmUsers; u != "" && u != config.UsersLlmgw && u != "site" {
return fmt.Errorf("--slurm-users %q: llmgw or site", u)
}
}
if in.vip != "" {
ip, _, err := net.ParseCIDR(in.vip)
@@ -329,7 +336,7 @@ func (in *serverInit) tuneConfig(body string) string {
"engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n")
}
if in.slurm {
extra = append(extra, slurmSection(localNodeID()))
extra = append(extra, slurmSection(localNodeID(), slurmShared(in.slurmShared), in.slurmUsers != "site"))
}
if in.evictAfterIdle > 0 || in.minLifetime > 0 {
b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n"
@@ -347,15 +354,43 @@ func (in *serverInit) tuneConfig(body string) string {
return body
}
// slurmShared is --slurm-shared as settings: "here" is /shared exported
// by this machine's node, "none" nothing, anything else the site's export.
func slurmShared(v string) config.SharedStorage {
switch v {
case "", "none":
return config.SharedStorage{}
case "here":
return config.SharedStorage{Path: "/shared", Server: localNodeID()}
}
return config.SharedStorage{Path: "/shared", NFS: v}
}
// slurmSection is server.yaml's slurm block as an install chooses it:
// slurmctld on this machine's node, slurmd on every node.
func slurmSection(controller string) string {
return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
// slurmctld on this machine's node, slurmd on every node, the shared files
// and the users as chosen.
func slurmSection(controller string, shared config.SharedStorage, users bool) string {
b := "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" +
"# for the jobs' records, on this machine; slurmd on\n" +
"# every node, those that join later too. Each node is llmgw's, drained in\n" +
"# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" +
"# reclaim NODE' takes it back once its jobs end.\n" +
"slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n"
switch {
case shared.Server != "":
b += " # /shared on every Slurm machine, exported over NFS from this one\n" +
" # ('llmgw ctl slurm shared' points it at the site's storage instead).\n" +
" shared:\n path: " + yamlString(shared.Path) + "\n server: " + yamlString(shared.Server) + "\n"
case shared.NFS != "":
b += " # /shared on every Slurm machine, mounted from the site's export.\n" +
" shared:\n path: " + yamlString(shared.Path) + "\n nfs: " + yamlString(shared.NFS) + "\n"
}
if users {
b += " # A Linux user for each llmgw account, the same UID everywhere; SSH keys\n" +
" # in the portal's Profile. Empty leaves users to the site (sssd, a directory).\n" +
" users: llmgw\n"
}
return b
}
// localNodeID is the node id this machine's agent takes: its hostname.
@@ -501,6 +536,12 @@ func (in *serverInit) initFlags() []string {
}
if in.slurm {
add("--slurm")
if in.slurmShared != "" && in.slurmShared != "here" {
add("--slurm-shared", in.slurmShared)
}
if in.slurmUsers == "site" {
add("--slurm-users", "site")
}
}
if in.evictAfterIdle > 0 {
add("--evict-after-idle", durationYAML(in.evictAfterIdle))
+3 -1
View File
@@ -17,7 +17,7 @@ import (
func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) {
in := &serverInit{
language: "tr", signupOpen: true, defaultRPM: 120, defaultTPM: 50000, maxKeyTTL: 30 * 24 * time.Hour,
media: true, slurm: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second,
media: true, slurm: true, slurmShared: "here", slurmUsers: "llmgw", evictAfterIdle: time.Minute, minLifetime: 90 * time.Second,
judge: judgeSpec{Fail: "open", Categories: []string{"Violent", "PII"}, Replicas: 3, Budget: 3 * time.Second},
}
j := in.judgeFor()
@@ -43,6 +43,8 @@ func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) {
t.Errorf("pictures and 3D: presets %v comfyui %q", cfg.Presets.Auto, cfg.Engines["comfyui"])
case !cfg.Slurm.Enabled || !cfg.Slurm.AllNodes || cfg.Slurm.Controller != localNodeID():
t.Errorf("slurm %+v", cfg.Slurm)
case cfg.Slurm.Shared.Path != "/shared" || cfg.Slurm.Shared.Server != localNodeID() || cfg.Slurm.Users != "llmgw":
t.Errorf("slurm's shared files and users %+v", cfg.Slurm)
case cfg.OnDemand.EvictAfterIdle != time.Minute || cfg.OnDemand.MinLifetime != 90*time.Second:
t.Errorf("on demand %+v", cfg.OnDemand)
}
+6 -2
View File
@@ -292,9 +292,13 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
"cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" +
"Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " +
"its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " +
"are submitted on this machine with sbatch and srun.",
"are submitted on this machine with sbatch and srun.\n\n" +
"Every Slurm machine sees /shared, exported over NFS from this machine (homes, datasets, the jobs' " +
"output); 'llmgw ctl slurm shared' points it at the site's own storage instead. Each llmgw account " +
"is a Linux user there, the same UID on every machine, logging in with the SSH keys given in the " +
"portal; 'llmgw ctl slurm users off' leaves users to the site's directory.",
off: "no Slurm; 'llmgw ctl slurm setup' adds it later",
summary: func() string { return "slurmctld here, slurmd on every node; none lent" },
summary: func() string { return "slurmctld here, slurmd on every node, /shared from here; none lent" },
needs: func() string {
if !f.node.on {
return "slurmctld runs on this machine's node: turn the node on, or Slurm off"
+20 -3
View File
@@ -44,6 +44,16 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
}
head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl)
head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
if st.Shared.Path == "" {
head += "\n shared files: " + sWarn.Render("none") + sDim.Render(" (llmgw ctl slurm shared; output stays on the job's node)")
} else {
head += "\n shared files: " + sharedLine(st.Shared)
}
if st.Users != "" {
head += "\n " + fmt.Sprintf("users: %d from llmgw's accounts", st.UserCount) + sDim.Render(", SSH keys in the portal's Profile")
} else {
head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)")
}
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
if len(st.Waiting) > 0 {
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
@@ -56,13 +66,20 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) {
for _, n := range st.Nodes {
rows = append(rows, slurmRow(n))
}
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "SHARED", "NOTE"}, rows, m.row, pw, m.inPanel), keys
}
func slurmRow(n slurmsvc.NodeStatus) []string {
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe()
if r := n.Status; r != nil {
state = or(r.State, "-")
switch r.Shared {
case "":
case "missing":
shared = sBad.Render(r.Shared)
default:
shared = sOK.Render(r.Shared)
}
jobs = fmt.Sprint(r.Jobs)
slurmd = sBad.Render("stopped")
if r.SlurmdRunning {
@@ -84,7 +101,7 @@ func slurmRow(n slurmsvc.NodeStatus) []string {
case slurmsvc.Lending, slurmsvc.Reclaiming:
whose = sWarn.Render(whose)
}
return []string{n.Node, whose, state, jobs, slurmd, note}
return []string{n.Node, whose, state, jobs, slurmd, shared, note}
}
// slurmMenu is what can be done with one of Slurm's nodes.
+5 -3
View File
@@ -11,6 +11,7 @@ import (
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
)
@@ -37,16 +38,17 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, cSlurm, true
m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"},
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"},
Shared: config.SharedStorage{Path: "/shared", Server: "node06"}, Users: "llmgw", UserCount: 2,
Nodes: []slurmsvc.NodeStatus{
{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true, Shared: "mounted"}},
{Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true,
Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}},
}}
var tm tea.Model = m
view := tm.View()
for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "/shared, exported over NFS by node06", "users: 2", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
if !strings.Contains(view, want) {
t.Errorf("the Slurm view has no %q:\n%s", want, view)
}