diff --git a/BACKLOG.md b/BACKLOG.md index 0755f29..2bfc274 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2047,6 +2047,11 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5, a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are taken to be nvidia-smi's index. +- In the first install too (2026-11-11): `llmgw server init --slurm`, and a Slurm part on the choices + screen, write `slurm: {enabled, controller: , all_nodes: true}` into server.yaml. + `slurm.all_nodes` (also `ctl slurm setup --all-nodes`) runs slurmd on every node in the leader's + inventory, those that join later too; none is lent. Refused with `--no-node`, or a hostname Slurm does + not take. ## D. Facts worth knowing before starting diff --git a/docs/install.md b/docs/install.md index 228c5ba..dd14d2b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -166,13 +166,26 @@ is a flag too, and the plan shows the command with them: `--starter qwen3-8b` `--judge-fail open`, `--judge-categories`, `--judge-replicas`, `--judge-budget`, `--language tr`, `--signup-open`, `--default-rpm`, `--default-tpm`, `--max-key-ttl`, `--media` (pictures and 3D: ComfyUI on the -nodes, and `presets.auto`), `--evict-after-idle` and `--min-lifetime`. They +nodes, and `presets.auto`), `--slurm` (see below), `--evict-after-idle` and `--min-lifetime`. They go into `server.yaml` as init writes it, and become the cluster's first settings when it starts; a `server.yaml` already there is never overwritten, so run again they change nothing — the settings page does. An NVIDIA card with no working driver is reported rather than silently ignored; install the driver and run init again to serve on it. +**Slurm, for training and batch jobs.** `--slurm`, or the Slurm part on the +choices screen, runs a Slurm cluster beside llmgw: slurmctld on this +machine's node, slurmd on every node, those that join later too +(`slurm.all_nodes`). The agents install munge and Slurm (22.05 or newer) +from the machines' own package sources. No node is Slurm's at first: each is +drained in Slurm and serves models until `llmgw ctl slurm lend NODE` gives it +to Slurm — its models move off first — and `llmgw ctl slurm reclaim NODE` +takes it back once its jobs end. Jobs are submitted on this machine with +`sbatch` and `srun`; `llmgw ctl slurm status` shows where each node is. It +needs this machine to be a node, and its hostname to be a name Slurm takes. +On a cluster installed without it, `llmgw ctl slurm setup --controller NODE +--all-nodes` does the same later. + **Secure by default.** What init writes is the hardened configuration, not a demonstration one: diff --git a/docs/reference.md b/docs/reference.md index d582b95..b6a0de5 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -406,6 +406,7 @@ rather than repeated. | `cluster_name` | `string` | [SlurmSettings](#slurmsettings) | | `controller` | `string` | [SlurmSettings](#slurmsettings) | | `nodes` | `[]string` | [SlurmSettings](#slurmsettings) | +| `all_nodes` | `bool` | [SlurmSettings](#slurmsettings) | | `lent` | `[]string` | [SlurmSettings](#slurmsettings) | | `partition` | `string` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | @@ -3525,6 +3526,13 @@ GPU, and need not be one of Nodes. Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained in Slurm, until it is also in Lent. +### `all_nodes` + +`bool` + +AllNodes runs slurmd on every llmgw node, those that join later too, +besides any named in Nodes. Each is llmgw's until it is lent, as any. + ### `lent` `[]string` diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index 79ac4ec..aab49f2 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -91,17 +91,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { var controller, cluster, partition, note string var nodes []string + var all bool c := &cobra.Command{ - Use: "setup --controller NODE --nodes NODE,...", + Use: "setup --controller NODE --nodes NODE,... | --all-nodes", Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named", - Long: "Turns slurm.enabled on and names the nodes. Their agents install munge and Slurm (22.05 or\n" + + Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" + "newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" + "llmgw's until `llmgw ctl slurm lend` gives it to Slurm.", - Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02", - Args: cobra.NoArgs, + Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" + + " llmgw ctl slurm setup --controller node06 --all-nodes", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { - if controller == "" || len(nodes) == 0 { - return fmt.Errorf("give --controller and --nodes") + if controller == "" || (len(nodes) == 0 && !all) { + return fmt.Errorf("give --controller, and --nodes or --all-nodes") } return changeSettings(cmd, client, note, func(doc *settings.Doc) error { if err := doc.SetYAML([]string{"slurm", "enabled"}, "true"); err != nil { @@ -110,12 +112,15 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { if err := doc.SetYAML([]string{"slurm", "controller"}, yamlString(controller)); err != nil { return err } - if err := doc.SetYAML([]string{"slurm", "nodes"}, yamlList(nodes)); err != nil { + if err := setList(doc, []string{"slurm", "nodes"}, nodes); err != nil { + return err + } + if err := doc.SetYAML([]string{"slurm", "all_nodes"}, fmt.Sprint(all)); err != nil { return err } // Lent nodes no longer named are not lent. lent := slices.DeleteFunc(docList(doc, []string{"slurm", "lent"}), - func(n string) bool { return !slices.Contains(nodes, n) }) + func(n string) bool { return !all && !slices.Contains(nodes, n) }) if err := setList(doc, []string{"slurm", "lent"}, lent); err != nil { return err } @@ -132,6 +137,7 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { } c.Flags().StringVar(&controller, "controller", "", "the node that runs slurmctld") c.Flags().StringSliceVar(&nodes, "nodes", nil, "the nodes that run slurmd") + c.Flags().BoolVar(&all, "all-nodes", false, "every node runs slurmd, those that join later too") c.Flags().StringVar(&cluster, "cluster-name", "", "Slurm's ClusterName (llmgw)") c.Flags().StringVar(&partition, "partition", "", "the partition every node is in (gpu)") c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") @@ -173,10 +179,11 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob RunE: func(cmd *cobra.Command, args []string) error { return changeSettings(cmd, client, note, func(doc *settings.Doc) error { nodes := docList(doc, []string{"slurm", "nodes"}) + all := docBool(doc, []string{"slurm", "all_nodes"}) lent := docList(doc, []string{"slurm", "lent"}) for _, n := range args { switch { - case add && !slices.Contains(nodes, n): + case add && !all && !slices.Contains(nodes, n): return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) case add && !slices.Contains(lent, n): lent = append(lent, n) @@ -205,6 +212,15 @@ func docList(doc *settings.Doc, path []string) []string { return out } +// docBool reads a boolean from the settings, false when unset. +func docBool(doc *settings.Doc, path []string) bool { + var b bool + if n := doc.Node(path); n != nil { + _ = n.Decode(&b) + } + return b +} + func setList(doc *settings.Doc, path []string, list []string) error { if len(list) == 0 { if doc.Node(path) != nil { diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 0286442..bb1da4d 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -124,6 +124,9 @@ type serverInit struct { // media has the nodes install ComfyUI and the leader add a picture model // and a 3D model once a card fits one. media bool + // slurm runs Slurm beside llmgw: slurmctld on this machine's node, + // slurmd on every node, each llmgw's until it is lent. + slurm bool // evictAfterIdle and minLifetime are on_demand's; zero leaves its own. evictAfterIdle, minLifetime time.Duration // standbys makes each machine of the replicated store a control plane @@ -258,6 +261,8 @@ func newServerInitCmd() *cobra.Command { f.DurationVar(&in.maxKeyTTL, "max-key-ttl", defaultMaxKeyTTL, "the longest an API key lives") f.BoolVar(&in.media, "media", false, "pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one") + f.BoolVar(&in.slurm, "slurm", false, + "Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0, "how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)") f.DurationVar(&in.minLifetime, "min-lifetime", 0, diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index 801d593..bc0ef9f 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -13,6 +13,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/preflight" ) @@ -132,9 +133,18 @@ func (in *serverInit) checkChoices() error { return fmt.Errorf("max key ttl %s: an hour at least", in.maxKeyTTL) case in.evictAfterIdle < 0 || in.minLifetime < 0: return fmt.Errorf("on-demand times cannot be negative") + case in.slurm && in.noNode: + return fmt.Errorf("slurm runs slurmctld on this machine's node: it needs the node (no --no-node)") case in.vip != "" && !in.standbys: return fmt.Errorf("a floating address moves between control planes: it needs the standbys") } + if in.slurm { + // The controller is this machine's node, named by its hostname: a + // name Slurm does not take would keep the server from starting. + if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil { + return fmt.Errorf("slurm: this machine's hostname: %w", err) + } + } if in.vip != "" { ip, _, err := net.ParseCIDR(in.vip) if err != nil || ip.To4() == nil { @@ -318,6 +328,9 @@ func (in *serverInit) tuneConfig(body string) string { "# one ('llmgw ctl model presets auto off' stops it).\n"+ "engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n") } + if in.slurm { + extra = append(extra, slurmSection(localNodeID())) + } if in.evictAfterIdle > 0 || in.minLifetime > 0 { b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n" if in.evictAfterIdle > 0 { @@ -334,6 +347,24 @@ func (in *serverInit) tuneConfig(body string) string { return body } +// slurmSection is server.yaml's slurm block as an install chooses it: +// slurmctld on this machine's node, slurmd on every node. +func slurmSection(controller string) string { + return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" + + "# every node, those that join later too. Each node is llmgw's, drained in\n" + + "# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" + + "# reclaim NODE' takes it back once its jobs end.\n" + + "slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n" +} + +// localNodeID is the node id this machine's agent takes: its hostname. +func localNodeID() string { + if h, err := os.Hostname(); err == nil && h != "" { + return h + } + return "localhost" +} + // durationYAML is a duration as a person writes it: 2h, 90m, 1m30s. func durationYAML(d time.Duration) string { switch { @@ -467,6 +498,9 @@ func (in *serverInit) initFlags() []string { if in.media { add("--media") } + if in.slurm { + add("--slurm") + } if in.evictAfterIdle > 0 { add("--evict-after-idle", durationYAML(in.evictAfterIdle)) } diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index b6ca6ec..4f0b154 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -17,7 +17,7 @@ import ( func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { in := &serverInit{ language: "tr", signupOpen: true, defaultRPM: 120, defaultTPM: 50000, maxKeyTTL: 30 * 24 * time.Hour, - media: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, + media: true, slurm: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, judge: judgeSpec{Fail: "open", Categories: []string{"Violent", "PII"}, Replicas: 3, Budget: 3 * time.Second}, } j := in.judgeFor() @@ -41,6 +41,8 @@ func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { t.Errorf("limits %d %d %s", cfg.Auth.DefaultRPM, cfg.Auth.DefaultTPM, cfg.Auth.MaxKeyTTL) case !cfg.Presets.Auto || cfg.Engines["comfyui"] == "": t.Errorf("pictures and 3D: presets %v comfyui %q", cfg.Presets.Auto, cfg.Engines["comfyui"]) + case !cfg.Slurm.Enabled || !cfg.Slurm.AllNodes || cfg.Slurm.Controller != localNodeID(): + t.Errorf("slurm %+v", cfg.Slurm) case cfg.OnDemand.EvictAfterIdle != time.Minute || cfg.OnDemand.MinLifetime != 90*time.Second: t.Errorf("on demand %+v", cfg.OnDemand) } @@ -95,14 +97,14 @@ func TestAStarterIsChosenByNameOrByItsFile(t *testing.T) { func TestTheChoicesAreTheirFlagsAgain(t *testing.T) { in := &serverInit{admin: "ops@example.com", hosts: []string{"llm.example.com"}, postgres: true, - mirrorPath: "/data/models", noNode: true, language: "tr", media: true, + mirrorPath: "/data/models", language: "tr", media: true, slurm: true, webhooks: []string{"https://hooks.slack.com/services/T/B/x, json:https://a.example.com/in"}, judge: judgeSpec{Categories: []string{"Violent", "Jailbreak"}}, maxKeyTTL: 7 * 24 * time.Hour} in.starter, _ = starterFrom("qwen3-4b") got := strings.Join(in.initFlags(), " ") want := "--admin ops@example.com --host llm.example.com --language tr --max-key-ttl 168h --postgres " + - "--mirror-dir /data/models --no-node --starter qwen3-4b --judge-categories Violent,Jailbreak " + - "--webhook https://hooks.slack.com/services/T/B/x --webhook json:https://a.example.com/in --media" + "--mirror-dir /data/models --starter qwen3-4b --judge-categories Violent,Jailbreak " + + "--webhook https://hooks.slack.com/services/T/B/x --webhook json:https://a.example.com/in --media --slurm" if got != want { t.Errorf("flags\n got %s\nwant %s", got, want) } @@ -124,6 +126,7 @@ func TestChoicesThatCannotBeInstalledAreRefused(t *testing.T) { {maxKeyTTL: time.Minute}, {vip: "10.0.0.160/24"}, {standbys: true, vip: "10.0.0.160"}, + {slurm: true, noNode: true}, } { if err := in.checkChoices(); err == nil { t.Errorf("taken: %+v", in) diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index e1db2c9..ead679d 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -57,7 +57,7 @@ type installForm struct { standby *installPart vipF *setupField - control, store, mirror, node, starter, media, judge, firewall, alerts, limits, turns, bundle *installPart + control, store, mirror, node, starter, media, slurm, judge, firewall, alerts, limits, turns, bundle *installPart admin, hosts, language, signup *setupField storeKind *setupField @@ -282,6 +282,25 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta summary: func() string { return "ComfyUI; a picture model and a 3D model once a card fits" }, } + // --- Slurm --- + f.slurm = &installPart{ + title: "Slurm", on: in.slurm, + about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " + + "slurmd on every node, those that join later too. The agents install munge and Slurm from the " + + "machines' own package sources.\n\n" + + "Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " + + "its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " + + "are submitted on this machine with sbatch and srun.", + off: "no Slurm; 'llmgw ctl slurm setup' adds it later", + summary: func() string { return "slurmctld here, slurmd on every node; none lent" }, + needs: func() string { + if !f.node.on { + return "slurmctld runs on this machine's node: turn the node on, or Slurm off" + } + return "" + }, + } + // --- the guardrails' judge --- j := in.judgeFor() f.judgeFail = choiceField("When no judge answers in time", []string{"refuse", "serve flagged"}, @@ -421,7 +440,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta if ha { f.parts = append(f.parts, f.standby) } - f.parts = append(f.parts, f.mirror, f.node, f.starter, f.media, f.judge, f.firewall, + f.parts = append(f.parts, f.mirror, f.node, f.starter, f.media, f.slurm, f.judge, f.firewall, f.alerts, f.limits, f.turns, f.bundle) return f } @@ -653,6 +672,7 @@ func (f *installForm) apply(in *serverInit) error { in.noModel = !f.starter.on in.starter, in.starterCtx = starter, ctx in.media = f.media.on + in.slurm = f.slurm.on in.noGuardModel = !f.judge.on in.judge = judge in.noFirewall = !f.firewall.on diff --git a/internal/config/slurm.go b/internal/config/slurm.go index 938ba56..e1b946e 100644 --- a/internal/config/slurm.go +++ b/internal/config/slurm.go @@ -26,6 +26,10 @@ type SlurmSettings struct { // in Slurm, until it is also in Lent. Nodes []string `koanf:"nodes"` + // AllNodes runs slurmd on every llmgw node, those that join later too, + // besides any named in Nodes. Each is llmgw's until it is lent, as any. + AllNodes bool `koanf:"all_nodes"` + // Lent are the nodes of Nodes Slurm may run jobs on now. A node put here // is drained of llmgw's replicas first, as `ctl drain` does, and resumed // in Slurm once none is left; a node taken out is drained in Slurm and @@ -57,7 +61,23 @@ func (s SlurmSettings) PartitionName() string { // IsLent is whether Slurm may use the node now. func (s SlurmSettings) IsLent(node string) bool { - return s.Enabled && slices.Contains(s.Nodes, node) && slices.Contains(s.Lent, node) + return s.Enabled && (s.AllNodes || slices.Contains(s.Nodes, node)) && slices.Contains(s.Lent, node) +} + +// WithNodes is the settings with every node of ids in Nodes when AllNodes: +// the nodes that run slurmd, as the leader's inventory has them. +func (s SlurmSettings) WithNodes(ids []string) SlurmSettings { + if !s.AllNodes { + return s + } + nodes := slices.Clone(s.Nodes) + for _, id := range ids { + if !slices.Contains(nodes, id) { + nodes = append(nodes, id) + } + } + s.Nodes = nodes + return s } // Check refuses settings Slurm could not run on. @@ -87,7 +107,7 @@ func (s SlurmSettings) Check() error { seen[n] = true } for i, n := range s.Lent { - if !seen[n] { + if !seen[n] && !s.AllNodes { return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n) } } diff --git a/internal/config/slurm_test.go b/internal/config/slurm_test.go index c4fdf9f..78dbeb8 100644 --- a/internal/config/slurm_test.go +++ b/internal/config/slurm_test.go @@ -34,3 +34,20 @@ func TestSlurmCheck(t *testing.T) { t.Error("a node is lent with Slurm off") } } + +func TestSlurmAllNodes(t *testing.T) { + s := SlurmSettings{Enabled: true, Controller: "c", AllNodes: true, Lent: []string{"joined"}} + if err := s.Check(); err != nil { + t.Fatalf("a node lent with all_nodes refused: %v", err) + } + if !s.IsLent("joined") { + t.Fatal("not lent with all_nodes") + } + if got := s.WithNodes([]string{"a", "joined"}).Nodes; len(got) != 2 { + t.Fatalf("nodes %v", got) + } + s.AllNodes = false + if s.Check() == nil || len(s.WithNodes([]string{"a"}).Nodes) != 0 { + t.Fatal("all_nodes off still takes the inventory's nodes") + } +} diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 2245c76..a6321be 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -41,6 +41,7 @@ type Inventory interface { Get(id string) (inventory.Node, bool) SetState(id string, st inventory.NodeState) bool Allocations(nodeID string) []inventory.Allocation + Nodes() []inventory.Node } // Phase is where a node of slurm.nodes is between llmgw and Slurm. @@ -128,6 +129,20 @@ func New(settings func() config.SlurmSettings, inv Inventory, agents Agents, kick: make(chan struct{}, 1)} } +// current is the settings, slurm.all_nodes resolved to the nodes the +// inventory has. +func (s *Service) current() config.SlurmSettings { + set := s.settings() + if !set.AllNodes { + return set + } + var ids []string + for _, n := range s.inv.Nodes() { + ids = append(ids, n.ID) + } + return set.WithNodes(ids) +} + // Every is how often the service looks again without being asked. var Every = 10 * time.Second @@ -182,7 +197,7 @@ func (s *Service) OnRegister(node string) { // Tick does one pass: the nodes handed over as far as they can be, and // each told what it is. func (s *Service) Tick(ctx context.Context) { - set := s.settings() + set := s.current() s.mu.Lock() defer s.mu.Unlock() phases := s.handOver(set) @@ -361,7 +376,7 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key // Status is the cluster as the service sees it now. func (s *Service) Status() Status { - set := s.settings() + set := s.current() s.mu.Lock() defer s.mu.Unlock() out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, diff --git a/internal/control/slurmsvc/service_test.go b/internal/control/slurmsvc/service_test.go index 66d2753..e802004 100644 --- a/internal/control/slurmsvc/service_test.go +++ b/internal/control/slurmsvc/service_test.go @@ -2,6 +2,7 @@ package slurmsvc import ( "context" + "slices" "strings" "testing" @@ -36,6 +37,15 @@ func (f *fakeInv) Allocations(id string) []inventory.Allocation { return make([]inventory.Allocation, f.allocs[id]) } +func (f *fakeInv) Nodes() []inventory.Node { + var out []inventory.Node + for _, n := range f.nodes { + out = append(out, *n) + } + slices.SortFunc(out, func(a, b inventory.Node) int { return strings.Compare(a.ID, b.ID) }) + return out +} + type fakeAgents struct { conn []string got map[string][]*llmgwv1.Slurm @@ -241,3 +251,24 @@ func TestNoControllerYet(t *testing.T) { t.Fatalf("a node was started with no controller to fetch its files from: %v", m) } } + +func TestAllNodes(t *testing.T) { + r := newRig() + r.set.Nodes, r.set.AllNodes = nil, true + r.inv.nodes["cuda02"] = &inventory.Node{ID: "cuda02", MgmtAddr: "10.0.0.150"} + r.agents.conn = append(r.agents.conn, "cuda02") + r.tick() + for _, n := range []string{"cuda01", "cuda02", "node06"} { + if !r.agents.last(n).GetNode() { + t.Fatalf("%s, a node that joined, does not run slurmd", n) + } + } + if got := len(r.svc.Status().Nodes); got != 3 { + t.Fatalf("%d nodes in the status", got) + } + r.set.Lent = []string{"cuda02"} + r.tick() + if p := r.svc.Status().Nodes[1]; p.Node != "cuda02" || p.Phase != Lent { + t.Fatalf("cuda02 not lent: %+v", p) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 0e8abe5..77d7366 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -518,6 +518,7 @@ var Reference = []Block{ {Key: "cluster_name", Type: "string", Doc: "ClusterName is Slurm's ClusterName. Empty is \"llmgw\".", Nested: ""}, {Key: "controller", Type: "string", Doc: "Controller is the llmgw node whose agent runs slurmctld. It needs no\nGPU, and need not be one of Nodes.", Nested: ""}, {Key: "nodes", Type: "[]string", Doc: "Nodes are the llmgw nodes that run slurmd. Each stays llmgw's, drained\nin Slurm, until it is also in Lent.", Nested: ""}, + {Key: "all_nodes", Type: "bool", Doc: "AllNodes runs slurmd on every llmgw node, those that join later too,\nbesides any named in Nodes. Each is llmgw's until it is lent, as any.", Nested: ""}, {Key: "lent", Type: "[]string", Doc: "Lent are the nodes of Nodes Slurm may run jobs on now. A node put here\nis drained of llmgw's replicas first, as `ctl drain` does, and resumed\nin Slurm once none is left; a node taken out is drained in Slurm and\ngiven back to llmgw when its last job ends.", Nested: ""}, {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, }},