Slurm in the first install: server init --slurm, and slurm.all_nodes
CI / test (push) Successful in 9m30s
CI / test (push) Successful in 9m30s
The choices screen has a Slurm part, and --slurm writes slurmctld on this machine's node and slurmd on every node, those that join later too, into server.yaml. slurm.all_nodes (ctl slurm setup --all-nodes) takes the nodes from the leader's inventory. No node is lent at install. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b15944aa1f
commit
914985bf9f
13 files changed
+208
-20
No files matched your search
@@ -91,17 +91,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
|
||||
var controller, cluster, partition, note string
|
||||
var nodes []string
|
||||
var all bool
|
||||
c := &cobra.Command{
|
||||
Use: "setup --controller NODE --nodes NODE,...",
|
||||
Use: "setup --controller NODE --nodes NODE,... | --all-nodes",
|
||||
Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named",
|
||||
Long: "Turns slurm.enabled on and names the nodes. Their agents install munge and Slurm (22.05 or\n" +
|
||||
Long: "Turns slurm.enabled on and names the nodes, or with --all-nodes every node. Their agents install munge and Slurm (22.05 or\n" +
|
||||
"newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" +
|
||||
"llmgw's until `llmgw ctl slurm lend` gives it to Slurm.",
|
||||
Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02",
|
||||
Args: cobra.NoArgs,
|
||||
Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02\n" +
|
||||
" llmgw ctl slurm setup --controller node06 --all-nodes",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if controller == "" || len(nodes) == 0 {
|
||||
return fmt.Errorf("give --controller and --nodes")
|
||||
if controller == "" || (len(nodes) == 0 && !all) {
|
||||
return fmt.Errorf("give --controller, and --nodes or --all-nodes")
|
||||
}
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
if err := doc.SetYAML([]string{"slurm", "enabled"}, "true"); err != nil {
|
||||
@@ -110,12 +112,15 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := doc.SetYAML([]string{"slurm", "controller"}, yamlString(controller)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := doc.SetYAML([]string{"slurm", "nodes"}, yamlList(nodes)); err != nil {
|
||||
if err := setList(doc, []string{"slurm", "nodes"}, nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := doc.SetYAML([]string{"slurm", "all_nodes"}, fmt.Sprint(all)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Lent nodes no longer named are not lent.
|
||||
lent := slices.DeleteFunc(docList(doc, []string{"slurm", "lent"}),
|
||||
func(n string) bool { return !slices.Contains(nodes, n) })
|
||||
func(n string) bool { return !all && !slices.Contains(nodes, n) })
|
||||
if err := setList(doc, []string{"slurm", "lent"}, lent); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,6 +137,7 @@ func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
c.Flags().StringVar(&controller, "controller", "", "the node that runs slurmctld")
|
||||
c.Flags().StringSliceVar(&nodes, "nodes", nil, "the nodes that run slurmd")
|
||||
c.Flags().BoolVar(&all, "all-nodes", false, "every node runs slurmd, those that join later too")
|
||||
c.Flags().StringVar(&cluster, "cluster-name", "", "Slurm's ClusterName (llmgw)")
|
||||
c.Flags().StringVar(&partition, "partition", "", "the partition every node is in (gpu)")
|
||||
c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version")
|
||||
@@ -173,10 +179,11 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
nodes := docList(doc, []string{"slurm", "nodes"})
|
||||
all := docBool(doc, []string{"slurm", "all_nodes"})
|
||||
lent := docList(doc, []string{"slurm", "lent"})
|
||||
for _, n := range args {
|
||||
switch {
|
||||
case add && !slices.Contains(nodes, n):
|
||||
case add && !all && !slices.Contains(nodes, n):
|
||||
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
|
||||
case add && !slices.Contains(lent, n):
|
||||
lent = append(lent, n)
|
||||
@@ -205,6 +212,15 @@ func docList(doc *settings.Doc, path []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// docBool reads a boolean from the settings, false when unset.
|
||||
func docBool(doc *settings.Doc, path []string) bool {
|
||||
var b bool
|
||||
if n := doc.Node(path); n != nil {
|
||||
_ = n.Decode(&b)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func setList(doc *settings.Doc, path []string, list []string) error {
|
||||
if len(list) == 0 {
|
||||
if doc.Node(path) != nil {
|
||||
|
||||
@@ -124,6 +124,9 @@ type serverInit struct {
|
||||
// media has the nodes install ComfyUI and the leader add a picture model
|
||||
// and a 3D model once a card fits one.
|
||||
media bool
|
||||
// slurm runs Slurm beside llmgw: slurmctld on this machine's node,
|
||||
// slurmd on every node, each llmgw's until it is lent.
|
||||
slurm bool
|
||||
// evictAfterIdle and minLifetime are on_demand's; zero leaves its own.
|
||||
evictAfterIdle, minLifetime time.Duration
|
||||
// standbys makes each machine of the replicated store a control plane
|
||||
@@ -258,6 +261,8 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.DurationVar(&in.maxKeyTTL, "max-key-ttl", defaultMaxKeyTTL, "the longest an API key lives")
|
||||
f.BoolVar(&in.media, "media", false,
|
||||
"pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one")
|
||||
f.BoolVar(&in.slurm, "slurm", false,
|
||||
"Slurm for training and batch jobs: slurmctld on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'")
|
||||
f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0,
|
||||
"how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)")
|
||||
f.DurationVar(&in.minLifetime, "min-lifetime", 0,
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/preflight"
|
||||
)
|
||||
@@ -132,9 +133,18 @@ func (in *serverInit) checkChoices() error {
|
||||
return fmt.Errorf("max key ttl %s: an hour at least", in.maxKeyTTL)
|
||||
case in.evictAfterIdle < 0 || in.minLifetime < 0:
|
||||
return fmt.Errorf("on-demand times cannot be negative")
|
||||
case in.slurm && in.noNode:
|
||||
return fmt.Errorf("slurm runs slurmctld on this machine's node: it needs the node (no --no-node)")
|
||||
case in.vip != "" && !in.standbys:
|
||||
return fmt.Errorf("a floating address moves between control planes: it needs the standbys")
|
||||
}
|
||||
if in.slurm {
|
||||
// The controller is this machine's node, named by its hostname: a
|
||||
// name Slurm does not take would keep the server from starting.
|
||||
if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil {
|
||||
return fmt.Errorf("slurm: this machine's hostname: %w", err)
|
||||
}
|
||||
}
|
||||
if in.vip != "" {
|
||||
ip, _, err := net.ParseCIDR(in.vip)
|
||||
if err != nil || ip.To4() == nil {
|
||||
@@ -318,6 +328,9 @@ func (in *serverInit) tuneConfig(body string) string {
|
||||
"# one ('llmgw ctl model presets auto off' stops it).\n"+
|
||||
"engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n")
|
||||
}
|
||||
if in.slurm {
|
||||
extra = append(extra, slurmSection(localNodeID()))
|
||||
}
|
||||
if in.evictAfterIdle > 0 || in.minLifetime > 0 {
|
||||
b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n"
|
||||
if in.evictAfterIdle > 0 {
|
||||
@@ -334,6 +347,24 @@ func (in *serverInit) tuneConfig(body string) string {
|
||||
return body
|
||||
}
|
||||
|
||||
// slurmSection is server.yaml's slurm block as an install chooses it:
|
||||
// slurmctld on this machine's node, slurmd on every node.
|
||||
func slurmSection(controller string) string {
|
||||
return "# Slurm beside llmgw, chosen at install: slurmctld on this machine, slurmd on\n" +
|
||||
"# every node, those that join later too. Each node is llmgw's, drained in\n" +
|
||||
"# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" +
|
||||
"# reclaim NODE' takes it back once its jobs end.\n" +
|
||||
"slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n"
|
||||
}
|
||||
|
||||
// localNodeID is the node id this machine's agent takes: its hostname.
|
||||
func localNodeID() string {
|
||||
if h, err := os.Hostname(); err == nil && h != "" {
|
||||
return h
|
||||
}
|
||||
return "localhost"
|
||||
}
|
||||
|
||||
// durationYAML is a duration as a person writes it: 2h, 90m, 1m30s.
|
||||
func durationYAML(d time.Duration) string {
|
||||
switch {
|
||||
@@ -467,6 +498,9 @@ func (in *serverInit) initFlags() []string {
|
||||
if in.media {
|
||||
add("--media")
|
||||
}
|
||||
if in.slurm {
|
||||
add("--slurm")
|
||||
}
|
||||
if in.evictAfterIdle > 0 {
|
||||
add("--evict-after-idle", durationYAML(in.evictAfterIdle))
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) {
|
||||
in := &serverInit{
|
||||
language: "tr", signupOpen: true, defaultRPM: 120, defaultTPM: 50000, maxKeyTTL: 30 * 24 * time.Hour,
|
||||
media: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second,
|
||||
media: true, slurm: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second,
|
||||
judge: judgeSpec{Fail: "open", Categories: []string{"Violent", "PII"}, Replicas: 3, Budget: 3 * time.Second},
|
||||
}
|
||||
j := in.judgeFor()
|
||||
@@ -41,6 +41,8 @@ func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) {
|
||||
t.Errorf("limits %d %d %s", cfg.Auth.DefaultRPM, cfg.Auth.DefaultTPM, cfg.Auth.MaxKeyTTL)
|
||||
case !cfg.Presets.Auto || cfg.Engines["comfyui"] == "":
|
||||
t.Errorf("pictures and 3D: presets %v comfyui %q", cfg.Presets.Auto, cfg.Engines["comfyui"])
|
||||
case !cfg.Slurm.Enabled || !cfg.Slurm.AllNodes || cfg.Slurm.Controller != localNodeID():
|
||||
t.Errorf("slurm %+v", cfg.Slurm)
|
||||
case cfg.OnDemand.EvictAfterIdle != time.Minute || cfg.OnDemand.MinLifetime != 90*time.Second:
|
||||
t.Errorf("on demand %+v", cfg.OnDemand)
|
||||
}
|
||||
@@ -95,14 +97,14 @@ func TestAStarterIsChosenByNameOrByItsFile(t *testing.T) {
|
||||
|
||||
func TestTheChoicesAreTheirFlagsAgain(t *testing.T) {
|
||||
in := &serverInit{admin: "ops@example.com", hosts: []string{"llm.example.com"}, postgres: true,
|
||||
mirrorPath: "/data/models", noNode: true, language: "tr", media: true,
|
||||
mirrorPath: "/data/models", language: "tr", media: true, slurm: true,
|
||||
webhooks: []string{"https://hooks.slack.com/services/T/B/x, json:https://a.example.com/in"},
|
||||
judge: judgeSpec{Categories: []string{"Violent", "Jailbreak"}}, maxKeyTTL: 7 * 24 * time.Hour}
|
||||
in.starter, _ = starterFrom("qwen3-4b")
|
||||
got := strings.Join(in.initFlags(), " ")
|
||||
want := "--admin ops@example.com --host llm.example.com --language tr --max-key-ttl 168h --postgres " +
|
||||
"--mirror-dir /data/models --no-node --starter qwen3-4b --judge-categories Violent,Jailbreak " +
|
||||
"--webhook https://hooks.slack.com/services/T/B/x --webhook json:https://a.example.com/in --media"
|
||||
"--mirror-dir /data/models --starter qwen3-4b --judge-categories Violent,Jailbreak " +
|
||||
"--webhook https://hooks.slack.com/services/T/B/x --webhook json:https://a.example.com/in --media --slurm"
|
||||
if got != want {
|
||||
t.Errorf("flags\n got %s\nwant %s", got, want)
|
||||
}
|
||||
@@ -124,6 +126,7 @@ func TestChoicesThatCannotBeInstalledAreRefused(t *testing.T) {
|
||||
{maxKeyTTL: time.Minute},
|
||||
{vip: "10.0.0.160/24"},
|
||||
{standbys: true, vip: "10.0.0.160"},
|
||||
{slurm: true, noNode: true},
|
||||
} {
|
||||
if err := in.checkChoices(); err == nil {
|
||||
t.Errorf("taken: %+v", in)
|
||||
|
||||
@@ -57,7 +57,7 @@ type installForm struct {
|
||||
standby *installPart
|
||||
vipF *setupField
|
||||
|
||||
control, store, mirror, node, starter, media, judge, firewall, alerts, limits, turns, bundle *installPart
|
||||
control, store, mirror, node, starter, media, slurm, judge, firewall, alerts, limits, turns, bundle *installPart
|
||||
|
||||
admin, hosts, language, signup *setupField
|
||||
storeKind *setupField
|
||||
@@ -282,6 +282,25 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
|
||||
summary: func() string { return "ComfyUI; a picture model and a 3D model once a card fits" },
|
||||
}
|
||||
|
||||
// --- Slurm ---
|
||||
f.slurm = &installPart{
|
||||
title: "Slurm", on: in.slurm,
|
||||
about: "A Slurm cluster beside llmgw, for training and other batch work: slurmctld on this machine, " +
|
||||
"slurmd on every node, those that join later too. The agents install munge and Slurm from the " +
|
||||
"machines' own package sources.\n\n" +
|
||||
"Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " +
|
||||
"its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " +
|
||||
"are submitted on this machine with sbatch and srun.",
|
||||
off: "no Slurm; 'llmgw ctl slurm setup' adds it later",
|
||||
summary: func() string { return "slurmctld here, slurmd on every node; none lent" },
|
||||
needs: func() string {
|
||||
if !f.node.on {
|
||||
return "slurmctld runs on this machine's node: turn the node on, or Slurm off"
|
||||
}
|
||||
return ""
|
||||
},
|
||||
}
|
||||
|
||||
// --- the guardrails' judge ---
|
||||
j := in.judgeFor()
|
||||
f.judgeFail = choiceField("When no judge answers in time", []string{"refuse", "serve flagged"},
|
||||
@@ -421,7 +440,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
|
||||
if ha {
|
||||
f.parts = append(f.parts, f.standby)
|
||||
}
|
||||
f.parts = append(f.parts, f.mirror, f.node, f.starter, f.media, f.judge, f.firewall,
|
||||
f.parts = append(f.parts, f.mirror, f.node, f.starter, f.media, f.slurm, f.judge, f.firewall,
|
||||
f.alerts, f.limits, f.turns, f.bundle)
|
||||
return f
|
||||
}
|
||||
@@ -653,6 +672,7 @@ func (f *installForm) apply(in *serverInit) error {
|
||||
in.noModel = !f.starter.on
|
||||
in.starter, in.starterCtx = starter, ctx
|
||||
in.media = f.media.on
|
||||
in.slurm = f.slurm.on
|
||||
in.noGuardModel = !f.judge.on
|
||||
in.judge = judge
|
||||
in.noFirewall = !f.firewall.on
|
||||
|
||||
Reference in new issue
Block a user