From 356f7e0ced0835691ec607fc688f973a3c5a89a0 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 11 Nov 2026 22:53:04 +0100 Subject: [PATCH] Slurm in the console: each node, and lend or take back from its row The console's Slurm section shows the controller, and each node that runs slurmd with whose it is now, its state in Slurm and its jobs. Enter on a row lends the node (its replicas move off first) or takes it back, by the same change to slurm.lent as ctl slurm lend and reclaim. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 3 +- docs/console.md | 10 +++ internal/cli/ctl_slurm.go | 33 ++++---- internal/cli/tui_console.go | 13 ++- internal/cli/tui_slurm.go | 148 +++++++++++++++++++++++++++++++++ internal/cli/tui_slurm_test.go | 92 ++++++++++++++++++++ 6 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 internal/cli/tui_slurm.go create mode 100644 internal/cli/tui_slurm_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 0567330..2c78d3a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2053,7 +2053,8 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t way: a node not heard from or with Slurm jobs is cordoned, not drained (nothing of llmgw's to move, and a new leader must not move models off every Slurm node); jobs are counted from the node's own slurmstepd, so a controller that is down neither hides them nor, with none, keeps the node from llmgw; a drain whose - reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5, + reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: ~~the TUI screen~~ + (done 2026-11-11: console **Slurm**, lend/take back on a row), P5, a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are taken to be nvidia-smi's index. - In the first install too (2026-11-11): `llmgw server init --slurm`, and a Slurm part on the choices diff --git a/docs/console.md b/docs/console.md index 4d7e116..cdd69a0 100644 --- a/docs/console.md +++ b/docs/console.md @@ -207,6 +207,16 @@ starting — and whether it ended done, failed, or waiting for room on busy cards. The entry shows how many are running. Enter on one shows its steps. `llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one. +**Slurm** is the Slurm cluster run beside llmgw, when there is one: its +controller and whether slurmctld runs, and each node that runs slurmd — +whose it is now (llmgw's, Slurm's, or on its way between them), its state in +Slurm, the jobs running there. On a row, **Lend to Slurm…** moves the node's +replicas off and then lets Slurm run jobs there; **Take back from Slurm…** +starts no new job there and gives it back to llmgw when its jobs have ended. +Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend` +and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm +setup`, or the Slurm part of the first install. + **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index aab49f2..cc7e604 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -178,20 +178,7 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return changeSettings(cmd, client, note, func(doc *settings.Doc) error { - nodes := docList(doc, []string{"slurm", "nodes"}) - all := docBool(doc, []string{"slurm", "all_nodes"}) - lent := docList(doc, []string{"slurm", "lent"}) - for _, n := range args { - switch { - case add && !all && !slices.Contains(nodes, n): - return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) - case add && !slices.Contains(lent, n): - lent = append(lent, n) - case !add: - lent = slices.DeleteFunc(lent, func(l string) bool { return l == n }) - } - } - return setList(doc, []string{"slurm", "lent"}, lent) + return lendChange(doc, args, add) }) }, } @@ -199,6 +186,24 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob return c } +// lendChange adds nodes to slurm.lent, or with add false takes them out. +func lendChange(doc *settings.Doc, names []string, add bool) error { + nodes := docList(doc, []string{"slurm", "nodes"}) + all := docBool(doc, []string{"slurm", "all_nodes"}) + lent := docList(doc, []string{"slurm", "lent"}) + for _, n := range names { + switch { + case add && !all && !slices.Contains(nodes, n): + return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) + case add && !slices.Contains(lent, n): + lent = append(lent, n) + case !add: + lent = slices.DeleteFunc(lent, func(l string) bool { return l == n }) + } + } + return setList(doc, []string{"slurm", "lent"}, lent) +} + // docList reads a list of strings from the settings, empty when unset. func docList(doc *settings.Doc, path []string) []string { n := doc.Node(path) diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index a53f1d6..bf22da1 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -24,6 +24,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -50,6 +51,7 @@ const ( cFederation cMirror cJobs + cSlurm cLogs cAccounts cKeys @@ -71,7 +73,7 @@ const ( cCount ) -var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual", +var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Slurm", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual", "server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster", "Restart all replicas", "Back up now", "Wipe this machine", "Leave"} @@ -92,6 +94,8 @@ type consoleData struct { // federation is the other llmgw clusters whose models are offered // here; empty on a control plane without a store. federation adminapi.FederationView + // slurm is the Slurm cluster beside llmgw; off when it has none. + slurm slurmsvc.Status // presets is the models that come with llmgw as they stand here; nil // from a control plane that cannot say. presets *adminapi.PresetsView @@ -284,6 +288,7 @@ func (m consoleModel) fetch() tea.Cmd { _ = c.get(ctx, "/api/v1/jobs", &d.jobs) _ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit) _ = c.get(ctx, "/api/v1/federation", &d.federation) + _ = c.get(ctx, "/api/v1/slurm", &d.slurm) var presets adminapi.PresetsView if c.get(ctx, "/api/v1/presets", &presets) == nil { d.presets = &presets @@ -588,6 +593,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cFederation: m.dialog = "error" m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row]) + case cSlurm: + m.menuD = m.slurmMenu(m.data.slurm.Nodes[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: @@ -1110,6 +1117,8 @@ func (m consoleModel) rowCount() int { return len(m.data.audit) case cFederation: return len(m.data.federation.Peers) + case cSlurm: + return len(m.data.slurm.Nodes) case cServe: return len(m.hf.models) case cAccounts: @@ -1369,6 +1378,8 @@ func (m consoleModel) panel() (title, body, keys string) { return m.auditPanel(pw) case cFederation: return m.federationPanel(pw) + case cSlurm: + return m.slurmPanel(pw) case cMonitor: return m.monitorPanel(pw) case cServe: diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go new file mode 100644 index 0000000..11f42ae --- /dev/null +++ b/internal/cli/tui_slurm.go @@ -0,0 +1,148 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "time" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P), +// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows +// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm +// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm +// part: a form for it would be server.yaml written again. + +func (m consoleModel) slurmPanel(pw int) (string, string, string) { + keys := "→ or Enter open · ↑↓ choose · Q leave" + if m.inPanel { + keys = "↑↓ node · Enter lend or take back · ← back · Q leave" + } + st := m.data.slurm + head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.") + if !st.Enabled { + return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+ + "llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys + } + ctl := sDim.Render("not heard from yet") + switch c := st.Ctld; { + case c == nil: + case c.Error != "": + ctl = sBad.Render(c.Error) + case c.SlurmctldRunning: + ctl = sOK.Render("slurmctld " + c.Version + " running") + default: + ctl = sBad.Render("slurmctld not running") + } + head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl) + head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller)) + head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") + if len(st.Waiting) > 0 { + head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) + } + head += "\n\n" + if len(st.Nodes) == 0 { + return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys + } + var rows [][]string + for _, n := range st.Nodes { + rows = append(rows, slurmRow(n)) + } + return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys +} + +func slurmRow(n slurmsvc.NodeStatus) []string { + state, jobs, slurmd, note := "-", "-", "-", n.Describe() + if r := n.Status; r != nil { + state = or(r.State, "-") + jobs = fmt.Sprint(r.Jobs) + slurmd = sBad.Render("stopped") + if r.SlurmdRunning { + slurmd = sOK.Render("running") + } + if r.Error != "" { + note = sBad.Render(r.Error) + } + } + if !n.Connected { + note = sWarn.Render("agent not connected") + "; " + note + } + whose := string(n.Phase) + switch n.Phase { + case slurmsvc.Lent: + whose = sOK.Render("Slurm's") + case slurmsvc.Llmgw: + whose = "llmgw's" + case slurmsvc.Lending, slurmsvc.Reclaiming: + whose = sWarn.Render(whose) + } + return []string{n.Node, whose, state, jobs, slurmd, note} +} + +// slurmMenu is what can be done with one of Slurm's nodes. +func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog { + var items []menuItem + if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending { + items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Take " + n.Node + " back", + text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.", + yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }} + return nil + }}) + } else { + items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd { + text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there." + if n.Replicas > 0 { + text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+ + "a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas) + } + m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text, + yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }} + return nil + }}) + } + return &menuDialog{title: n.Node, items: items} +} + +// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm +// lend|reclaim` does: the newest settings, changed and put back. +func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd { + title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first." + if !add { + title, note, done = "Take back "+node, "taken back from Slurm in the console", + node+" is being taken back: llmgw has it once its Slurm jobs end." + } + m.busy = title + c := m.client + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var cur adminapi.SettingsView + if err := c.get(ctx, "/api/v1/settings", &cur); err != nil { + return actionDoneMsg{title: title, err: err} + } + doc, err := settings.FromBytes("settings", []byte(cur.Body)) + if err == nil { + err = lendChange(doc, []string{node}, add) + } + var body []byte + if err == nil { + body, err = doc.Bytes() + } + if err != nil { + return actionDoneMsg{title: title, err: err} + } + req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note}) + var v adminapi.SettingsView + if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil { + return actionDoneMsg{title: title, err: err} + } + return actionDoneMsg{title: title, text: done} + } +} diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go new file mode 100644 index 0000000..b6d03c2 --- /dev/null +++ b/internal/cli/tui_slurm_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" +) + +func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { + var mu sync.Mutex + var put adminapi.SettingsRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings": + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7, + Body: "slurm:\n enabled: true\n controller: node06\n nodes: [node05, cuda01]\n"}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings": + b, _ := io.ReadAll(r.Body) + mu.Lock() + _ = json.Unmarshal(b, &put) + mu.Unlock() + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8}) + default: + _, _ = w.Write([]byte("[]")) + } + })) + t.Cleanup(srv.Close) + m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") + m.loaded, m.menu, m.inPanel = true, cSlurm, true + m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"}, + Nodes: []slurmsvc.NodeStatus{ + {Node: "node05", Phase: slurmsvc.Lent, Connected: true, + Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, + {Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true, + Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}}, + }} + var tm tea.Model = m + view := tm.View() + for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + if !strings.Contains(view, want) { + t.Errorf("the Slurm view has no %q:\n%s", want, view) + } + } + + // cuda01, the second row: lent, after saying its replicas move. + tm = press(tm, "down", "enter") + if !strings.Contains(tm.View(), "Lend to Slurm") { + t.Fatalf("no lend in the menu:\n%s", tm.View()) + } + tm = press(tm, "enter") + if !strings.Contains(tm.View(), "3 replica(s) move off first") { + t.Fatalf("the question does not say what moves:\n%s", tm.View()) + } + tm = pressRun(tm, "y") + mu.Lock() + defer mu.Unlock() + if put.Base != 7 || !strings.Contains(put.Body, "lent:") || !strings.Contains(put.Body, "cuda01") { + t.Fatalf("the settings put back: %+v", put) + } + if !strings.Contains(tm.View(), "cuda01 is being lent to Slurm") { + t.Errorf("not said:\n%s", tm.View()) + } +} + +func TestTheConsoleTakesALentNodeBack(t *testing.T) { + m, _ := consoleWith(t, cSlurm) + m.data.slurm = slurmsvc.Status{Enabled: true, Controller: "node06", + Nodes: []slurmsvc.NodeStatus{{Node: "node05", Phase: slurmsvc.Lent, Connected: true}}} + var tm tea.Model = m + tm = press(tm, "enter") + if v := tm.View(); !strings.Contains(v, "Take back from Slurm") || strings.Contains(v, "Lend to Slurm") { + t.Fatalf("a lent node's menu:\n%s", v) + } +} + +func TestTheConsoleSaysHowToTurnSlurmOn(t *testing.T) { + m, _ := consoleWith(t, cSlurm) + var tm tea.Model = m + if !strings.Contains(tm.View(), "llmgw ctl slurm setup") { + t.Fatalf("Slurm off says nothing of turning it on:\n%s", tm.View()) + } +}