Slurm in the console: each node, and lend or take back from its row

The console's Slurm section shows the controller, and each node that runs
slurmd with whose it is now, its state in Slurm and its jobs. Enter on a
row lends the node (its replicas move off first) or takes it back, by the
same change to slurm.lent as ctl slurm lend and reclaim.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

BACKLOG.md

@@ -2053,7 +2053,8 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
   way: a node not heard from or with Slurm jobs is cordoned, not drained (nothing of llmgw's to move, and a
   new leader must not move models off every Slurm node); jobs are counted from the node's own slurmstepd,
   so a controller that is down neither hides them nor, with none, keeps the node from llmgw; a drain whose
-  reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5,
+  reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: ~~the TUI screen~~
+  (done 2026-11-11: console **Slurm**, lend/take back on a row), P5,
   a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are
   taken to be nvidia-smi's index.
 - In the first install too (2026-11-11): `llmgw server init --slurm`, and a Slurm part on the choices

docs/console.md

@@ -207,6 +207,16 @@ starting — and whether it ended done, failed, or waiting for room on busy
 cards. The entry shows how many are running. Enter on one shows its steps.
 `llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one.
 
+**Slurm** is the Slurm cluster run beside llmgw, when there is one: its
+controller and whether slurmctld runs, and each node that runs slurmd —
+whose it is now (llmgw's, Slurm's, or on its way between them), its state in
+Slurm, the jobs running there. On a row, **Lend to Slurm…** moves the node's
+replicas off and then lets Slurm run jobs there; **Take back from Slurm…**
+starts no new job there and gives it back to llmgw when its jobs have ended.
+Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend`
+and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm
+setup`, or the Slurm part of the first install.
+
 **Logs** shows a node's agent log, newest at the bottom, errors and warnings
 coloured.
 

internal/cli/ctl_slurm.go

@@ -178,20 +178,7 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
 		Args:  cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
-				nodes := docList(doc, []string{"slurm", "nodes"})
-				all := docBool(doc, []string{"slurm", "all_nodes"})
-				lent := docList(doc, []string{"slurm", "lent"})
-				for _, n := range args {
-					switch {
-					case add && !all && !slices.Contains(nodes, n):
-						return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
-					case add && !slices.Contains(lent, n):
-						lent = append(lent, n)
-					case !add:
-						lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
-					}
-				}
-				return setList(doc, []string{"slurm", "lent"}, lent)
+				return lendChange(doc, args, add)
 			})
 		},
 	}
@@ -199,6 +186,24 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
 	return c
 }
 
+// lendChange adds nodes to slurm.lent, or with add false takes them out.
+func lendChange(doc *settings.Doc, names []string, add bool) error {
+	nodes := docList(doc, []string{"slurm", "nodes"})
+	all := docBool(doc, []string{"slurm", "all_nodes"})
+	lent := docList(doc, []string{"slurm", "lent"})
+	for _, n := range names {
+		switch {
+		case add && !all && !slices.Contains(nodes, n):
+			return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
+		case add && !slices.Contains(lent, n):
+			lent = append(lent, n)
+		case !add:
+			lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
+		}
+	}
+	return setList(doc, []string{"slurm", "lent"}, lent)
+}
+
 // docList reads a list of strings from the settings, empty when unset.
 func docList(doc *settings.Doc, path []string) []string {
 	n := doc.Node(path)

internal/cli/tui_console.go

@@ -24,6 +24,7 @@ import (
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
@@ -50,6 +51,7 @@ const (
 	cFederation
 	cMirror
 	cJobs
+	cSlurm
 	cLogs
 	cAccounts
 	cKeys
@@ -71,7 +73,7 @@ const (
 	cCount
 )
 
-var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
+var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Slurm", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
 	"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster",
 	"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
 
@@ -92,6 +94,8 @@ type consoleData struct {
 	// federation is the other llmgw clusters whose models are offered
 	// here; empty on a control plane without a store.
 	federation adminapi.FederationView
+	// slurm is the Slurm cluster beside llmgw; off when it has none.
+	slurm slurmsvc.Status
 	// presets is the models that come with llmgw as they stand here; nil
 	// from a control plane that cannot say.
 	presets *adminapi.PresetsView
@@ -284,6 +288,7 @@ func (m consoleModel) fetch() tea.Cmd {
 		_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
 		_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
 		_ = c.get(ctx, "/api/v1/federation", &d.federation)
+		_ = c.get(ctx, "/api/v1/slurm", &d.slurm)
 		var presets adminapi.PresetsView
 		if c.get(ctx, "/api/v1/presets", &presets) == nil {
 			d.presets = &presets
@@ -588,6 +593,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 		case cFederation:
 			m.dialog = "error"
 			m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
+		case cSlurm:
+			m.menuD = m.slurmMenu(m.data.slurm.Nodes[m.row])
 		case cServe:
 			return m, m.fetchRepo(m.hf.models[m.row].ID)
 		case cAccounts:
@@ -1110,6 +1117,8 @@ func (m consoleModel) rowCount() int {
 		return len(m.data.audit)
 	case cFederation:
 		return len(m.data.federation.Peers)
+	case cSlurm:
+		return len(m.data.slurm.Nodes)
 	case cServe:
 		return len(m.hf.models)
 	case cAccounts:
@@ -1369,6 +1378,8 @@ func (m consoleModel) panel() (title, body, keys string) {
 		return m.auditPanel(pw)
 	case cFederation:
 		return m.federationPanel(pw)
+	case cSlurm:
+		return m.slurmPanel(pw)
 	case cMonitor:
 		return m.monitorPanel(pw)
 	case cServe:

internal/cli/tui_slurm.go

@@ -0,0 +1,148 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	tea "github.com/charmbracelet/bubbletea"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
+)
+
+// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P),
+// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows
+// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm
+// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm
+// part: a form for it would be server.yaml written again.
+
+func (m consoleModel) slurmPanel(pw int) (string, string, string) {
+	keys := "→ or Enter open · ↑↓ choose · Q leave"
+	if m.inPanel {
+		keys = "↑↓ node · Enter lend or take back · ← back · Q leave"
+	}
+	st := m.data.slurm
+	head := "\n  " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.")
+	if !st.Enabled {
+		return "Slurm", head + "\n\n  " + sDim.Render("Slurm is off. Turn it on with: "+
+			"llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys
+	}
+	ctl := sDim.Render("not heard from yet")
+	switch c := st.Ctld; {
+	case c == nil:
+	case c.Error != "":
+		ctl = sBad.Render(c.Error)
+	case c.SlurmctldRunning:
+		ctl = sOK.Render("slurmctld " + c.Version + " running")
+	default:
+		ctl = sBad.Render("slurmctld not running")
+	}
+	head += fmt.Sprintf("\n  controller %s: %s", st.Controller, ctl)
+	head += "\n  " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
+	head += "\n  " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
+	if len(st.Waiting) > 0 {
+		head += "\n  " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
+	}
+	head += "\n\n"
+	if len(st.Nodes) == 0 {
+		return "Slurm", head + "  " + sDim.Render("No node runs slurmd."), keys
+	}
+	var rows [][]string
+	for _, n := range st.Nodes {
+		rows = append(rows, slurmRow(n))
+	}
+	return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys
+}
+
+func slurmRow(n slurmsvc.NodeStatus) []string {
+	state, jobs, slurmd, note := "-", "-", "-", n.Describe()
+	if r := n.Status; r != nil {
+		state = or(r.State, "-")
+		jobs = fmt.Sprint(r.Jobs)
+		slurmd = sBad.Render("stopped")
+		if r.SlurmdRunning {
+			slurmd = sOK.Render("running")
+		}
+		if r.Error != "" {
+			note = sBad.Render(r.Error)
+		}
+	}
+	if !n.Connected {
+		note = sWarn.Render("agent not connected") + "; " + note
+	}
+	whose := string(n.Phase)
+	switch n.Phase {
+	case slurmsvc.Lent:
+		whose = sOK.Render("Slurm's")
+	case slurmsvc.Llmgw:
+		whose = "llmgw's"
+	case slurmsvc.Lending, slurmsvc.Reclaiming:
+		whose = sWarn.Render(whose)
+	}
+	return []string{n.Node, whose, state, jobs, slurmd, note}
+}
+
+// slurmMenu is what can be done with one of Slurm's nodes.
+func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog {
+	var items []menuItem
+	if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending {
+		items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd {
+			m.confirmD = &confirmDialog{title: "Take " + n.Node + " back",
+				text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.",
+				yes:  func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }}
+			return nil
+		}})
+	} else {
+		items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd {
+			text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there."
+			if n.Replicas > 0 {
+				text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+
+					"a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas)
+			}
+			m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text,
+				yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }}
+			return nil
+		}})
+	}
+	return &menuDialog{title: n.Node, items: items}
+}
+
+// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm
+// lend|reclaim` does: the newest settings, changed and put back.
+func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd {
+	title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first."
+	if !add {
+		title, note, done = "Take back "+node, "taken back from Slurm in the console",
+			node+" is being taken back: llmgw has it once its Slurm jobs end."
+	}
+	m.busy = title
+	c := m.client
+	return func() tea.Msg {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		var cur adminapi.SettingsView
+		if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
+			return actionDoneMsg{title: title, err: err}
+		}
+		doc, err := settings.FromBytes("settings", []byte(cur.Body))
+		if err == nil {
+			err = lendChange(doc, []string{node}, add)
+		}
+		var body []byte
+		if err == nil {
+			body, err = doc.Bytes()
+		}
+		if err != nil {
+			return actionDoneMsg{title: title, err: err}
+		}
+		req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
+		var v adminapi.SettingsView
+		if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil {
+			return actionDoneMsg{title: title, err: err}
+		}
+		return actionDoneMsg{title: title, text: done}
+	}
+}

internal/cli/tui_slurm_test.go

@@ -0,0 +1,92 @@
+package cli
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+
+	tea "github.com/charmbracelet/bubbletea"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
+)
+
+func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
+	var mu sync.Mutex
+	var put adminapi.SettingsRequest
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
+			_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7,
+				Body: "slurm:\n  enabled: true\n  controller: node06\n  nodes: [node05, cuda01]\n"})
+		case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
+			b, _ := io.ReadAll(r.Body)
+			mu.Lock()
+			_ = json.Unmarshal(b, &put)
+			mu.Unlock()
+			_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8})
+		default:
+			_, _ = w.Write([]byte("[]"))
+		}
+	}))
+	t.Cleanup(srv.Close)
+	m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
+	m.loaded, m.menu, m.inPanel = true, cSlurm, true
+	m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
+		Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"},
+		Nodes: []slurmsvc.NodeStatus{
+			{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
+				Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
+			{Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true,
+				Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}},
+		}}
+	var tm tea.Model = m
+	view := tm.View()
+	for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
+		if !strings.Contains(view, want) {
+			t.Errorf("the Slurm view has no %q:\n%s", want, view)
+		}
+	}
+
+	// cuda01, the second row: lent, after saying its replicas move.
+	tm = press(tm, "down", "enter")
+	if !strings.Contains(tm.View(), "Lend to Slurm") {
+		t.Fatalf("no lend in the menu:\n%s", tm.View())
+	}
+	tm = press(tm, "enter")
+	if !strings.Contains(tm.View(), "3 replica(s) move off first") {
+		t.Fatalf("the question does not say what moves:\n%s", tm.View())
+	}
+	tm = pressRun(tm, "y")
+	mu.Lock()
+	defer mu.Unlock()
+	if put.Base != 7 || !strings.Contains(put.Body, "lent:") || !strings.Contains(put.Body, "cuda01") {
+		t.Fatalf("the settings put back: %+v", put)
+	}
+	if !strings.Contains(tm.View(), "cuda01 is being lent to Slurm") {
+		t.Errorf("not said:\n%s", tm.View())
+	}
+}
+
+func TestTheConsoleTakesALentNodeBack(t *testing.T) {
+	m, _ := consoleWith(t, cSlurm)
+	m.data.slurm = slurmsvc.Status{Enabled: true, Controller: "node06",
+		Nodes: []slurmsvc.NodeStatus{{Node: "node05", Phase: slurmsvc.Lent, Connected: true}}}
+	var tm tea.Model = m
+	tm = press(tm, "enter")
+	if v := tm.View(); !strings.Contains(v, "Take back from Slurm") || strings.Contains(v, "Lend to Slurm") {
+		t.Fatalf("a lent node's menu:\n%s", v)
+	}
+}
+
+func TestTheConsoleSaysHowToTurnSlurmOn(t *testing.T) {
+	m, _ := consoleWith(t, cSlurm)
+	var tm tea.Model = m
+	if !strings.Contains(tm.View(), "llmgw ctl slurm setup") {
+		t.Fatalf("Slurm off says nothing of turning it on:\n%s", tm.View())
+	}
+}