Slurm in the console: each node, and lend or take back from its row
CI / test (push) Successful in 9m39s

The console's Slurm section shows the controller, and each node that runs
slurmd with whose it is now, its state in Slurm and its jobs. Enter on a
row lends the node (its replicas move off first) or takes it back, by the
same change to slurm.lent as ctl slurm lend and reclaim.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-11 22:53:26 +01:00
1 parent 17287ee8b0
commit 356f7e0ced
6 files changed
+283 -16

No files matched your search

+19 -14
View File
@@ -178,20 +178,7 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
Args: cobra.MinimumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
nodes := docList(doc, []string{"slurm", "nodes"})
all := docBool(doc, []string{"slurm", "all_nodes"})
lent := docList(doc, []string{"slurm", "lent"})
for _, n := range args {
switch {
case add && !all && !slices.Contains(nodes, n):
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
case add && !slices.Contains(lent, n):
lent = append(lent, n)
case !add:
lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
}
}
return setList(doc, []string{"slurm", "lent"}, lent)
return lendChange(doc, args, add)
})
},
}
@@ -199,6 +186,24 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
return c
}
// lendChange adds nodes to slurm.lent, or with add false takes them out.
func lendChange(doc *settings.Doc, names []string, add bool) error {
nodes := docList(doc, []string{"slurm", "nodes"})
all := docBool(doc, []string{"slurm", "all_nodes"})
lent := docList(doc, []string{"slurm", "lent"})
for _, n := range names {
switch {
case add && !all && !slices.Contains(nodes, n):
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
case add && !slices.Contains(lent, n):
lent = append(lent, n)
case !add:
lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
}
}
return setList(doc, []string{"slurm", "lent"}, lent)
}
// docList reads a list of strings from the settings, empty when unset.
func docList(doc *settings.Doc, path []string) []string {
n := doc.Node(path)
+12 -1
View File
@@ -24,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
@@ -50,6 +51,7 @@ const (
cFederation
cMirror
cJobs
cSlurm
cLogs
cAccounts
cKeys
@@ -71,7 +73,7 @@ const (
cCount
)
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Slurm", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
@@ -92,6 +94,8 @@ type consoleData struct {
// federation is the other llmgw clusters whose models are offered
// here; empty on a control plane without a store.
federation adminapi.FederationView
// slurm is the Slurm cluster beside llmgw; off when it has none.
slurm slurmsvc.Status
// presets is the models that come with llmgw as they stand here; nil
// from a control plane that cannot say.
presets *adminapi.PresetsView
@@ -284,6 +288,7 @@ func (m consoleModel) fetch() tea.Cmd {
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
_ = c.get(ctx, "/api/v1/federation", &d.federation)
_ = c.get(ctx, "/api/v1/slurm", &d.slurm)
var presets adminapi.PresetsView
if c.get(ctx, "/api/v1/presets", &presets) == nil {
d.presets = &presets
@@ -588,6 +593,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
case cFederation:
m.dialog = "error"
m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
case cSlurm:
m.menuD = m.slurmMenu(m.data.slurm.Nodes[m.row])
case cServe:
return m, m.fetchRepo(m.hf.models[m.row].ID)
case cAccounts:
@@ -1110,6 +1117,8 @@ func (m consoleModel) rowCount() int {
return len(m.data.audit)
case cFederation:
return len(m.data.federation.Peers)
case cSlurm:
return len(m.data.slurm.Nodes)
case cServe:
return len(m.hf.models)
case cAccounts:
@@ -1369,6 +1378,8 @@ func (m consoleModel) panel() (title, body, keys string) {
return m.auditPanel(pw)
case cFederation:
return m.federationPanel(pw)
case cSlurm:
return m.slurmPanel(pw)
case cMonitor:
return m.monitorPanel(pw)
case cServe:
+148
View File
@@ -0,0 +1,148 @@
package cli
import (
"context"
"encoding/json"
"fmt"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P),
// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows
// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm
// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm
// part: a form for it would be server.yaml written again.
func (m consoleModel) slurmPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ node · Enter lend or take back · ← back · Q leave"
}
st := m.data.slurm
head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.")
if !st.Enabled {
return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+
"llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys
}
ctl := sDim.Render("not heard from yet")
switch c := st.Ctld; {
case c == nil:
case c.Error != "":
ctl = sBad.Render(c.Error)
case c.SlurmctldRunning:
ctl = sOK.Render("slurmctld " + c.Version + " running")
default:
ctl = sBad.Render("slurmctld not running")
}
head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl)
head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
if len(st.Waiting) > 0 {
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
}
head += "\n\n"
if len(st.Nodes) == 0 {
return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys
}
var rows [][]string
for _, n := range st.Nodes {
rows = append(rows, slurmRow(n))
}
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys
}
func slurmRow(n slurmsvc.NodeStatus) []string {
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
if r := n.Status; r != nil {
state = or(r.State, "-")
jobs = fmt.Sprint(r.Jobs)
slurmd = sBad.Render("stopped")
if r.SlurmdRunning {
slurmd = sOK.Render("running")
}
if r.Error != "" {
note = sBad.Render(r.Error)
}
}
if !n.Connected {
note = sWarn.Render("agent not connected") + "; " + note
}
whose := string(n.Phase)
switch n.Phase {
case slurmsvc.Lent:
whose = sOK.Render("Slurm's")
case slurmsvc.Llmgw:
whose = "llmgw's"
case slurmsvc.Lending, slurmsvc.Reclaiming:
whose = sWarn.Render(whose)
}
return []string{n.Node, whose, state, jobs, slurmd, note}
}
// slurmMenu is what can be done with one of Slurm's nodes.
func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog {
var items []menuItem
if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending {
items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Take " + n.Node + " back",
text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.",
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }}
return nil
}})
} else {
items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd {
text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there."
if n.Replicas > 0 {
text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+
"a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas)
}
m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text,
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }}
return nil
}})
}
return &menuDialog{title: n.Node, items: items}
}
// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm
// lend|reclaim` does: the newest settings, changed and put back.
func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd {
title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first."
if !add {
title, note, done = "Take back "+node, "taken back from Slurm in the console",
node+" is being taken back: llmgw has it once its Slurm jobs end."
}
m.busy = title
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var cur adminapi.SettingsView
if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
return actionDoneMsg{title: title, err: err}
}
doc, err := settings.FromBytes("settings", []byte(cur.Body))
if err == nil {
err = lendChange(doc, []string{node}, add)
}
var body []byte
if err == nil {
body, err = doc.Bytes()
}
if err != nil {
return actionDoneMsg{title: title, err: err}
}
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
var v adminapi.SettingsView
if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil {
return actionDoneMsg{title: title, err: err}
}
return actionDoneMsg{title: title, text: done}
}
}
+92
View File
@@ -0,0 +1,92 @@
package cli
import (
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
)
func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
var mu sync.Mutex
var put adminapi.SettingsRequest
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7,
Body: "slurm:\n enabled: true\n controller: node06\n nodes: [node05, cuda01]\n"})
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
b, _ := io.ReadAll(r.Body)
mu.Lock()
_ = json.Unmarshal(b, &put)
mu.Unlock()
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8})
default:
_, _ = w.Write([]byte("[]"))
}
}))
t.Cleanup(srv.Close)
m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
m.loaded, m.menu, m.inPanel = true, cSlurm, true
m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"},
Nodes: []slurmsvc.NodeStatus{
{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
{Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true,
Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}},
}}
var tm tea.Model = m
view := tm.View()
for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
if !strings.Contains(view, want) {
t.Errorf("the Slurm view has no %q:\n%s", want, view)
}
}
// cuda01, the second row: lent, after saying its replicas move.
tm = press(tm, "down", "enter")
if !strings.Contains(tm.View(), "Lend to Slurm") {
t.Fatalf("no lend in the menu:\n%s", tm.View())
}
tm = press(tm, "enter")
if !strings.Contains(tm.View(), "3 replica(s) move off first") {
t.Fatalf("the question does not say what moves:\n%s", tm.View())
}
tm = pressRun(tm, "y")
mu.Lock()
defer mu.Unlock()
if put.Base != 7 || !strings.Contains(put.Body, "lent:") || !strings.Contains(put.Body, "cuda01") {
t.Fatalf("the settings put back: %+v", put)
}
if !strings.Contains(tm.View(), "cuda01 is being lent to Slurm") {
t.Errorf("not said:\n%s", tm.View())
}
}
func TestTheConsoleTakesALentNodeBack(t *testing.T) {
m, _ := consoleWith(t, cSlurm)
m.data.slurm = slurmsvc.Status{Enabled: true, Controller: "node06",
Nodes: []slurmsvc.NodeStatus{{Node: "node05", Phase: slurmsvc.Lent, Connected: true}}}
var tm tea.Model = m
tm = press(tm, "enter")
if v := tm.View(); !strings.Contains(v, "Take back from Slurm") || strings.Contains(v, "Lend to Slurm") {
t.Fatalf("a lent node's menu:\n%s", v)
}
}
func TestTheConsoleSaysHowToTurnSlurmOn(t *testing.T) {
m, _ := consoleWith(t, cSlurm)
var tm tea.Model = m
if !strings.Contains(tm.View(), "llmgw ctl slurm setup") {
t.Fatalf("Slurm off says nothing of turning it on:\n%s", tm.View())
}
}