Files
llmgw/internal/cli/tui_slurm.go
T
Ilker ManapandClaude Opus 5.5 748f777a56
CI / test (push) Successful in 10m57s
Slurm's accounting: slurmdbd and a MariaDB on the controller
Since 23.11 Slurm runs badly without accounting storage: the main
scheduler passes every job by as InvalidAccount. slurmdbd takes MySQL or
MariaDB only, so the controller's agent installs MariaDB beside it,
answering that machine alone, makes the database and its user (the
password made from the munge key, never on a command line), runs
slurmdbd and adds the cluster, all before slurmctld. A MariaDB that was
on the machine before keeps its settings. slurm.conf accounts the jobs
there, with what each used from its cgroup and the cards, and the
priority is multifactor again. Port 6819 joins Slurm's in the firewall.
ctl slurm status and the console say whether slurmdbd runs; the install
screen, server init --slurm and the guide say what is put where.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 07:59:08 +01:00

151 lines
5.6 KiB
Go

package cli
import (
"context"
"encoding/json"
"fmt"
"time"
tea "github.com/charmbracelet/bubbletea"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P),
// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows
// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm
// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm
// part: a form for it would be server.yaml written again.
func (m consoleModel) slurmPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ node · Enter lend or take back · ← back · Q leave"
}
st := m.data.slurm
head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.")
if !st.Enabled {
return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+
"llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys
}
ctl := sDim.Render("not heard from yet")
switch c := st.Ctld; {
case c == nil:
case c.Error != "":
ctl = sBad.Render(c.Error)
case c.SlurmctldRunning && c.SlurmdbdRunning:
ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running")
case c.SlurmctldRunning:
ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not")
default:
ctl = sBad.Render("slurmctld not running")
}
head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl)
head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
if len(st.Waiting) > 0 {
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
}
head += "\n\n"
if len(st.Nodes) == 0 {
return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys
}
var rows [][]string
for _, n := range st.Nodes {
rows = append(rows, slurmRow(n))
}
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys
}
func slurmRow(n slurmsvc.NodeStatus) []string {
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
if r := n.Status; r != nil {
state = or(r.State, "-")
jobs = fmt.Sprint(r.Jobs)
slurmd = sBad.Render("stopped")
if r.SlurmdRunning {
slurmd = sOK.Render("running")
}
if r.Error != "" {
note = sBad.Render(r.Error)
}
}
if !n.Connected {
note = sWarn.Render("agent not connected") + "; " + note
}
whose := string(n.Phase)
switch n.Phase {
case slurmsvc.Lent:
whose = sOK.Render("Slurm's")
case slurmsvc.Llmgw:
whose = "llmgw's"
case slurmsvc.Lending, slurmsvc.Reclaiming:
whose = sWarn.Render(whose)
}
return []string{n.Node, whose, state, jobs, slurmd, note}
}
// slurmMenu is what can be done with one of Slurm's nodes.
func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog {
var items []menuItem
if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending {
items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd {
m.confirmD = &confirmDialog{title: "Take " + n.Node + " back",
text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.",
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }}
return nil
}})
} else {
items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd {
text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there."
if n.Replicas > 0 {
text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+
"a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas)
}
m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text,
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }}
return nil
}})
}
return &menuDialog{title: n.Node, items: items}
}
// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm
// lend|reclaim` does: the newest settings, changed and put back.
func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd {
title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first."
if !add {
title, note, done = "Take back "+node, "taken back from Slurm in the console",
node+" is being taken back: llmgw has it once its Slurm jobs end."
}
m.busy = title
c := m.client
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var cur adminapi.SettingsView
if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
return actionDoneMsg{title: title, err: err}
}
doc, err := settings.FromBytes("settings", []byte(cur.Body))
if err == nil {
err = lendChange(doc, []string{node}, add)
}
var body []byte
if err == nil {
body, err = doc.Bytes()
}
if err != nil {
return actionDoneMsg{title: title, err: err}
}
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
var v adminapi.SettingsView
if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil {
return actionDoneMsg{title: title, err: err}
}
return actionDoneMsg{title: title, text: done}
}
}