CI / test (push) Failing after 7m1s
slurm.shared mounts the site's NFS export, has an llmgw node export the path over NFS to the Slurm machines alone (2049 in the firewall), or looks for a file system the site mounts. slurm.users "llmgw" makes each active account a Linux user on every Slurm machine: login from its e-mail, a UID from 200000 kept in slurm_users and never reused, its home on the shared path, its SSH keys from the portal's Profile (or ctl slurm users keys) read by sshd from /etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first machine and llmgw's users; --slurm-shared and --slurm-users change that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
168 lines
6.2 KiB
Go
168 lines
6.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
|
)
|
|
|
|
// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P),
|
|
// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows
|
|
// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm
|
|
// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm
|
|
// part: a form for it would be server.yaml written again.
|
|
|
|
func (m consoleModel) slurmPanel(pw int) (string, string, string) {
|
|
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
|
if m.inPanel {
|
|
keys = "↑↓ node · Enter lend or take back · ← back · Q leave"
|
|
}
|
|
st := m.data.slurm
|
|
head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.")
|
|
if !st.Enabled {
|
|
return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+
|
|
"llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys
|
|
}
|
|
ctl := sDim.Render("not heard from yet")
|
|
switch c := st.Ctld; {
|
|
case c == nil:
|
|
case c.Error != "":
|
|
ctl = sBad.Render(c.Error)
|
|
case c.SlurmctldRunning && c.SlurmdbdRunning:
|
|
ctl = sOK.Render("slurmctld " + c.Version + " and slurmdbd running")
|
|
case c.SlurmctldRunning:
|
|
ctl = sOK.Render("slurmctld "+c.Version+" running") + ", " + sWarn.Render("slurmdbd not")
|
|
default:
|
|
ctl = sBad.Render("slurmctld not running")
|
|
}
|
|
head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl)
|
|
head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
|
|
if st.Shared.Path == "" {
|
|
head += "\n shared files: " + sWarn.Render("none") + sDim.Render(" (llmgw ctl slurm shared; output stays on the job's node)")
|
|
} else {
|
|
head += "\n shared files: " + sharedLine(st.Shared)
|
|
}
|
|
if st.Users != "" {
|
|
head += "\n " + fmt.Sprintf("users: %d from llmgw's accounts", st.UserCount) + sDim.Render(", SSH keys in the portal's Profile")
|
|
} else {
|
|
head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)")
|
|
}
|
|
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
|
|
if len(st.Waiting) > 0 {
|
|
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
|
|
}
|
|
head += "\n\n"
|
|
if len(st.Nodes) == 0 {
|
|
return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys
|
|
}
|
|
var rows [][]string
|
|
for _, n := range st.Nodes {
|
|
rows = append(rows, slurmRow(n))
|
|
}
|
|
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "SHARED", "NOTE"}, rows, m.row, pw, m.inPanel), keys
|
|
}
|
|
|
|
func slurmRow(n slurmsvc.NodeStatus) []string {
|
|
state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe()
|
|
if r := n.Status; r != nil {
|
|
state = or(r.State, "-")
|
|
switch r.Shared {
|
|
case "":
|
|
case "missing":
|
|
shared = sBad.Render(r.Shared)
|
|
default:
|
|
shared = sOK.Render(r.Shared)
|
|
}
|
|
jobs = fmt.Sprint(r.Jobs)
|
|
slurmd = sBad.Render("stopped")
|
|
if r.SlurmdRunning {
|
|
slurmd = sOK.Render("running")
|
|
}
|
|
if r.Error != "" {
|
|
note = sBad.Render(r.Error)
|
|
}
|
|
}
|
|
if !n.Connected {
|
|
note = sWarn.Render("agent not connected") + "; " + note
|
|
}
|
|
whose := string(n.Phase)
|
|
switch n.Phase {
|
|
case slurmsvc.Lent:
|
|
whose = sOK.Render("Slurm's")
|
|
case slurmsvc.Llmgw:
|
|
whose = "llmgw's"
|
|
case slurmsvc.Lending, slurmsvc.Reclaiming:
|
|
whose = sWarn.Render(whose)
|
|
}
|
|
return []string{n.Node, whose, state, jobs, slurmd, shared, note}
|
|
}
|
|
|
|
// slurmMenu is what can be done with one of Slurm's nodes.
|
|
func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog {
|
|
var items []menuItem
|
|
if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending {
|
|
items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd {
|
|
m.confirmD = &confirmDialog{title: "Take " + n.Node + " back",
|
|
text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.",
|
|
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }}
|
|
return nil
|
|
}})
|
|
} else {
|
|
items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd {
|
|
text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there."
|
|
if n.Replicas > 0 {
|
|
text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+
|
|
"a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas)
|
|
}
|
|
m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text,
|
|
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }}
|
|
return nil
|
|
}})
|
|
}
|
|
return &menuDialog{title: n.Node, items: items}
|
|
}
|
|
|
|
// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm
|
|
// lend|reclaim` does: the newest settings, changed and put back.
|
|
func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd {
|
|
title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first."
|
|
if !add {
|
|
title, note, done = "Take back "+node, "taken back from Slurm in the console",
|
|
node+" is being taken back: llmgw has it once its Slurm jobs end."
|
|
}
|
|
m.busy = title
|
|
c := m.client
|
|
return func() tea.Msg {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
var cur adminapi.SettingsView
|
|
if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
|
|
return actionDoneMsg{title: title, err: err}
|
|
}
|
|
doc, err := settings.FromBytes("settings", []byte(cur.Body))
|
|
if err == nil {
|
|
err = lendChange(doc, []string{node}, add)
|
|
}
|
|
var body []byte
|
|
if err == nil {
|
|
body, err = doc.Bytes()
|
|
}
|
|
if err != nil {
|
|
return actionDoneMsg{title: title, err: err}
|
|
}
|
|
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
|
|
var v adminapi.SettingsView
|
|
if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil {
|
|
return actionDoneMsg{title: title, err: err}
|
|
}
|
|
return actionDoneMsg{title: title, text: done}
|
|
}
|
|
}
|