diff --git a/BACKLOG.md b/BACKLOG.md index 0567330..2c78d3a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2053,7 +2053,8 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t way: a node not heard from or with Slurm jobs is cordoned, not drained (nothing of llmgw's to move, and a new leader must not move models off every Slurm node); jobs are counted from the node's own slurmstepd, so a controller that is down neither hides them nor, with none, keeps the node from llmgw; a drain whose - reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5, + reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: ~~the TUI screen~~ + (done 2026-11-11: console **Slurm**, lend/take back on a row), P5, a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are taken to be nvidia-smi's index. - In the first install too (2026-11-11): `llmgw server init --slurm`, and a Slurm part on the choices diff --git a/docs/console.md b/docs/console.md index 4d7e116..cdd69a0 100644 --- a/docs/console.md +++ b/docs/console.md @@ -207,6 +207,16 @@ starting — and whether it ended done, failed, or waiting for room on busy cards. The entry shows how many are running. Enter on one shows its steps. `llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one. +**Slurm** is the Slurm cluster run beside llmgw, when there is one: its +controller and whether slurmctld runs, and each node that runs slurmd — +whose it is now (llmgw's, Slurm's, or on its way between them), its state in +Slurm, the jobs running there. On a row, **Lend to Slurm…** moves the node's +replicas off and then lets Slurm run jobs there; **Take back from Slurm…** +starts no new job there and gives it back to llmgw when its jobs have ended. +Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend` +and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm +setup`, or the Slurm part of the first install. + **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index aab49f2..cc7e604 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -178,20 +178,7 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return changeSettings(cmd, client, note, func(doc *settings.Doc) error { - nodes := docList(doc, []string{"slurm", "nodes"}) - all := docBool(doc, []string{"slurm", "all_nodes"}) - lent := docList(doc, []string{"slurm", "lent"}) - for _, n := range args { - switch { - case add && !all && !slices.Contains(nodes, n): - return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) - case add && !slices.Contains(lent, n): - lent = append(lent, n) - case !add: - lent = slices.DeleteFunc(lent, func(l string) bool { return l == n }) - } - } - return setList(doc, []string{"slurm", "lent"}, lent) + return lendChange(doc, args, add) }) }, } @@ -199,6 +186,24 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob return c } +// lendChange adds nodes to slurm.lent, or with add false takes them out. +func lendChange(doc *settings.Doc, names []string, add bool) error { + nodes := docList(doc, []string{"slurm", "nodes"}) + all := docBool(doc, []string{"slurm", "all_nodes"}) + lent := docList(doc, []string{"slurm", "lent"}) + for _, n := range names { + switch { + case add && !all && !slices.Contains(nodes, n): + return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) + case add && !slices.Contains(lent, n): + lent = append(lent, n) + case !add: + lent = slices.DeleteFunc(lent, func(l string) bool { return l == n }) + } + } + return setList(doc, []string{"slurm", "lent"}, lent) +} + // docList reads a list of strings from the settings, empty when unset. func docList(doc *settings.Doc, path []string) []string { n := doc.Node(path) diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index a53f1d6..bf22da1 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -24,6 +24,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -50,6 +51,7 @@ const ( cFederation cMirror cJobs + cSlurm cLogs cAccounts cKeys @@ -71,7 +73,7 @@ const ( cCount ) -var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual", +var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Slurm", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual", "server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster", "Restart all replicas", "Back up now", "Wipe this machine", "Leave"} @@ -92,6 +94,8 @@ type consoleData struct { // federation is the other llmgw clusters whose models are offered // here; empty on a control plane without a store. federation adminapi.FederationView + // slurm is the Slurm cluster beside llmgw; off when it has none. + slurm slurmsvc.Status // presets is the models that come with llmgw as they stand here; nil // from a control plane that cannot say. presets *adminapi.PresetsView @@ -284,6 +288,7 @@ func (m consoleModel) fetch() tea.Cmd { _ = c.get(ctx, "/api/v1/jobs", &d.jobs) _ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit) _ = c.get(ctx, "/api/v1/federation", &d.federation) + _ = c.get(ctx, "/api/v1/slurm", &d.slurm) var presets adminapi.PresetsView if c.get(ctx, "/api/v1/presets", &presets) == nil { d.presets = &presets @@ -588,6 +593,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { case cFederation: m.dialog = "error" m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row]) + case cSlurm: + m.menuD = m.slurmMenu(m.data.slurm.Nodes[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: @@ -1110,6 +1117,8 @@ func (m consoleModel) rowCount() int { return len(m.data.audit) case cFederation: return len(m.data.federation.Peers) + case cSlurm: + return len(m.data.slurm.Nodes) case cServe: return len(m.hf.models) case cAccounts: @@ -1369,6 +1378,8 @@ func (m consoleModel) panel() (title, body, keys string) { return m.auditPanel(pw) case cFederation: return m.federationPanel(pw) + case cSlurm: + return m.slurmPanel(pw) case cMonitor: return m.monitorPanel(pw) case cServe: diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go new file mode 100644 index 0000000..11f42ae --- /dev/null +++ b/internal/cli/tui_slurm.go @@ -0,0 +1,148 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "time" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P), +// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows +// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm +// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm +// part: a form for it would be server.yaml written again. + +func (m consoleModel) slurmPanel(pw int) (string, string, string) { + keys := "→ or Enter open · ↑↓ choose · Q leave" + if m.inPanel { + keys = "↑↓ node · Enter lend or take back · ← back · Q leave" + } + st := m.data.slurm + head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.") + if !st.Enabled { + return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+ + "llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys + } + ctl := sDim.Render("not heard from yet") + switch c := st.Ctld; { + case c == nil: + case c.Error != "": + ctl = sBad.Render(c.Error) + case c.SlurmctldRunning: + ctl = sOK.Render("slurmctld " + c.Version + " running") + default: + ctl = sBad.Render("slurmctld not running") + } + head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl) + head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller)) + head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") + if len(st.Waiting) > 0 { + head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) + } + head += "\n\n" + if len(st.Nodes) == 0 { + return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys + } + var rows [][]string + for _, n := range st.Nodes { + rows = append(rows, slurmRow(n)) + } + return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys +} + +func slurmRow(n slurmsvc.NodeStatus) []string { + state, jobs, slurmd, note := "-", "-", "-", n.Describe() + if r := n.Status; r != nil { + state = or(r.State, "-") + jobs = fmt.Sprint(r.Jobs) + slurmd = sBad.Render("stopped") + if r.SlurmdRunning { + slurmd = sOK.Render("running") + } + if r.Error != "" { + note = sBad.Render(r.Error) + } + } + if !n.Connected { + note = sWarn.Render("agent not connected") + "; " + note + } + whose := string(n.Phase) + switch n.Phase { + case slurmsvc.Lent: + whose = sOK.Render("Slurm's") + case slurmsvc.Llmgw: + whose = "llmgw's" + case slurmsvc.Lending, slurmsvc.Reclaiming: + whose = sWarn.Render(whose) + } + return []string{n.Node, whose, state, jobs, slurmd, note} +} + +// slurmMenu is what can be done with one of Slurm's nodes. +func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog { + var items []menuItem + if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending { + items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd { + m.confirmD = &confirmDialog{title: "Take " + n.Node + " back", + text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.", + yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }} + return nil + }}) + } else { + items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd { + text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there." + if n.Replicas > 0 { + text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+ + "a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas) + } + m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text, + yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }} + return nil + }}) + } + return &menuDialog{title: n.Node, items: items} +} + +// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm +// lend|reclaim` does: the newest settings, changed and put back. +func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd { + title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first." + if !add { + title, note, done = "Take back "+node, "taken back from Slurm in the console", + node+" is being taken back: llmgw has it once its Slurm jobs end." + } + m.busy = title + c := m.client + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var cur adminapi.SettingsView + if err := c.get(ctx, "/api/v1/settings", &cur); err != nil { + return actionDoneMsg{title: title, err: err} + } + doc, err := settings.FromBytes("settings", []byte(cur.Body)) + if err == nil { + err = lendChange(doc, []string{node}, add) + } + var body []byte + if err == nil { + body, err = doc.Bytes() + } + if err != nil { + return actionDoneMsg{title: title, err: err} + } + req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note}) + var v adminapi.SettingsView + if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil { + return actionDoneMsg{title: title, err: err} + } + return actionDoneMsg{title: title, text: done} + } +} diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go new file mode 100644 index 0000000..b6d03c2 --- /dev/null +++ b/internal/cli/tui_slurm_test.go @@ -0,0 +1,92 @@ +package cli + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + tea "github.com/charmbracelet/bubbletea" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" +) + +func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { + var mu sync.Mutex + var put adminapi.SettingsRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings": + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7, + Body: "slurm:\n enabled: true\n controller: node06\n nodes: [node05, cuda01]\n"}) + case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings": + b, _ := io.ReadAll(r.Body) + mu.Lock() + _ = json.Unmarshal(b, &put) + mu.Unlock() + _ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8}) + default: + _, _ = w.Write([]byte("[]")) + } + })) + t.Cleanup(srv.Close) + m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") + m.loaded, m.menu, m.inPanel = true, cSlurm, true + m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"}, + Nodes: []slurmsvc.NodeStatus{ + {Node: "node05", Phase: slurmsvc.Lent, Connected: true, + Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, + {Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true, + Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}}, + }} + var tm tea.Model = m + view := tm.View() + for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + if !strings.Contains(view, want) { + t.Errorf("the Slurm view has no %q:\n%s", want, view) + } + } + + // cuda01, the second row: lent, after saying its replicas move. + tm = press(tm, "down", "enter") + if !strings.Contains(tm.View(), "Lend to Slurm") { + t.Fatalf("no lend in the menu:\n%s", tm.View()) + } + tm = press(tm, "enter") + if !strings.Contains(tm.View(), "3 replica(s) move off first") { + t.Fatalf("the question does not say what moves:\n%s", tm.View()) + } + tm = pressRun(tm, "y") + mu.Lock() + defer mu.Unlock() + if put.Base != 7 || !strings.Contains(put.Body, "lent:") || !strings.Contains(put.Body, "cuda01") { + t.Fatalf("the settings put back: %+v", put) + } + if !strings.Contains(tm.View(), "cuda01 is being lent to Slurm") { + t.Errorf("not said:\n%s", tm.View()) + } +} + +func TestTheConsoleTakesALentNodeBack(t *testing.T) { + m, _ := consoleWith(t, cSlurm) + m.data.slurm = slurmsvc.Status{Enabled: true, Controller: "node06", + Nodes: []slurmsvc.NodeStatus{{Node: "node05", Phase: slurmsvc.Lent, Connected: true}}} + var tm tea.Model = m + tm = press(tm, "enter") + if v := tm.View(); !strings.Contains(v, "Take back from Slurm") || strings.Contains(v, "Lend to Slurm") { + t.Fatalf("a lent node's menu:\n%s", v) + } +} + +func TestTheConsoleSaysHowToTurnSlurmOn(t *testing.T) { + m, _ := consoleWith(t, cSlurm) + var tm tea.Model = m + if !strings.Contains(tm.View(), "llmgw ctl slurm setup") { + t.Fatalf("Slurm off says nothing of turning it on:\n%s", tm.View()) + } +}