Compare commits
2
Commits
0774e901c0
...
356f7e0ced
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
356f7e0ced | ||
|
|
17287ee8b0 |
No files matched your search
+12
-2
@@ -2037,14 +2037,24 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
|
||||
from the daemon); the packaged unit is Type=notify (slurmd needs --systemd); slurmd's own "slurmstepd
|
||||
infinity" was counted as a job; without accounting storage 23.11's main scheduler marks batch jobs
|
||||
InvalidAccount and backfill starts them, one a pass (bf_interval=10: 6-10 s each). So many small jobs
|
||||
want slurmdbd. Not tried yet: a GPU node (gres), a leader failover while lent.
|
||||
want slurmdbd. Not tried yet: a GPU node (gres).
|
||||
- Failover while lent, tried 2026-11-11 (v0.16.3-23): the first try killed the job (NODE_FAIL). A new
|
||||
leader had heard from no node yet and sent the controller a slurm.conf without them; slurmctld,
|
||||
restarted on it, dropped node05 and its job. Fixed (141a1f4, 0774e90): slurm.conf's nodes and the
|
||||
controller's address are kept in cluster_state (`slurm-nodes`) and a new leader starts from them; with
|
||||
nothing kept (an upgraded cluster) or the store unread, the controller's files wait up to 2 min for the
|
||||
nodes; a node is told nothing until its controller's address is known (no part stops its slurmd). Tried
|
||||
again: a 5-minute job on lent node05, leader 140 stopped, 150 took over; slurmctld not restarted, the job
|
||||
ran to its end; reclaimed during it, node05 was llmgw's within seconds of its end. Also 3387f97: srun in
|
||||
SrunPortRange=60001-63000, and Slurm's ports in llmgw's firewall for the cluster's networks.
|
||||
- Built 2026-11-11: P1 (`internal/slurm`, `slurm.*` settings), P2
|
||||
(`internal/agent/slurmnode`, proto Slurm/SlurmStatus), P3 (`internal/control/slurmsvc`, wired in the
|
||||
leader; `GET /api/v1/slurm`), P4's commands (`ctl slurm status|setup|lend|reclaim|off`). Decisions on the
|
||||
way: a node not heard from or with Slurm jobs is cordoned, not drained (nothing of llmgw's to move, and a
|
||||
new leader must not move models off every Slurm node); jobs are counted from the node's own slurmstepd,
|
||||
so a controller that is down neither hides them nor, with none, keeps the node from llmgw; a drain whose
|
||||
reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5,
|
||||
reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: ~~the TUI screen~~
|
||||
(done 2026-11-11: console **Slurm**, lend/take back on a row), P5,
|
||||
a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are
|
||||
taken to be nvidia-smi's index.
|
||||
- In the first install too (2026-11-11): `llmgw server init --slurm`, and a Slurm part on the choices
|
||||
|
||||
@@ -207,6 +207,16 @@ starting — and whether it ended done, failed, or waiting for room on busy
|
||||
cards. The entry shows how many are running. Enter on one shows its steps.
|
||||
`llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one.
|
||||
|
||||
**Slurm** is the Slurm cluster run beside llmgw, when there is one: its
|
||||
controller and whether slurmctld runs, and each node that runs slurmd —
|
||||
whose it is now (llmgw's, Slurm's, or on its way between them), its state in
|
||||
Slurm, the jobs running there. On a row, **Lend to Slurm…** moves the node's
|
||||
replicas off and then lets Slurm run jobs there; **Take back from Slurm…**
|
||||
starts no new job there and gives it back to llmgw when its jobs have ended.
|
||||
Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend`
|
||||
and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm
|
||||
setup`, or the Slurm part of the first install.
|
||||
|
||||
**Logs** shows a node's agent log, newest at the bottom, errors and warnings
|
||||
coloured.
|
||||
|
||||
|
||||
+19
-14
@@ -178,20 +178,7 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return changeSettings(cmd, client, note, func(doc *settings.Doc) error {
|
||||
nodes := docList(doc, []string{"slurm", "nodes"})
|
||||
all := docBool(doc, []string{"slurm", "all_nodes"})
|
||||
lent := docList(doc, []string{"slurm", "lent"})
|
||||
for _, n := range args {
|
||||
switch {
|
||||
case add && !all && !slices.Contains(nodes, n):
|
||||
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
|
||||
case add && !slices.Contains(lent, n):
|
||||
lent = append(lent, n)
|
||||
case !add:
|
||||
lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
|
||||
}
|
||||
}
|
||||
return setList(doc, []string{"slurm", "lent"}, lent)
|
||||
return lendChange(doc, args, add)
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -199,6 +186,24 @@ func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cob
|
||||
return c
|
||||
}
|
||||
|
||||
// lendChange adds nodes to slurm.lent, or with add false takes them out.
|
||||
func lendChange(doc *settings.Doc, names []string, add bool) error {
|
||||
nodes := docList(doc, []string{"slurm", "nodes"})
|
||||
all := docBool(doc, []string{"slurm", "all_nodes"})
|
||||
lent := docList(doc, []string{"slurm", "lent"})
|
||||
for _, n := range names {
|
||||
switch {
|
||||
case add && !all && !slices.Contains(nodes, n):
|
||||
return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n)
|
||||
case add && !slices.Contains(lent, n):
|
||||
lent = append(lent, n)
|
||||
case !add:
|
||||
lent = slices.DeleteFunc(lent, func(l string) bool { return l == n })
|
||||
}
|
||||
}
|
||||
return setList(doc, []string{"slurm", "lent"}, lent)
|
||||
}
|
||||
|
||||
// docList reads a list of strings from the settings, empty when unset.
|
||||
func docList(doc *settings.Doc, path []string) []string {
|
||||
n := doc.Node(path)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
@@ -50,6 +51,7 @@ const (
|
||||
cFederation
|
||||
cMirror
|
||||
cJobs
|
||||
cSlurm
|
||||
cLogs
|
||||
cAccounts
|
||||
cKeys
|
||||
@@ -71,7 +73,7 @@ const (
|
||||
cCount
|
||||
)
|
||||
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
|
||||
var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Knowledge", "MCP servers", "Federation", "Mirror", "Jobs", "Slurm", "Logs", "Accounts", "API keys", "Organizations", "Sign-in", "Audit", "Manual",
|
||||
"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster",
|
||||
"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
|
||||
|
||||
@@ -92,6 +94,8 @@ type consoleData struct {
|
||||
// federation is the other llmgw clusters whose models are offered
|
||||
// here; empty on a control plane without a store.
|
||||
federation adminapi.FederationView
|
||||
// slurm is the Slurm cluster beside llmgw; off when it has none.
|
||||
slurm slurmsvc.Status
|
||||
// presets is the models that come with llmgw as they stand here; nil
|
||||
// from a control plane that cannot say.
|
||||
presets *adminapi.PresetsView
|
||||
@@ -284,6 +288,7 @@ func (m consoleModel) fetch() tea.Cmd {
|
||||
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
|
||||
_ = c.get(ctx, "/api/v1/audit?since=7d&limit=300", &d.audit)
|
||||
_ = c.get(ctx, "/api/v1/federation", &d.federation)
|
||||
_ = c.get(ctx, "/api/v1/slurm", &d.slurm)
|
||||
var presets adminapi.PresetsView
|
||||
if c.get(ctx, "/api/v1/presets", &presets) == nil {
|
||||
d.presets = &presets
|
||||
@@ -588,6 +593,8 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cFederation:
|
||||
m.dialog = "error"
|
||||
m.dialogTitle, m.dialogText = peerDialog(m.data.federation.Peers[m.row])
|
||||
case cSlurm:
|
||||
m.menuD = m.slurmMenu(m.data.slurm.Nodes[m.row])
|
||||
case cServe:
|
||||
return m, m.fetchRepo(m.hf.models[m.row].ID)
|
||||
case cAccounts:
|
||||
@@ -1110,6 +1117,8 @@ func (m consoleModel) rowCount() int {
|
||||
return len(m.data.audit)
|
||||
case cFederation:
|
||||
return len(m.data.federation.Peers)
|
||||
case cSlurm:
|
||||
return len(m.data.slurm.Nodes)
|
||||
case cServe:
|
||||
return len(m.hf.models)
|
||||
case cAccounts:
|
||||
@@ -1369,6 +1378,8 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
return m.auditPanel(pw)
|
||||
case cFederation:
|
||||
return m.federationPanel(pw)
|
||||
case cSlurm:
|
||||
return m.slurmPanel(pw)
|
||||
case cMonitor:
|
||||
return m.monitorPanel(pw)
|
||||
case cServe:
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
// The console's Slurm view: the Slurm cluster run beside llmgw (BACKLOG P),
|
||||
// each of its nodes and whose it is now, as `llmgw ctl slurm status` shows
|
||||
// them. A node is lent to Slurm, or taken back, from its row. Turning Slurm
|
||||
// on, and choosing its nodes, is `ctl slurm setup` or server.yaml's slurm
|
||||
// part: a form for it would be server.yaml written again.
|
||||
|
||||
func (m consoleModel) slurmPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ node · Enter lend or take back · ← back · Q leave"
|
||||
}
|
||||
st := m.data.slurm
|
||||
head := "\n " + sDim.Render("Slurm beside llmgw, on its nodes: training and batch jobs.")
|
||||
if !st.Enabled {
|
||||
return "Slurm", head + "\n\n " + sDim.Render("Slurm is off. Turn it on with: "+
|
||||
"llmgw ctl slurm setup --controller NODE --nodes NODE,... (or --all-nodes)"), keys
|
||||
}
|
||||
ctl := sDim.Render("not heard from yet")
|
||||
switch c := st.Ctld; {
|
||||
case c == nil:
|
||||
case c.Error != "":
|
||||
ctl = sBad.Render(c.Error)
|
||||
case c.SlurmctldRunning:
|
||||
ctl = sOK.Render("slurmctld " + c.Version + " running")
|
||||
default:
|
||||
ctl = sBad.Render("slurmctld not running")
|
||||
}
|
||||
head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl)
|
||||
head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller))
|
||||
head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.")
|
||||
if len(st.Waiting) > 0 {
|
||||
head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting))
|
||||
}
|
||||
head += "\n\n"
|
||||
if len(st.Nodes) == 0 {
|
||||
return "Slurm", head + " " + sDim.Render("No node runs slurmd."), keys
|
||||
}
|
||||
var rows [][]string
|
||||
for _, n := range st.Nodes {
|
||||
rows = append(rows, slurmRow(n))
|
||||
}
|
||||
return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
func slurmRow(n slurmsvc.NodeStatus) []string {
|
||||
state, jobs, slurmd, note := "-", "-", "-", n.Describe()
|
||||
if r := n.Status; r != nil {
|
||||
state = or(r.State, "-")
|
||||
jobs = fmt.Sprint(r.Jobs)
|
||||
slurmd = sBad.Render("stopped")
|
||||
if r.SlurmdRunning {
|
||||
slurmd = sOK.Render("running")
|
||||
}
|
||||
if r.Error != "" {
|
||||
note = sBad.Render(r.Error)
|
||||
}
|
||||
}
|
||||
if !n.Connected {
|
||||
note = sWarn.Render("agent not connected") + "; " + note
|
||||
}
|
||||
whose := string(n.Phase)
|
||||
switch n.Phase {
|
||||
case slurmsvc.Lent:
|
||||
whose = sOK.Render("Slurm's")
|
||||
case slurmsvc.Llmgw:
|
||||
whose = "llmgw's"
|
||||
case slurmsvc.Lending, slurmsvc.Reclaiming:
|
||||
whose = sWarn.Render(whose)
|
||||
}
|
||||
return []string{n.Node, whose, state, jobs, slurmd, note}
|
||||
}
|
||||
|
||||
// slurmMenu is what can be done with one of Slurm's nodes.
|
||||
func (m consoleModel) slurmMenu(n slurmsvc.NodeStatus) *menuDialog {
|
||||
var items []menuItem
|
||||
if n.Phase == slurmsvc.Lent || n.Phase == slurmsvc.Lending {
|
||||
items = append(items, menuItem{"Take back from Slurm…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Take " + n.Node + " back",
|
||||
text: "Slurm starts no new job there. llmgw places replicas on it again once the jobs running there have ended.",
|
||||
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, false) }}
|
||||
return nil
|
||||
}})
|
||||
} else {
|
||||
items = append(items, menuItem{"Lend to Slurm…", func(m *consoleModel) tea.Cmd {
|
||||
text := "Its replicas move off first, each replacement up before the original stops; then Slurm runs jobs there."
|
||||
if n.Replicas > 0 {
|
||||
text = fmt.Sprintf("Its %d replica(s) move off first, each replacement up before the original stops; "+
|
||||
"a model with nowhere else to go stops serving. Then Slurm runs jobs there.", n.Replicas)
|
||||
}
|
||||
m.confirmD = &confirmDialog{title: "Lend " + n.Node + " to Slurm", text: text,
|
||||
yes: func(m *consoleModel) tea.Cmd { return m.slurmLend(n.Node, true) }}
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
return &menuDialog{title: n.Node, items: items}
|
||||
}
|
||||
|
||||
// slurmLend puts a node in slurm.lent, or takes it out, as `llmgw ctl slurm
|
||||
// lend|reclaim` does: the newest settings, changed and put back.
|
||||
func (m *consoleModel) slurmLend(node string, add bool) tea.Cmd {
|
||||
title, note, done := "Lend "+node, "lent to Slurm from the console", node+" is being lent to Slurm: its replicas move off first."
|
||||
if !add {
|
||||
title, note, done = "Take back "+node, "taken back from Slurm in the console",
|
||||
node+" is being taken back: llmgw has it once its Slurm jobs end."
|
||||
}
|
||||
m.busy = title
|
||||
c := m.client
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
var cur adminapi.SettingsView
|
||||
if err := c.get(ctx, "/api/v1/settings", &cur); err != nil {
|
||||
return actionDoneMsg{title: title, err: err}
|
||||
}
|
||||
doc, err := settings.FromBytes("settings", []byte(cur.Body))
|
||||
if err == nil {
|
||||
err = lendChange(doc, []string{node}, add)
|
||||
}
|
||||
var body []byte
|
||||
if err == nil {
|
||||
body, err = doc.Bytes()
|
||||
}
|
||||
if err != nil {
|
||||
return actionDoneMsg{title: title, err: err}
|
||||
}
|
||||
req, _ := json.Marshal(adminapi.SettingsRequest{Base: cur.Version, Body: string(body), Note: note})
|
||||
var v adminapi.SettingsView
|
||||
if err := c.put(ctx, "/api/v1/settings", req, &v); err != nil {
|
||||
return actionDoneMsg{title: title, err: err}
|
||||
}
|
||||
return actionDoneMsg{title: title, text: done}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
)
|
||||
|
||||
func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var put adminapi.SettingsRequest
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodGet && r.URL.Path == "/api/v1/settings":
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 7,
|
||||
Body: "slurm:\n enabled: true\n controller: node06\n nodes: [node05, cuda01]\n"})
|
||||
case r.Method == http.MethodPut && r.URL.Path == "/api/v1/settings":
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
_ = json.Unmarshal(b, &put)
|
||||
mu.Unlock()
|
||||
_ = json.NewEncoder(w).Encode(adminapi.SettingsView{Version: 8})
|
||||
default:
|
||||
_, _ = w.Write([]byte("[]"))
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
m := newConsoleModel(srv.URL, "/nonexistent/server.yaml")
|
||||
m.loaded, m.menu, m.inPanel = true, cSlurm, true
|
||||
m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu",
|
||||
Ctld: &slurmsvc.Reported{SlurmctldRunning: true, Version: "23.11.4"},
|
||||
Nodes: []slurmsvc.NodeStatus{
|
||||
{Node: "node05", Phase: slurmsvc.Lent, Connected: true,
|
||||
Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}},
|
||||
{Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true,
|
||||
Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}},
|
||||
}}
|
||||
var tm tea.Model = m
|
||||
view := tm.View()
|
||||
for _, want := range []string{"slurmctld 23.11.4 running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Errorf("the Slurm view has no %q:\n%s", want, view)
|
||||
}
|
||||
}
|
||||
|
||||
// cuda01, the second row: lent, after saying its replicas move.
|
||||
tm = press(tm, "down", "enter")
|
||||
if !strings.Contains(tm.View(), "Lend to Slurm") {
|
||||
t.Fatalf("no lend in the menu:\n%s", tm.View())
|
||||
}
|
||||
tm = press(tm, "enter")
|
||||
if !strings.Contains(tm.View(), "3 replica(s) move off first") {
|
||||
t.Fatalf("the question does not say what moves:\n%s", tm.View())
|
||||
}
|
||||
tm = pressRun(tm, "y")
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if put.Base != 7 || !strings.Contains(put.Body, "lent:") || !strings.Contains(put.Body, "cuda01") {
|
||||
t.Fatalf("the settings put back: %+v", put)
|
||||
}
|
||||
if !strings.Contains(tm.View(), "cuda01 is being lent to Slurm") {
|
||||
t.Errorf("not said:\n%s", tm.View())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleTakesALentNodeBack(t *testing.T) {
|
||||
m, _ := consoleWith(t, cSlurm)
|
||||
m.data.slurm = slurmsvc.Status{Enabled: true, Controller: "node06",
|
||||
Nodes: []slurmsvc.NodeStatus{{Node: "node05", Phase: slurmsvc.Lent, Connected: true}}}
|
||||
var tm tea.Model = m
|
||||
tm = press(tm, "enter")
|
||||
if v := tm.View(); !strings.Contains(v, "Take back from Slurm") || strings.Contains(v, "Lend to Slurm") {
|
||||
t.Fatalf("a lent node's menu:\n%s", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleSaysHowToTurnSlurmOn(t *testing.T) {
|
||||
m, _ := consoleWith(t, cSlurm)
|
||||
var tm tea.Model = m
|
||||
if !strings.Contains(tm.View(), "llmgw ctl slurm setup") {
|
||||
t.Fatalf("Slurm off says nothing of turning it on:\n%s", tm.View())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user