Slurm beside llmgw, P3 and P4: the leader hands nodes over; ctl slurm
CI / test (push) Successful in 9m40s

internal/control/slurmsvc tells each node what it is, writes slurm.conf
and gres.conf from the nodes' reports and the inventory, and moves each node
of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then
resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until
its last job step ends. Kept every tick from the settings and the agents'
reports, so a new leader carries on. A broken Slurm never keeps a node from
llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-11 12:33:14 +01:00
1 parent 35de47444c
commit 410fe545ce
18 files changed
+1178 -11

No files matched your search

+4 -1
View File
@@ -78,7 +78,9 @@ type API struct {
// tenancy is organizations, keys and people; nil without a store.
tenancy TenancyStore
// train is adapter training; nil without a store and agents.
train *TrainAdmin
train *TrainAdmin
// slurm is the Slurm cluster beside llmgw; nil without a store and agents.
slurm SlurmService
settings *Settings
auth *AuthAdmin
mcp MCPStore
@@ -136,6 +138,7 @@ func (a *API) Register(mux Mux) {
a.registerKB(mux)
a.registerTenancy(mux)
a.registerTrain(mux)
a.registerSlurm(mux)
a.registerGuard(mux)
a.registerSettings(mux)
a.registerAuth(mux)
+7
View File
@@ -12,6 +12,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
@@ -392,6 +393,12 @@ var Routes = []Route{
Description: "Their keys in projects they no longer reach are revoked.",
Request: MembershipRequest{}, Response: UserView{}},
{Method: "GET", Path: "/api/v1/slurm", Summary: "The Slurm cluster beside llmgw, and where each of its nodes is",
Description: "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw " +
"(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " +
"still running) or unknown (its agent has not said).",
Response: slurmsvc.Status{}},
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
"the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.",
+28
View File
@@ -0,0 +1,28 @@
package adminapi
import (
"net/http"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
)
// SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader
// keeps it.
type SlurmService interface {
Status() slurmsvc.Status
}
// SetSlurm gives the API the Slurm service.
func (a *API) SetSlurm(s SlurmService) { a.slurm = s }
func (a *API) registerSlurm(mux Mux) {
mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus)
}
func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) {
if a.slurm == nil {
writeError(w, http.StatusNotImplemented, "Slurm needs the store and the nodes' agents, which this control plane runs without")
return
}
writeJSON(w, http.StatusOK, a.slurm.Status())
}