Slurm beside llmgw, P3 and P4: the leader hands nodes over; ctl slurm
CI / test (push) Successful in 9m40s
CI / test (push) Successful in 9m40s
internal/control/slurmsvc tells each node what it is, writes slurm.conf and gres.conf from the nodes' reports and the inventory, and moves each node of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until its last job step ends. Kept every tick from the settings and the agents' reports, so a new leader carries on. A broken Slurm never keeps a node from llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35de47444c
commit
410fe545ce
18 files changed
+1178
-11
No files matched your search
@@ -78,7 +78,9 @@ type API struct {
|
||||
// tenancy is organizations, keys and people; nil without a store.
|
||||
tenancy TenancyStore
|
||||
// train is adapter training; nil without a store and agents.
|
||||
train *TrainAdmin
|
||||
train *TrainAdmin
|
||||
// slurm is the Slurm cluster beside llmgw; nil without a store and agents.
|
||||
slurm SlurmService
|
||||
settings *Settings
|
||||
auth *AuthAdmin
|
||||
mcp MCPStore
|
||||
@@ -136,6 +138,7 @@ func (a *API) Register(mux Mux) {
|
||||
a.registerKB(mux)
|
||||
a.registerTenancy(mux)
|
||||
a.registerTrain(mux)
|
||||
a.registerSlurm(mux)
|
||||
a.registerGuard(mux)
|
||||
a.registerSettings(mux)
|
||||
a.registerAuth(mux)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/federation"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
@@ -392,6 +393,12 @@ var Routes = []Route{
|
||||
Description: "Their keys in projects they no longer reach are revoked.",
|
||||
Request: MembershipRequest{}, Response: UserView{}},
|
||||
|
||||
{Method: "GET", Path: "/api/v1/slurm", Summary: "The Slurm cluster beside llmgw, and where each of its nodes is",
|
||||
Description: "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw " +
|
||||
"(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " +
|
||||
"still running) or unknown (its agent has not said).",
|
||||
Response: slurmsvc.Status{}},
|
||||
|
||||
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
|
||||
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
|
||||
"the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.",
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
)
|
||||
|
||||
// SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader
|
||||
// keeps it.
|
||||
type SlurmService interface {
|
||||
Status() slurmsvc.Status
|
||||
}
|
||||
|
||||
// SetSlurm gives the API the Slurm service.
|
||||
func (a *API) SetSlurm(s SlurmService) { a.slurm = s }
|
||||
|
||||
func (a *API) registerSlurm(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus)
|
||||
}
|
||||
|
||||
func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
if a.slurm == nil {
|
||||
writeError(w, http.StatusNotImplemented, "Slurm needs the store and the nodes' agents, which this control plane runs without")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, a.slurm.Status())
|
||||
}
|
||||
Reference in new issue
Block a user