Slurm beside llmgw, P3 and P4: the leader hands nodes over; ctl slurm
CI / test (push) Successful in 9m40s

internal/control/slurmsvc tells each node what it is, writes slurm.conf
and gres.conf from the nodes' reports and the inventory, and moves each node
of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then
resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until
its last job step ends. Kept every tick from the settings and the agents'
reports, so a new leader carries on. A broken Slurm never keeps a node from
llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-11 12:33:14 +01:00
1 parent 35de47444c
commit 410fe545ce
18 files changed
+1178 -11

No files matched your search

+123
View File
@@ -116,6 +116,9 @@
{
"name": "settings"
},
{
"name": "slurm"
},
{
"name": "summary"
},
@@ -5078,6 +5081,34 @@
}
}
},
"/api/v1/slurm": {
"get": {
"operationId": "getSlurm",
"summary": "The Slurm cluster beside llmgw, and where each of its nodes is",
"description": "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw (drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs still running) or unknown (its agent has not said).",
"tags": [
"slurm"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/slurmsvc.Status"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/summary": {
"get": {
"operationId": "getSummary",
@@ -12178,6 +12209,98 @@
}
}
},
"slurmsvc.NodeStatus": {
"type": "object",
"description": "NodeStatus is one node as the service sees it.",
"properties": {
"node": {
"type": "string"
},
"phase": {
"type": "string"
},
"replicas": {
"type": "integer"
},
"slurm": {
"$ref": "#/components/schemas/slurmsvc.Reported"
},
"connected": {
"type": "boolean"
}
}
},
"slurmsvc.Reported": {
"type": "object",
"description": "Reported is the agent's SlurmStatus, for the admin API.",
"properties": {
"version": {
"type": "string"
},
"hardware": {
"type": "string"
},
"slurmd_running": {
"type": "boolean"
},
"slurmctld_running": {
"type": "boolean"
},
"state": {
"type": "string"
},
"reason": {
"type": "string"
},
"jobs": {
"type": "integer"
},
"error": {
"type": "string"
},
"files_sha256": {
"type": "string"
},
"at": {
"type": "string",
"format": "date-time"
}
}
},
"slurmsvc.Status": {
"type": "object",
"description": "Status is the whole Slurm cluster, for `ctl slurm status`.",
"properties": {
"enabled": {
"type": "boolean"
},
"cluster": {
"type": "string"
},
"controller": {
"type": "string"
},
"partition": {
"type": "string"
},
"controller_status": {
"$ref": "#/components/schemas/slurmsvc.Reported"
},
"nodes": {
"type": "array",
"items": {
"$ref": "#/components/schemas/slurmsvc.NodeStatus"
}
},
"waiting": {
"type": "array",
"items": {
"type": "string"
},
"description": "Waiting are nodes left out of slurm.conf until they report their hardware."
}
}
},
"store.AuditEvent": {
"type": "object",
"description": "AuditEvent is one change: who made it, through what, to what, and how it ended. Detail never holds a secret; the writer takes them out.",