Slurm beside llmgw, P3 and P4: the leader hands nodes over; ctl slurm
CI / test (push) Successful in 9m40s
CI / test (push) Successful in 9m40s
internal/control/slurmsvc tells each node what it is, writes slurm.conf and gres.conf from the nodes' reports and the inventory, and moves each node of slurm.nodes between llmgw and Slurm: lent, drained of replicas first, then resumed in Slurm; taken back, drained in Slurm and cordoned in llmgw until its last job step ends. Kept every tick from the settings and the agents' reports, so a new leader carries on. A broken Slurm never keeps a node from llmgw. GET /api/v1/slurm; ctl slurm status, setup, lend, reclaim, off. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
35de47444c
commit
410fe545ce
18 files changed
+1178
-11
No files matched your search
@@ -116,6 +116,9 @@
|
||||
{
|
||||
"name": "settings"
|
||||
},
|
||||
{
|
||||
"name": "slurm"
|
||||
},
|
||||
{
|
||||
"name": "summary"
|
||||
},
|
||||
@@ -5078,6 +5081,34 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/slurm": {
|
||||
"get": {
|
||||
"operationId": "getSlurm",
|
||||
"summary": "The Slurm cluster beside llmgw, and where each of its nodes is",
|
||||
"description": "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw (drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs still running) or unknown (its agent has not said).",
|
||||
"tags": [
|
||||
"slurm"
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/slurmsvc.Status"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"401": {
|
||||
"$ref": "#/components/responses/Unauthorized"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/Error"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/v1/summary": {
|
||||
"get": {
|
||||
"operationId": "getSummary",
|
||||
@@ -12178,6 +12209,98 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"slurmsvc.NodeStatus": {
|
||||
"type": "object",
|
||||
"description": "NodeStatus is one node as the service sees it.",
|
||||
"properties": {
|
||||
"node": {
|
||||
"type": "string"
|
||||
},
|
||||
"phase": {
|
||||
"type": "string"
|
||||
},
|
||||
"replicas": {
|
||||
"type": "integer"
|
||||
},
|
||||
"slurm": {
|
||||
"$ref": "#/components/schemas/slurmsvc.Reported"
|
||||
},
|
||||
"connected": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"slurmsvc.Reported": {
|
||||
"type": "object",
|
||||
"description": "Reported is the agent's SlurmStatus, for the admin API.",
|
||||
"properties": {
|
||||
"version": {
|
||||
"type": "string"
|
||||
},
|
||||
"hardware": {
|
||||
"type": "string"
|
||||
},
|
||||
"slurmd_running": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"slurmctld_running": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"state": {
|
||||
"type": "string"
|
||||
},
|
||||
"reason": {
|
||||
"type": "string"
|
||||
},
|
||||
"jobs": {
|
||||
"type": "integer"
|
||||
},
|
||||
"error": {
|
||||
"type": "string"
|
||||
},
|
||||
"files_sha256": {
|
||||
"type": "string"
|
||||
},
|
||||
"at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
}
|
||||
}
|
||||
},
|
||||
"slurmsvc.Status": {
|
||||
"type": "object",
|
||||
"description": "Status is the whole Slurm cluster, for `ctl slurm status`.",
|
||||
"properties": {
|
||||
"enabled": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"cluster": {
|
||||
"type": "string"
|
||||
},
|
||||
"controller": {
|
||||
"type": "string"
|
||||
},
|
||||
"partition": {
|
||||
"type": "string"
|
||||
},
|
||||
"controller_status": {
|
||||
"$ref": "#/components/schemas/slurmsvc.Reported"
|
||||
},
|
||||
"nodes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/slurmsvc.NodeStatus"
|
||||
}
|
||||
},
|
||||
"waiting": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "Waiting are nodes left out of slurm.conf until they report their hardware."
|
||||
}
|
||||
}
|
||||
},
|
||||
"store.AuditEvent": {
|
||||
"type": "object",
|
||||
"description": "AuditEvent is one change: who made it, through what, to what, and how it ended. Detail never holds a secret; the writer takes them out.",
|
||||
|
||||
Reference in new issue
Block a user