ilker/llmgw
2
0 Fork 1
Code Issues Pull Requests Actions Packages Projects Releases 95 Wiki Activity

Facility-aware placement: racks, PDUs, holds and power caps

Browse Source 
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/admin-api.openapi.json
+385
View File
@@ -38,6 +38,9 @@
{
"name": "fabrics"
},
{
"name": "facility"
},
{
"name": "guard"
},
@@ -476,6 +479,118 @@
}
}
},
"/api/v1/facility": {
"get": {
"operationId": "getFacility",
"summary": "The failure domains, their power against their caps, and what is held",
"tags": [
"facility"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/FacilityView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/facility/holds": {
"post": {
"operationId": "postFacilityHolds",
"summary": "Hold a domain for maintenance: nothing new is placed in it",
"description": "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.",
"tags": [
"facility"
],
"parameters": [
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/facility/holds/{domain}": {
"delete": {
"operationId": "deleteFacilityHoldsDomain",
"summary": "Release a held domain",
"description": "Nodes a draining hold drained are made ready again.",
"tags": [
"facility"
],
"parameters": [
{
"name": "domain",
"in": "path",
"required": true,
"description": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/guard": {
"get": {
"operationId": "getGuard",
@@ -1931,6 +2046,58 @@
}
}
},
"/api/v1/nodes/{id}/labels": {
"post": {
"operationId": "postNodesIdLabels",
"summary": "Set or remove labels on a node, over its agent's own",
"description": "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.",
"tags": [
"nodes"
],
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"description": "The node's ID, as `llmgw ctl nodes` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NodeLabelsRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NodeLabelsView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/nodes/{id}/logs": {
"get": {
"operationId": "getNodesIdLogs",
@@ -3070,6 +3237,51 @@
}
}
},
"DomainView": {
"type": "object",
"description": "DomainView is one domain: its nodes, its power and whether it is held.",
"properties": {
"domain": {
"type": "string"
},
"level": {
"type": "string"
},
"name": {
"type": "string"
},
"nodes": {
"type": "array",
"items": {
"type": "string"
}
},
"cap_watts": {
"type": "number",
"description": "CapWatts is the domain's cap, LimitWatts the cap less the margin; both zero when it has none."
},
"limit_watts": {
"type": "number"
},
"other_watts": {
"type": "number"
},
"reserved_watts": {
"type": "number",
"description": "ReservedWatts is what is placed in it is estimated to draw; MeasuredWatts what its nodes' sensors read; UsedWatts what counts against the cap, the larger of the two node by node."
},
"measured_watts": {
"type": "number"
},
"used_watts": {
"type": "number"
},
"held": {
"$ref": "#/components/schemas/HoldView",
"description": "Held is set when the domain is held."
}
}
},
"ErrorView": {
"type": "object",
"description": "ErrorView is every refusal: a 4xx or 5xx status with what went wrong.",
@@ -3202,6 +3414,37 @@
}
}
},
"FacilityView": {
"type": "object",
"description": "FacilityView is the failure domains as placement sees them.",
"properties": {
"domains": {
"type": "array",
"items": {
"$ref": "#/components/schemas/DomainView"
},
"description": "Domains are every domain a node is in or a budget names, by level (site, room, row, rack, pdu) and then name."
},
"holds": {
"type": "array",
"items": {
"$ref": "#/components/schemas/HoldView"
},
"description": "Holds are every domain held, including one no node is in yet."
},
"margin": {
"type": "number",
"description": "Margin is the share of every cap kept free."
},
"unlocated": {
"type": "array",
"items": {
"type": "string"
},
"description": "Unlocated are the nodes with no location label, which placement treats as each in a rack of its own."
}
}
},
"FetchView": {
"type": "object",
"description": "FetchView is a download in flight.",
@@ -3394,6 +3637,52 @@
}
}
},
"HoldRequest": {
"type": "object",
"description": "HoldRequest holds a domain.",
"properties": {
"domain": {
"type": "string",
"description": "Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12."
},
"reason": {
"type": "string"
},
"drain": {
"type": "boolean",
"description": "Drain moves the work in the domain elsewhere too."
}
}
},
"HoldView": {
"type": "object",
"description": "HoldView is a hold, and the nodes holding or releasing it changed.",
"properties": {
"domain": {
"type": "string"
},
"reason": {
"type": "string"
},
"drain": {
"type": "boolean"
},
"by": {
"type": "string"
},
"at": {
"type": "string",
"format": "date-time"
},
"nodes": {
"type": "array",
"items": {
"type": "string"
},
"description": "Nodes are those this call drained, or made ready again on release."
}
}
},
"IBPortView": {
"type": "object",
"description": "IBPortView is one port as a page shows it.",
@@ -3804,6 +4093,51 @@
}
}
},
"NodeLabelsRequest": {
"type": "object",
"description": "NodeLabelsRequest sets and removes labels on a node.",
"properties": {
"set": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"remove": {
"type": "array",
"items": {
"type": "string"
},
"description": "Remove are keys whose label set through the control plane goes; the agent's own value for the key, if it has one, is in force again."
}
}
},
"NodeLabelsView": {
"type": "object",
"description": "NodeLabelsView is a node's labels after a change.",
"properties": {
"id": {
"type": "string"
},
"operator_labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "OperatorLabels are those set through the control plane."
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "Labels are what placement matches: the agent's with the operator's over them. Absent for a node that has not registered yet."
},
"location": {
"$ref": "#/components/schemas/facility.Location"
}
}
},
"NodeRemoved": {
"type": "object",
"description": "NodeRemoved names the node forgotten. Removed is the string \"true\".",
@@ -3966,6 +4300,25 @@
},
"description": "Unhealthy lists GPUs kept out of placement and why, which is usually the answer when a node looks idle but nothing will schedule onto it."
},
"location": {
"$ref": "#/components/schemas/facility.Location",
"description": "Location is where the node is in the building, from its llmgw.io/site, room, row, rack and pdu labels; absent when it carries none."
},
"held": {
"type": "string",
"description": "Held names the hold keeping the node out of placement, \"rack r12-07 is held: PDU swap\"; empty when none is."
},
"held_domain": {
"type": "string",
"description": "HeldDomain is the held domain, level:name, for releasing it."
},
"operator_labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "OperatorLabels are the labels set through the control plane, which Labels already includes over the agent's own."
},
"last_seen_ago": {
"type": "string"
},
@@ -5241,6 +5594,14 @@
},
"description": "NodeSelector restricts this profile to nodes carrying every one of these labels, with these values.\n\nA CPU profile otherwise fits everywhere, because there is no card to be free — which is right for a 0.5B model and wrong for a 14B one. Nothing in the inventory says how much memory a node has, so the operator says which nodes are meant for a model instead of the scheduler inferring it. Agents have reported labels since the beginning; this is what reads them."
},
"power_watts": {
"type": "number",
"description": "PowerWatts is what one replica of this profile draws, in watts, split evenly over its nodes, counted against the power cap of every rack, PDU or other domain its nodes are in (facility.budgets in server.yaml). Zero derives it: each card at the power limit it reports, each reserved core at facility.cpu_core_watts. Only matters where a domain has a cap."
},
"keep_within": {
"type": "string",
"description": "KeepWithin keeps a multi-node replica's nodes inside one rack, row, room or site, read from their llmgw.io/rack (and so on) labels: for a fabric whose leaves do not reach across racks, or cables that do not. Nodes without the label are passed over for such a profile. Empty is no constraint."
},
"weight": {
"type": "number",
"description": "Weight is the router's capacity weight. It should come from ctl bench rather than a guess (plan §6.2.1)."
@@ -5634,6 +5995,30 @@
}
}
},
"facility.Location": {
"type": "object",
"description": "Location is where a node is, read from its labels. Empty fields are not known, which places as it always did.",
"properties": {
"site": {
"type": "string"
},
"room": {
"type": "string"
},
"row": {
"type": "string"
},
"rack": {
"type": "string"
},
"pdus": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"guard.Attachment": {
"type": "object",
"description": "Attachment ties a policy to a part of the tenancy tree: scope is org, team, project, user or key, and ScopeID which one.",