Merge C24 facility-aware placement

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:18:20 +02:00
commit 3ec90ded72
42 files changed
+3510 -63

No files matched your search

+385
View File
@@ -38,6 +38,9 @@
{
"name": "fabrics"
},
{
"name": "facility"
},
{
"name": "guard"
},
@@ -476,6 +479,118 @@
}
}
},
"/api/v1/facility": {
"get": {
"operationId": "getFacility",
"summary": "The failure domains, their power against their caps, and what is held",
"tags": [
"facility"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/FacilityView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/facility/holds": {
"post": {
"operationId": "postFacilityHolds",
"summary": "Hold a domain for maintenance: nothing new is placed in it",
"description": "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.",
"tags": [
"facility"
],
"parameters": [
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/facility/holds/{domain}": {
"delete": {
"operationId": "deleteFacilityHoldsDomain",
"summary": "Release a held domain",
"description": "Nodes a draining hold drained are made ready again.",
"tags": [
"facility"
],
"parameters": [
{
"name": "domain",
"in": "path",
"required": true,
"description": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HoldView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/guard": {
"get": {
"operationId": "getGuard",
@@ -1931,6 +2046,58 @@
}
}
},
"/api/v1/nodes/{id}/labels": {
"post": {
"operationId": "postNodesIdLabels",
"summary": "Set or remove labels on a node, over its agent's own",
"description": "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.",
"tags": [
"nodes"
],
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"description": "The node's ID, as `llmgw ctl nodes` lists it.",
"schema": {
"type": "string"
}
},
{
"$ref": "#/components/parameters/Actor"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NodeLabelsRequest"
}
}
}
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/NodeLabelsView"
}
}
}
},
"401": {
"$ref": "#/components/responses/Unauthorized"
},
"default": {
"$ref": "#/components/responses/Error"
}
}
}
},
"/api/v1/nodes/{id}/logs": {
"get": {
"operationId": "getNodesIdLogs",
@@ -3070,6 +3237,51 @@
}
}
},
"DomainView": {
"type": "object",
"description": "DomainView is one domain: its nodes, its power and whether it is held.",
"properties": {
"domain": {
"type": "string"
},
"level": {
"type": "string"
},
"name": {
"type": "string"
},
"nodes": {
"type": "array",
"items": {
"type": "string"
}
},
"cap_watts": {
"type": "number",
"description": "CapWatts is the domain's cap, LimitWatts the cap less the margin; both zero when it has none."
},
"limit_watts": {
"type": "number"
},
"other_watts": {
"type": "number"
},
"reserved_watts": {
"type": "number",
"description": "ReservedWatts is what is placed in it is estimated to draw; MeasuredWatts what its nodes' sensors read; UsedWatts what counts against the cap, the larger of the two node by node."
},
"measured_watts": {
"type": "number"
},
"used_watts": {
"type": "number"
},
"held": {
"$ref": "#/components/schemas/HoldView",
"description": "Held is set when the domain is held."
}
}
},
"ErrorView": {
"type": "object",
"description": "ErrorView is every refusal: a 4xx or 5xx status with what went wrong.",
@@ -3202,6 +3414,37 @@
}
}
},
"FacilityView": {
"type": "object",
"description": "FacilityView is the failure domains as placement sees them.",
"properties": {
"domains": {
"type": "array",
"items": {
"$ref": "#/components/schemas/DomainView"
},
"description": "Domains are every domain a node is in or a budget names, by level (site, room, row, rack, pdu) and then name."
},
"holds": {
"type": "array",
"items": {
"$ref": "#/components/schemas/HoldView"
},
"description": "Holds are every domain held, including one no node is in yet."
},
"margin": {
"type": "number",
"description": "Margin is the share of every cap kept free."
},
"unlocated": {
"type": "array",
"items": {
"type": "string"
},
"description": "Unlocated are the nodes with no location label, which placement treats as each in a rack of its own."
}
}
},
"FetchView": {
"type": "object",
"description": "FetchView is a download in flight.",
@@ -3394,6 +3637,52 @@
}
}
},
"HoldRequest": {
"type": "object",
"description": "HoldRequest holds a domain.",
"properties": {
"domain": {
"type": "string",
"description": "Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12."
},
"reason": {
"type": "string"
},
"drain": {
"type": "boolean",
"description": "Drain moves the work in the domain elsewhere too."
}
}
},
"HoldView": {
"type": "object",
"description": "HoldView is a hold, and the nodes holding or releasing it changed.",
"properties": {
"domain": {
"type": "string"
},
"reason": {
"type": "string"
},
"drain": {
"type": "boolean"
},
"by": {
"type": "string"
},
"at": {
"type": "string",
"format": "date-time"
},
"nodes": {
"type": "array",
"items": {
"type": "string"
},
"description": "Nodes are those this call drained, or made ready again on release."
}
}
},
"IBPortView": {
"type": "object",
"description": "IBPortView is one port as a page shows it.",
@@ -3804,6 +4093,51 @@
}
}
},
"NodeLabelsRequest": {
"type": "object",
"description": "NodeLabelsRequest sets and removes labels on a node.",
"properties": {
"set": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"remove": {
"type": "array",
"items": {
"type": "string"
},
"description": "Remove are keys whose label set through the control plane goes; the agent's own value for the key, if it has one, is in force again."
}
}
},
"NodeLabelsView": {
"type": "object",
"description": "NodeLabelsView is a node's labels after a change.",
"properties": {
"id": {
"type": "string"
},
"operator_labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "OperatorLabels are those set through the control plane."
},
"labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "Labels are what placement matches: the agent's with the operator's over them. Absent for a node that has not registered yet."
},
"location": {
"$ref": "#/components/schemas/facility.Location"
}
}
},
"NodeRemoved": {
"type": "object",
"description": "NodeRemoved names the node forgotten. Removed is the string \"true\".",
@@ -3966,6 +4300,25 @@
},
"description": "Unhealthy lists GPUs kept out of placement and why, which is usually the answer when a node looks idle but nothing will schedule onto it."
},
"location": {
"$ref": "#/components/schemas/facility.Location",
"description": "Location is where the node is in the building, from its llmgw.io/site, room, row, rack and pdu labels; absent when it carries none."
},
"held": {
"type": "string",
"description": "Held names the hold keeping the node out of placement, \"rack r12-07 is held: PDU swap\"; empty when none is."
},
"held_domain": {
"type": "string",
"description": "HeldDomain is the held domain, level:name, for releasing it."
},
"operator_labels": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"description": "OperatorLabels are the labels set through the control plane, which Labels already includes over the agent's own."
},
"last_seen_ago": {
"type": "string"
},
@@ -5245,6 +5598,14 @@
},
"description": "NodeSelector restricts this profile to nodes carrying every one of these labels, with these values.\n\nA CPU profile otherwise fits everywhere, because there is no card to be free — which is right for a 0.5B model and wrong for a 14B one. Nothing in the inventory says how much memory a node has, so the operator says which nodes are meant for a model instead of the scheduler inferring it. Agents have reported labels since the beginning; this is what reads them."
},
"power_watts": {
"type": "number",
"description": "PowerWatts is what one replica of this profile draws, in watts, split evenly over its nodes, counted against the power cap of every rack, PDU or other domain its nodes are in (facility.budgets in server.yaml). Zero derives it: each card at the power limit it reports, each reserved core at facility.cpu_core_watts. Only matters where a domain has a cap."
},
"keep_within": {
"type": "string",
"description": "KeepWithin keeps a multi-node replica's nodes inside one rack, row, room or site, read from their llmgw.io/rack (and so on) labels: for a fabric whose leaves do not reach across racks, or cables that do not. Nodes without the label are passed over for such a profile. Empty is no constraint."
},
"weight": {
"type": "number",
"description": "Weight is the router's capacity weight. It should come from ctl bench rather than a guess (plan §6.2.1)."
@@ -5638,6 +5999,30 @@
}
}
},
"facility.Location": {
"type": "object",
"description": "Location is where a node is, read from its labels. Empty fields are not known, which places as it always did.",
"properties": {
"site": {
"type": "string"
},
"room": {
"type": "string"
},
"row": {
"type": "string"
},
"rack": {
"type": "string"
},
"pdus": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"guard.Attachment": {
"type": "object",
"description": "Attachment ties a policy to a part of the tenancy tree: scope is org, team, project, user or key, and ScopeID which one.",
+10 -3
View File
@@ -83,11 +83,18 @@ installs. **N** sends a test notification to each webhook in server.yaml's
`notify` section and says how each answered (`llmgw ctl notify test`). A node's and a model's own pages have their charts too, over the
same range.
**Nodes** lists every node with its state, load, GPUs, replicas, disk and
agent version. On a row:
**Nodes** lists every node with its state, where it is (its rack, or row,
room or site, and PDUs), load, GPUs, replicas, disk and agent version. A node
in a held rack reads *ready, held*. On a row:
- **Details** — its address, system, load, engines and GPUs, and any warning.
- **Details** — its address, location and any hold, system, load, engines and
GPUs, and any warning.
- **Cordon** places nothing new on it; **Uncordon** undoes it.
- **Location…** sets its site, room, row, rack and PDUs (the `llmgw.io/`
labels), which placement spreads replicas across.
- **Hold rack …** places nothing new in the node's rack, and can drain it;
**Release …** on a node in a held domain undoes it. See
[facility-aware-placement.md](facility-aware-placement.md).
- **Drain** moves its replicas off one at a time, each replacement up before
the original stops.
- **vllm build / sglang build** (nodes with GPUs) — which build of the engine
+109 -12
View File
@@ -1,13 +1,109 @@
# Facility-aware placement: rack power, temperature and cooling
**Status: step 0, reading the nodes' own sensors, is built; steps 1–5 are
deferred** until the rack, PDU and power information of a real site is at
hand — without it they would be designed against guesses.
This is the design for step 5 of resource allocation, after reserved CPU cores
and memory (1), capacity classes and preemption (2), tenant capacity quotas (3)
and sizing by measured throughput (4), all of which shipped in v0.10.5. Of
what follows, only the node sensors (rollout step 0) exist in the code: they
are read, published and shown, and change no decision.
**Status: steps 0, 1 and 2 are built, with the manual hold of step 4;
temperature conditions (steps 3 and 4) and shedding (step 5) are not**,
until the thermal data of a real site is at hand — without it thresholds would
be designed against guesses. This is the design for step 5 of resource
allocation, after reserved CPU cores and memory (1), capacity classes and
preemption (2), tenant capacity quotas (3) and sizing by measured throughput
(4). [What is built](#what-is-built) says what works today and where it
departs from the design below; the rest of the document is the design.
## What is built
**Where a node is.** Five labels: `llmgw.io/site`, `room`, `row`, `rack` and
`pdu` (a comma separated list, for a dual-fed rack). They come from the node's
`agent.yaml` under `labels:`, or from the control plane:
```
llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b
llmgw ctl label gpu01 llmgw.io/rack- # back to agent.yaml's value, if any
```
Labels set with `ctl label` are kept in the store, win over the agent's, and
may be set before the node joins. A misspelt `llmgw.io/` key is refused. A
domain is named `level:name` — `rack:r12-07`, `pdu:r12-07-a` — and names are
taken as given, so two rooms that both have a rack `r1` must label them apart.
A node without location labels is in no domain and places as it always did.
**Spreading.** A model's next replica goes first to a node sharing no rack and
no PDU with the nodes the model already runs on; nodes sharing fewer come
before nodes sharing more. It is an ordering, never a filter: a rack holding
the only free cards still takes the replica. It ranks above preferring a node
that already holds the weights — a second replica one breaker away from the
first is worth less than the copy saved — and above bin packing.
**Keeping a replica together.** A profile's `keep_within: rack` (or `row`,
`room`, `site`) keeps a multi-node replica's nodes in one such domain, inside
the fabric or NVLink domain placement already requires; nodes without the
label are passed over for it, with the reason.
**Holds.** `llmgw ctl facility hold rack:r12-07 --reason "PDU swap"` places
nothing new in the domain, as a cordon of each of its nodes would, including
nodes labelled into it later. `--drain` also drains its nodes, one replica at
a time with the replacement up first; `llmgw ctl facility release` undoes
both. Holds are kept in the store across restarts (the drain itself is node
state, which is not: after a restart the hold still keeps work out, and
replicas not yet moved stay where they are). Any domain can be held: a PDU
holds every rack on it.
**Power budgets.** In `server.yaml`:
```yaml
facility:
margin: 0.1 # share of every cap kept free
gpu_watts: 700 # a card that reports no power limit
cpu_core_watts: 10 # a core reserved by a CPU replica (cpu_cores)
node_watts: 350 # each node's own draw beyond its cards
budgets:
- domain: rack:r12-07
cap_watts: 16000 # what one feed carries alone, derated
other_watts: 800 # switches, storage
- domain: pdu:r12-07-a
cap_watts: 7400
```
A replica's draw is the profile's `power_watts` (split over its nodes) when
declared; otherwise each card at the power limit it reports through the agent's
sensors (or `gpu_watts`), and each reserved core at `cpu_core_watts`. A
domain's use is, node by node, `node_watts` plus the larger of what is placed
there and what the node's sensors read in the last two minutes
(`measured_fresh`), plus the domain's `other_watts`. A shard that would take
any of its node's domains past `cap × (1 − margin)` is refused; a multi-node
replica's shards are counted together, so two that each fit a rack alone do not
both go into it. A draw nobody knows — no declared figure, no card limit, no
`gpu_watts` — counts as nothing: missing data never blocks.
**What people see.** `llmgw ctl explain-placement` names the domain: *rack
r12-07 is held: PDU swap*, *rack r12-07: 14.2 kW of a 14.4 kW limit (16.0 kW
cap less 10%) in use, the shard needs ~2.8 kW*. The rejections of a placed
replica are kept too, so a replica in an unexpected rack says why. `llmgw ctl
nodes` has a LOCATION column, shows *ready, held*, and filters with `--domain
rack:r12-07`. `llmgw ctl facility` lists every domain with its nodes, use,
limit, reserved and measured draw, and holds. The console's Nodes view has a
WHERE column; a node's page shows its location and hold, and its menu sets
the location (**Location…**) and holds or releases its rack. The admin API:
`GET /api/v1/facility`, `POST /api/v1/facility/holds`, `DELETE
/api/v1/facility/holds/{domain}`, `POST /api/v1/nodes/{id}/labels`.
**Where it departs from the design below, and why.**
- Domains come from labels alone; there is no facility file declaring them.
Budgets are in `server.yaml`, the one file the control plane already reads.
A DCIM export would generate that block.
- Feeds are not modelled. The cap given is what one feed carries alone,
derated — the figure the design says matters — so the operator gives the
one number instead of llmgw deriving it from two.
- The manual hold of step 4 is built now, without conditions: it needs no
sensor data and answers the commonest facility event, planned maintenance.
Its command is `ctl facility hold`, with a `--drain` the design did not have.
- `keep_within` is new: the request that a multi-node replica stay in one rack
where the cabling requires it.
- Spreading is an ordering of candidates, not the scoring of the design; power
headroom does not yet order racks ("level load across racks"), and
multi-node replicas are not spread across racks.
- No `llmgw_domain_*` metrics or alerts yet; `ctl facility` and the API carry
the same figures.
## Why
@@ -260,16 +356,17 @@ Each step ships on its own and is useful without the next:
0. **Node sensors, observe-only (built).** GPU heat, power and throttling,
CPU heat and power, inlet air where the board has a sensor; published and
shown, no decision changed.
1. **Location and spreading.** Well-known labels, domains from labels, replicas
1. **Location and spreading (built).** Well-known labels, domains from labels, replicas
spread across racks and PDUs, locations on the node views. No facility data
needed.
2. **Power budgets from a static file.** Caps and margins, estimated draw per
2. **Power budgets from a static file (built, without headroom
scoring).** Caps and margins, estimated draw per
replica (declared or derived), hard limit in placement and headroom.
3. **Readings, observe-only.** The Prometheus source beside the agents',
conditions with hysteresis (throttling first among them), measured power
folded into headroom; decisions shown, not enforced.
4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and
release.
4. **Thermal enforcement.** `warm` scored down, `hot` refused; the manual hold
and release are built.
5. **Shedding and tokens per watt.** Opt-in shedding through step 2's
preemption; measured power in profile preference.
+23 -2
View File
@@ -862,8 +862,8 @@ llmgw ctl nodes
```
```
NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
llmgw-node01 ready 8/8 free 503G h100-80g ib-a 0 2s
NODE STATE LOCATION GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN
llmgw-node01 ready - 8/8 free 503G h100-80g ib-a 0 2s
```
If it does not appear, the control plane's log says why. A certificate whose
@@ -875,6 +875,27 @@ Repeat for each node. Nodes in the same fabric must be given the same `--fabric`
name; nodes with no InfiniBand take no `--fabric` at all and are each their own
island, which is fine for single-node models.
### Saying where the node is
On more than one rack, tell llmgw which rack, and which PDUs, each node is in.
Placement then keeps a model's replicas in different racks and on different
PDUs, places nothing in a rack held for maintenance, and keeps each rack under
a power cap if you give it one:
```
llmgw ctl label llmgw-node01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b
llmgw ctl label llmgw-node01 llmgw.io/row=12 llmgw.io/room=hall-b llmgw.io/site=sto1
```
Labels set this way are kept by the control plane and win over the node's
`agent.yaml`, where the same keys can go under `labels:` instead; a node need
not have joined yet. `llmgw ctl nodes` shows the LOCATION, `--domain
rack:r12-07` lists one rack, and `llmgw ctl facility` lists every rack, row and
PDU with what it draws. In the console it is the node's menu, **Location…**.
A node with no location is placed as before, as a rack of its own.
See [facility-aware-placement.md](facility-aware-placement.md) for power caps
and holds.
## 6. Setting the node's CPU model, if it is a VM
Proxmox defaults a VM to `kvm64`, which hides AVX, AVX2, FMA and even SSE4.2
+113
View File
@@ -86,6 +86,15 @@ rather than repeated.
| `url` | `string` | [External](#external) |
| `model` | `string` | [External](#external) |
| `api_key_file` | `string` | [External](#external) |
| `budgets` | `[]FacilityBudget` | [Facility](#facility) |
| `margin` | `float64` | [Facility](#facility) |
| `gpu_watts` | `float64` | [Facility](#facility) |
| `cpu_core_watts` | `float64` | [Facility](#facility) |
| `node_watts` | `float64` | [Facility](#facility) |
| `measured_fresh` | `time.Duration` | [Facility](#facility) |
| `domain` | `string` | [FacilityBudget](#facilitybudget) |
| `cap_watts` | `float64` | [FacilityBudget](#facilitybudget) |
| `other_watts` | `float64` | [FacilityBudget](#facilitybudget) |
| `format` | `string` | [GuardModel](#guardmodel) |
| `categories` | `[]string` | [GuardModel](#guardmodel) |
| `model` | `string` | [JudgePolicy](#judgepolicy) |
@@ -198,6 +207,8 @@ rather than repeated.
| `cpu_cores` | `int` | [Profile](#profile) |
| `min_disk_gb` | `int` | [Profile](#profile) |
| `node_selector` | `map[string]string` | [Profile](#profile) |
| `power_watts` | `float64` | [Profile](#profile) |
| `keep_within` | `string` | [Profile](#profile) |
| `weight` | `float64` | [Profile](#profile) |
| `priority` | `int` | [Profile](#profile) |
| `startup_timeout` | `time.Duration` | [Profile](#profile) |
@@ -276,6 +287,7 @@ rather than repeated.
| `router` | `Router` | [Server](#server) |
| `control` | `Control` | [Server](#server) |
| `models` | `[]catalog.Model` | [Server](#server) |
| `facility` | `Facility` | [Server](#server) |
| `endpoints` | `[]Endpoint` | [Server](#server) |
| `bin` | `string` | [SimConfig](#simconfig) |
| `python` | `string` | [TRTLLMConfig](#trtllmconfig) |
@@ -846,6 +858,75 @@ APIKeyFile holds the provider's key, on the control plane. A file
rather than the key itself, so the key is in neither the catalog, the
store nor the audit log; empty for a provider that asks for none.
## Facility
### `budgets`
`[]FacilityBudget`
Budgets are the domains with a power cap.
### `margin`
`float64`
Margin is the share of every cap kept free, 0.1 for ten percent. Zero
keeps none, for caps already given derated.
### `gpu_watts`
`float64`
GPUWatts is a card's draw when it does not report its power limit:
an agent without nvidia-smi's power fields, or a node that has not
sent a heartbeat yet. Zero counts such a card as drawing nothing.
### `cpu_core_watts`
`float64`
CPUCoreWatts is the draw of a core reserved by a replica with no GPU
(cpu_cores). Zero counts such replicas as drawing nothing.
### `node_watts`
`float64`
NodeWatts is what a node draws besides its cards and reserved cores —
idle CPUs, memory, fans, disks — counted for every node in a capped
domain, busy or not.
### `measured_fresh`
`time.Duration`
MeasuredFresh is how old a node's power reading may be and still be
counted against its domains. Zero takes two minutes.
## FacilityBudget
### `domain`
`string`
Domain is level:name, as the nodes' labels give it: rack:r12-07,
pdu:r12-07-a, row:12.
### `cap_watts`
`float64`
CapWatts is what the domain may draw. For a dual-fed rack it is what
one feed carries alone, derated as the facility derates it (usually
80%), since that is what the rack has to survive.
### `other_watts`
`float64`
OtherWatts is drawn in the domain by what llmgw does not place on:
switches, storage, machines outside the cluster.
## GuardModel
### `format`
@@ -1689,6 +1770,27 @@ in the inventory says how much memory a node has, so the operator says
which nodes are meant for a model instead of the scheduler inferring it.
Agents have reported labels since the beginning; this is what reads them.
### `power_watts`
`float64`
PowerWatts is what one replica of this profile draws, in watts, split
evenly over its nodes, counted against the power cap of every rack,
PDU or other domain its nodes are in (facility.budgets in server.yaml).
Zero derives it: each card at the power limit it reports, each
reserved core at facility.cpu_core_watts. Only matters where a domain
has a cap.
### `keep_within`
`string`
KeepWithin keeps a multi-node replica's nodes inside one rack, row,
room or site, read from their llmgw.io/rack (and so on) labels: for a
fabric whose leaves do not reach across racks, or cables that do not.
Nodes without the label are passed over for such a profile. Empty is
no constraint.
### `weight`
`float64`
@@ -2310,6 +2412,17 @@ Takes a [Control](#control) block.
`[]catalog.Model`
### `facility`
`Facility`
Facility is the power caps of racks, PDUs and other domains, and how
a replica's draw is estimated against them. Where a node is comes from
its llmgw.io/site, room, row, rack and pdu labels; holding a domain for
maintenance is `llmgw ctl facility hold`, not configuration.
Takes a [Facility](#facility) block.
### `endpoints`
`[]Endpoint`