From 0d3eb82b9bc3d5b4062ba9673c8f991328a47a85 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 5 Oct 2026 01:17:49 +0200 Subject: [PATCH] Facility-aware placement: racks, PDUs, holds and power caps Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/admin-api.openapi.json | 385 ++++++++++++++++++ docs/console.md | 13 +- docs/facility-aware-placement.md | 121 +++++- docs/install.md | 25 +- docs/reference.md | 113 +++++ internal/cli/ctl.go | 31 +- internal/cli/facility.go | 212 ++++++++++ internal/cli/facility_test.go | 75 ++++ internal/cli/filter.go | 20 +- internal/cli/server.go | 6 + internal/cli/tui_actions.go | 71 ++++ internal/cli/tui_console.go | 18 +- internal/config/config.go | 87 ++++ internal/config/facility_test.go | 52 +++ internal/control/adminapi/api.go | 29 ++ internal/control/adminapi/facility.go | 203 +++++++++ internal/control/adminapi/facility_test.go | 68 ++++ internal/control/adminapi/routes.go | 12 + internal/control/catalog/catalog.go | 26 ++ internal/control/control.go | 42 +- internal/control/facility.go | 231 +++++++++++ internal/control/facility/facility.go | 268 ++++++++++++ internal/control/facility/facility_test.go | 67 +++ internal/control/facility_test.go | 132 ++++++ internal/control/inventory/facility.go | 176 ++++++++ internal/control/inventory/facility_test.go | 79 ++++ internal/control/inventory/inventory.go | 37 +- internal/control/placement/facility.go | 193 +++++++++ internal/control/placement/facility_test.go | 325 +++++++++++++++ internal/control/placement/placement.go | 99 ++++- internal/control/reconciler/adopt.go | 1 + internal/control/reconciler/facility_test.go | 88 ++++ internal/control/reconciler/headroom.go | 2 +- internal/control/reconciler/reconciler.go | 13 +- internal/docgen/reference_gen.go | 16 + internal/store/copy.go | 3 + internal/store/facility.go | 93 +++++ internal/store/facility_test.go | 52 +++ .../migrations/postgres/0031_facility.sql | 24 ++ .../store/migrations/sqlite/0031_facility.sql | 23 ++ internal/store/postgres_test.go | 2 +- notes/c24-facility.md | 40 ++ 42 files changed, 3510 insertions(+), 63 deletions(-) create mode 100644 internal/cli/facility.go create mode 100644 internal/cli/facility_test.go create mode 100644 internal/config/facility_test.go create mode 100644 internal/control/adminapi/facility.go create mode 100644 internal/control/adminapi/facility_test.go create mode 100644 internal/control/facility.go create mode 100644 internal/control/facility/facility.go create mode 100644 internal/control/facility/facility_test.go create mode 100644 internal/control/facility_test.go create mode 100644 internal/control/inventory/facility.go create mode 100644 internal/control/inventory/facility_test.go create mode 100644 internal/control/placement/facility.go create mode 100644 internal/control/placement/facility_test.go create mode 100644 internal/control/reconciler/facility_test.go create mode 100644 internal/store/facility.go create mode 100644 internal/store/facility_test.go create mode 100644 internal/store/migrations/postgres/0031_facility.sql create mode 100644 internal/store/migrations/sqlite/0031_facility.sql create mode 100644 notes/c24-facility.md diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 74c449b..3fb2c32 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -38,6 +38,9 @@ { "name": "fabrics" }, + { + "name": "facility" + }, { "name": "guard" }, @@ -476,6 +479,118 @@ } } }, + "/api/v1/facility": { + "get": { + "operationId": "getFacility", + "summary": "The failure domains, their power against their caps, and what is held", + "tags": [ + "facility" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FacilityView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/api/v1/facility/holds": { + "post": { + "operationId": "postFacilityHolds", + "summary": "Hold a domain for maintenance: nothing new is placed in it", + "description": "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.", + "tags": [ + "facility" + ], + "parameters": [ + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HoldRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HoldView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/api/v1/facility/holds/{domain}": { + "delete": { + "operationId": "deleteFacilityHoldsDomain", + "summary": "Release a held domain", + "description": "Nodes a draining hold drained are made ready again.", + "tags": [ + "facility" + ], + "parameters": [ + { + "name": "domain", + "in": "path", + "required": true, + "description": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Actor" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HoldView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/guard": { "get": { "operationId": "getGuard", @@ -1931,6 +2046,58 @@ } } }, + "/api/v1/nodes/{id}/labels": { + "post": { + "operationId": "postNodesIdLabels", + "summary": "Set or remove labels on a node, over its agent's own", + "description": "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.", + "tags": [ + "nodes" + ], + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "description": "The node's ID, as `llmgw ctl nodes` lists it.", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeLabelsRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NodeLabelsView" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/nodes/{id}/logs": { "get": { "operationId": "getNodesIdLogs", @@ -3070,6 +3237,51 @@ } } }, + "DomainView": { + "type": "object", + "description": "DomainView is one domain: its nodes, its power and whether it is held.", + "properties": { + "domain": { + "type": "string" + }, + "level": { + "type": "string" + }, + "name": { + "type": "string" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + } + }, + "cap_watts": { + "type": "number", + "description": "CapWatts is the domain's cap, LimitWatts the cap less the margin; both zero when it has none." + }, + "limit_watts": { + "type": "number" + }, + "other_watts": { + "type": "number" + }, + "reserved_watts": { + "type": "number", + "description": "ReservedWatts is what is placed in it is estimated to draw; MeasuredWatts what its nodes' sensors read; UsedWatts what counts against the cap, the larger of the two node by node." + }, + "measured_watts": { + "type": "number" + }, + "used_watts": { + "type": "number" + }, + "held": { + "$ref": "#/components/schemas/HoldView", + "description": "Held is set when the domain is held." + } + } + }, "ErrorView": { "type": "object", "description": "ErrorView is every refusal: a 4xx or 5xx status with what went wrong.", @@ -3202,6 +3414,37 @@ } } }, + "FacilityView": { + "type": "object", + "description": "FacilityView is the failure domains as placement sees them.", + "properties": { + "domains": { + "type": "array", + "items": { + "$ref": "#/components/schemas/DomainView" + }, + "description": "Domains are every domain a node is in or a budget names, by level (site, room, row, rack, pdu) and then name." + }, + "holds": { + "type": "array", + "items": { + "$ref": "#/components/schemas/HoldView" + }, + "description": "Holds are every domain held, including one no node is in yet." + }, + "margin": { + "type": "number", + "description": "Margin is the share of every cap kept free." + }, + "unlocated": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Unlocated are the nodes with no location label, which placement treats as each in a rack of its own." + } + } + }, "FetchView": { "type": "object", "description": "FetchView is a download in flight.", @@ -3394,6 +3637,52 @@ } } }, + "HoldRequest": { + "type": "object", + "description": "HoldRequest holds a domain.", + "properties": { + "domain": { + "type": "string", + "description": "Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12." + }, + "reason": { + "type": "string" + }, + "drain": { + "type": "boolean", + "description": "Drain moves the work in the domain elsewhere too." + } + } + }, + "HoldView": { + "type": "object", + "description": "HoldView is a hold, and the nodes holding or releasing it changed.", + "properties": { + "domain": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "drain": { + "type": "boolean" + }, + "by": { + "type": "string" + }, + "at": { + "type": "string", + "format": "date-time" + }, + "nodes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Nodes are those this call drained, or made ready again on release." + } + } + }, "IBPortView": { "type": "object", "description": "IBPortView is one port as a page shows it.", @@ -3804,6 +4093,51 @@ } } }, + "NodeLabelsRequest": { + "type": "object", + "description": "NodeLabelsRequest sets and removes labels on a node.", + "properties": { + "set": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "remove": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Remove are keys whose label set through the control plane goes; the agent's own value for the key, if it has one, is in force again." + } + } + }, + "NodeLabelsView": { + "type": "object", + "description": "NodeLabelsView is a node's labels after a change.", + "properties": { + "id": { + "type": "string" + }, + "operator_labels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "OperatorLabels are those set through the control plane." + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Labels are what placement matches: the agent's with the operator's over them. Absent for a node that has not registered yet." + }, + "location": { + "$ref": "#/components/schemas/facility.Location" + } + } + }, "NodeRemoved": { "type": "object", "description": "NodeRemoved names the node forgotten. Removed is the string \"true\".", @@ -3966,6 +4300,25 @@ }, "description": "Unhealthy lists GPUs kept out of placement and why, which is usually the answer when a node looks idle but nothing will schedule onto it." }, + "location": { + "$ref": "#/components/schemas/facility.Location", + "description": "Location is where the node is in the building, from its llmgw.io/site, room, row, rack and pdu labels; absent when it carries none." + }, + "held": { + "type": "string", + "description": "Held names the hold keeping the node out of placement, \"rack r12-07 is held: PDU swap\"; empty when none is." + }, + "held_domain": { + "type": "string", + "description": "HeldDomain is the held domain, level:name, for releasing it." + }, + "operator_labels": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "OperatorLabels are the labels set through the control plane, which Labels already includes over the agent's own." + }, "last_seen_ago": { "type": "string" }, @@ -5241,6 +5594,14 @@ }, "description": "NodeSelector restricts this profile to nodes carrying every one of these labels, with these values.\n\nA CPU profile otherwise fits everywhere, because there is no card to be free — which is right for a 0.5B model and wrong for a 14B one. Nothing in the inventory says how much memory a node has, so the operator says which nodes are meant for a model instead of the scheduler inferring it. Agents have reported labels since the beginning; this is what reads them." }, + "power_watts": { + "type": "number", + "description": "PowerWatts is what one replica of this profile draws, in watts, split evenly over its nodes, counted against the power cap of every rack, PDU or other domain its nodes are in (facility.budgets in server.yaml). Zero derives it: each card at the power limit it reports, each reserved core at facility.cpu_core_watts. Only matters where a domain has a cap." + }, + "keep_within": { + "type": "string", + "description": "KeepWithin keeps a multi-node replica's nodes inside one rack, row, room or site, read from their llmgw.io/rack (and so on) labels: for a fabric whose leaves do not reach across racks, or cables that do not. Nodes without the label are passed over for such a profile. Empty is no constraint." + }, "weight": { "type": "number", "description": "Weight is the router's capacity weight. It should come from ctl bench rather than a guess (plan §6.2.1)." @@ -5634,6 +5995,30 @@ } } }, + "facility.Location": { + "type": "object", + "description": "Location is where a node is, read from its labels. Empty fields are not known, which places as it always did.", + "properties": { + "site": { + "type": "string" + }, + "room": { + "type": "string" + }, + "row": { + "type": "string" + }, + "rack": { + "type": "string" + }, + "pdus": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "guard.Attachment": { "type": "object", "description": "Attachment ties a policy to a part of the tenancy tree: scope is org, team, project, user or key, and ScopeID which one.", diff --git a/docs/console.md b/docs/console.md index 39c7eda..df542ca 100644 --- a/docs/console.md +++ b/docs/console.md @@ -71,11 +71,18 @@ installs. **N** sends a test notification to each webhook in server.yaml's `notify` section and says how each answered (`llmgw ctl notify test`). A node's and a model's own pages have their charts too, over the same range. -**Nodes** lists every node with its state, load, GPUs, replicas, disk and -agent version. On a row: +**Nodes** lists every node with its state, where it is (its rack, or row, +room or site, and PDUs), load, GPUs, replicas, disk and agent version. A node +in a held rack reads *ready, held*. On a row: -- **Details** — its address, system, load, engines and GPUs, and any warning. +- **Details** — its address, location and any hold, system, load, engines and + GPUs, and any warning. - **Cordon** places nothing new on it; **Uncordon** undoes it. +- **Location…** sets its site, room, row, rack and PDUs (the `llmgw.io/` + labels), which placement spreads replicas across. +- **Hold rack …** places nothing new in the node's rack, and can drain it; + **Release …** on a node in a held domain undoes it. See + [facility-aware-placement.md](facility-aware-placement.md). - **Drain** moves its replicas off one at a time, each replacement up before the original stops. - **vllm build / sglang build** (nodes with GPUs) — which build of the engine diff --git a/docs/facility-aware-placement.md b/docs/facility-aware-placement.md index a57efab..e2b0112 100644 --- a/docs/facility-aware-placement.md +++ b/docs/facility-aware-placement.md @@ -1,13 +1,109 @@ # Facility-aware placement: rack power, temperature and cooling -**Status: step 0, reading the nodes' own sensors, is built; steps 1–5 are -deferred** until the rack, PDU and power information of a real site is at -hand — without it they would be designed against guesses. -This is the design for step 5 of resource allocation, after reserved CPU cores -and memory (1), capacity classes and preemption (2), tenant capacity quotas (3) -and sizing by measured throughput (4), all of which shipped in v0.10.5. Of -what follows, only the node sensors (rollout step 0) exist in the code: they -are read, published and shown, and change no decision. +**Status: steps 0, 1 and 2 are built, with the manual hold of step 4; +temperature conditions (steps 3 and 4) and shedding (step 5) are not**, +until the thermal data of a real site is at hand — without it thresholds would +be designed against guesses. This is the design for step 5 of resource +allocation, after reserved CPU cores and memory (1), capacity classes and +preemption (2), tenant capacity quotas (3) and sizing by measured throughput +(4). [What is built](#what-is-built) says what works today and where it +departs from the design below; the rest of the document is the design. + +## What is built + +**Where a node is.** Five labels: `llmgw.io/site`, `room`, `row`, `rack` and +`pdu` (a comma separated list, for a dual-fed rack). They come from the node's +`agent.yaml` under `labels:`, or from the control plane: + +``` +llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b +llmgw ctl label gpu01 llmgw.io/rack- # back to agent.yaml's value, if any +``` + +Labels set with `ctl label` are kept in the store, win over the agent's, and +may be set before the node joins. A misspelt `llmgw.io/` key is refused. A +domain is named `level:name` — `rack:r12-07`, `pdu:r12-07-a` — and names are +taken as given, so two rooms that both have a rack `r1` must label them apart. +A node without location labels is in no domain and places as it always did. + +**Spreading.** A model's next replica goes first to a node sharing no rack and +no PDU with the nodes the model already runs on; nodes sharing fewer come +before nodes sharing more. It is an ordering, never a filter: a rack holding +the only free cards still takes the replica. It ranks above preferring a node +that already holds the weights — a second replica one breaker away from the +first is worth less than the copy saved — and above bin packing. + +**Keeping a replica together.** A profile's `keep_within: rack` (or `row`, +`room`, `site`) keeps a multi-node replica's nodes in one such domain, inside +the fabric or NVLink domain placement already requires; nodes without the +label are passed over for it, with the reason. + +**Holds.** `llmgw ctl facility hold rack:r12-07 --reason "PDU swap"` places +nothing new in the domain, as a cordon of each of its nodes would, including +nodes labelled into it later. `--drain` also drains its nodes, one replica at +a time with the replacement up first; `llmgw ctl facility release` undoes +both. Holds are kept in the store across restarts (the drain itself is node +state, which is not: after a restart the hold still keeps work out, and +replicas not yet moved stay where they are). Any domain can be held: a PDU +holds every rack on it. + +**Power budgets.** In `server.yaml`: + +```yaml +facility: + margin: 0.1 # share of every cap kept free + gpu_watts: 700 # a card that reports no power limit + cpu_core_watts: 10 # a core reserved by a CPU replica (cpu_cores) + node_watts: 350 # each node's own draw beyond its cards + budgets: + - domain: rack:r12-07 + cap_watts: 16000 # what one feed carries alone, derated + other_watts: 800 # switches, storage + - domain: pdu:r12-07-a + cap_watts: 7400 +``` + +A replica's draw is the profile's `power_watts` (split over its nodes) when +declared; otherwise each card at the power limit it reports through the agent's +sensors (or `gpu_watts`), and each reserved core at `cpu_core_watts`. A +domain's use is, node by node, `node_watts` plus the larger of what is placed +there and what the node's sensors read in the last two minutes +(`measured_fresh`), plus the domain's `other_watts`. A shard that would take +any of its node's domains past `cap × (1 − margin)` is refused; a multi-node +replica's shards are counted together, so two that each fit a rack alone do not +both go into it. A draw nobody knows — no declared figure, no card limit, no +`gpu_watts` — counts as nothing: missing data never blocks. + +**What people see.** `llmgw ctl explain-placement` names the domain: *rack +r12-07 is held: PDU swap*, *rack r12-07: 14.2 kW of a 14.4 kW limit (16.0 kW +cap less 10%) in use, the shard needs ~2.8 kW*. The rejections of a placed +replica are kept too, so a replica in an unexpected rack says why. `llmgw ctl +nodes` has a LOCATION column, shows *ready, held*, and filters with `--domain +rack:r12-07`. `llmgw ctl facility` lists every domain with its nodes, use, +limit, reserved and measured draw, and holds. The console's Nodes view has a +WHERE column; a node's page shows its location and hold, and its menu sets +the location (**Location…**) and holds or releases its rack. The admin API: +`GET /api/v1/facility`, `POST /api/v1/facility/holds`, `DELETE +/api/v1/facility/holds/{domain}`, `POST /api/v1/nodes/{id}/labels`. + +**Where it departs from the design below, and why.** + +- Domains come from labels alone; there is no facility file declaring them. + Budgets are in `server.yaml`, the one file the control plane already reads. + A DCIM export would generate that block. +- Feeds are not modelled. The cap given is what one feed carries alone, + derated — the figure the design says matters — so the operator gives the + one number instead of llmgw deriving it from two. +- The manual hold of step 4 is built now, without conditions: it needs no + sensor data and answers the commonest facility event, planned maintenance. + Its command is `ctl facility hold`, with a `--drain` the design did not have. +- `keep_within` is new: the request that a multi-node replica stay in one rack + where the cabling requires it. +- Spreading is an ordering of candidates, not the scoring of the design; power + headroom does not yet order racks ("level load across racks"), and + multi-node replicas are not spread across racks. +- No `llmgw_domain_*` metrics or alerts yet; `ctl facility` and the API carry + the same figures. ## Why @@ -260,16 +356,17 @@ Each step ships on its own and is useful without the next: 0. **Node sensors, observe-only (built).** GPU heat, power and throttling, CPU heat and power, inlet air where the board has a sensor; published and shown, no decision changed. -1. **Location and spreading.** Well-known labels, domains from labels, replicas +1. **Location and spreading (built).** Well-known labels, domains from labels, replicas spread across racks and PDUs, locations on the node views. No facility data needed. -2. **Power budgets from a static file.** Caps and margins, estimated draw per +2. **Power budgets from a static file (built, without headroom + scoring).** Caps and margins, estimated draw per replica (declared or derived), hard limit in placement and headroom. 3. **Readings, observe-only.** The Prometheus source beside the agents', conditions with hysteresis (throttling first among them), measured power folded into headroom; decisions shown, not enforced. -4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and - release. +4. **Thermal enforcement.** `warm` scored down, `hot` refused; the manual hold + and release are built. 5. **Shedding and tokens per watt.** Opt-in shedding through step 2's preemption; measured power in profile preference. diff --git a/docs/install.md b/docs/install.md index 114315a..6bbb43f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -857,8 +857,8 @@ llmgw ctl nodes ``` ``` -NODE STATE GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN -llmgw-node01 ready 8/8 free 503G h100-80g ib-a 0 2s +NODE STATE LOCATION GPUS MEM CLASSES FABRICS REPLICAS LAST SEEN +llmgw-node01 ready - 8/8 free 503G h100-80g ib-a 0 2s ``` If it does not appear, the control plane's log says why. A certificate whose @@ -870,6 +870,27 @@ Repeat for each node. Nodes in the same fabric must be given the same `--fabric` name; nodes with no InfiniBand take no `--fabric` at all and are each their own island, which is fine for single-node models. +### Saying where the node is + +On more than one rack, tell llmgw which rack, and which PDUs, each node is in. +Placement then keeps a model's replicas in different racks and on different +PDUs, places nothing in a rack held for maintenance, and keeps each rack under +a power cap if you give it one: + +``` +llmgw ctl label llmgw-node01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b +llmgw ctl label llmgw-node01 llmgw.io/row=12 llmgw.io/room=hall-b llmgw.io/site=sto1 +``` + +Labels set this way are kept by the control plane and win over the node's +`agent.yaml`, where the same keys can go under `labels:` instead; a node need +not have joined yet. `llmgw ctl nodes` shows the LOCATION, `--domain +rack:r12-07` lists one rack, and `llmgw ctl facility` lists every rack, row and +PDU with what it draws. In the console it is the node's menu, **Location…**. +A node with no location is placed as before, as a rack of its own. +See [facility-aware-placement.md](facility-aware-placement.md) for power caps +and holds. + ## 6. Setting the node's CPU model, if it is a VM Proxmox defaults a VM to `kvm64`, which hides AVX, AVX2, FMA and even SSE4.2 diff --git a/docs/reference.md b/docs/reference.md index d8a2484..31c367f 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -86,6 +86,15 @@ rather than repeated. | `url` | `string` | [External](#external) | | `model` | `string` | [External](#external) | | `api_key_file` | `string` | [External](#external) | +| `budgets` | `[]FacilityBudget` | [Facility](#facility) | +| `margin` | `float64` | [Facility](#facility) | +| `gpu_watts` | `float64` | [Facility](#facility) | +| `cpu_core_watts` | `float64` | [Facility](#facility) | +| `node_watts` | `float64` | [Facility](#facility) | +| `measured_fresh` | `time.Duration` | [Facility](#facility) | +| `domain` | `string` | [FacilityBudget](#facilitybudget) | +| `cap_watts` | `float64` | [FacilityBudget](#facilitybudget) | +| `other_watts` | `float64` | [FacilityBudget](#facilitybudget) | | `format` | `string` | [GuardModel](#guardmodel) | | `categories` | `[]string` | [GuardModel](#guardmodel) | | `model` | `string` | [JudgePolicy](#judgepolicy) | @@ -194,6 +203,8 @@ rather than repeated. | `cpu_cores` | `int` | [Profile](#profile) | | `min_disk_gb` | `int` | [Profile](#profile) | | `node_selector` | `map[string]string` | [Profile](#profile) | +| `power_watts` | `float64` | [Profile](#profile) | +| `keep_within` | `string` | [Profile](#profile) | | `weight` | `float64` | [Profile](#profile) | | `priority` | `int` | [Profile](#profile) | | `startup_timeout` | `time.Duration` | [Profile](#profile) | @@ -271,6 +282,7 @@ rather than repeated. | `router` | `Router` | [Server](#server) | | `control` | `Control` | [Server](#server) | | `models` | `[]catalog.Model` | [Server](#server) | +| `facility` | `Facility` | [Server](#server) | | `endpoints` | `[]Endpoint` | [Server](#server) | | `bin` | `string` | [SimConfig](#simconfig) | | `python` | `string` | [TRTLLMConfig](#trtllmconfig) | @@ -839,6 +851,75 @@ APIKeyFile holds the provider's key, on the control plane. A file rather than the key itself, so the key is in neither the catalog, the store nor the audit log; empty for a provider that asks for none. +## Facility + +### `budgets` + +`[]FacilityBudget` + +Budgets are the domains with a power cap. + +### `margin` + +`float64` + +Margin is the share of every cap kept free, 0.1 for ten percent. Zero +keeps none, for caps already given derated. + +### `gpu_watts` + +`float64` + +GPUWatts is a card's draw when it does not report its power limit: +an agent without nvidia-smi's power fields, or a node that has not +sent a heartbeat yet. Zero counts such a card as drawing nothing. + +### `cpu_core_watts` + +`float64` + +CPUCoreWatts is the draw of a core reserved by a replica with no GPU +(cpu_cores). Zero counts such replicas as drawing nothing. + +### `node_watts` + +`float64` + +NodeWatts is what a node draws besides its cards and reserved cores — +idle CPUs, memory, fans, disks — counted for every node in a capped +domain, busy or not. + +### `measured_fresh` + +`time.Duration` + +MeasuredFresh is how old a node's power reading may be and still be +counted against its domains. Zero takes two minutes. + +## FacilityBudget + +### `domain` + +`string` + +Domain is level:name, as the nodes' labels give it: rack:r12-07, +pdu:r12-07-a, row:12. + +### `cap_watts` + +`float64` + +CapWatts is what the domain may draw. For a dual-fed rack it is what +one feed carries alone, derated as the facility derates it (usually +80%), since that is what the rack has to survive. + +### `other_watts` + +`float64` + +OtherWatts is drawn in the domain by what llmgw does not place on: +switches, storage, machines outside the cluster. + ## GuardModel ### `format` @@ -1638,6 +1719,27 @@ in the inventory says how much memory a node has, so the operator says which nodes are meant for a model instead of the scheduler inferring it. Agents have reported labels since the beginning; this is what reads them. +### `power_watts` + +`float64` + +PowerWatts is what one replica of this profile draws, in watts, split +evenly over its nodes, counted against the power cap of every rack, +PDU or other domain its nodes are in (facility.budgets in server.yaml). +Zero derives it: each card at the power limit it reports, each +reserved core at facility.cpu_core_watts. Only matters where a domain +has a cap. + +### `keep_within` + +`string` + +KeepWithin keeps a multi-node replica's nodes inside one rack, row, +room or site, read from their llmgw.io/rack (and so on) labels: for a +fabric whose leaves do not reach across racks, or cables that do not. +Nodes without the label are passed over for such a profile. Empty is +no constraint. + ### `weight` `float64` @@ -2249,6 +2351,17 @@ Takes a [Control](#control) block. `[]catalog.Model` +### `facility` + +`Facility` + +Facility is the power caps of racks, PDUs and other domains, and how +a replica's draw is estimated against them. Where a node is comes from +its llmgw.io/site, room, row, rack and pdu labels; holding a domain for +maintenance is `llmgw ctl facility hold`, not configuration. + +Takes a [Facility](#facility) block. + ### `endpoints` `[]Endpoint` diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index e92456f..f829f17 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -139,6 +139,8 @@ func newCtlCmd() *cobra.Command { newReplicasCmd(client, &asJSON), newExplainCmd(client, &asJSON), newFabricTestCmd(client, &asJSON), + newFacilityCmd(client, &asJSON), + newLabelCmd(client, &asJSON), newNodeStateCmd(client, "cordon", "cordoned", "Stop placing new work on a node; what runs there keeps running"), newNodeStateCmd(client, "uncordon", "ready", @@ -205,10 +207,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } w := tab() - fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") + fmt.Fprintln(w, "NODE\tSTATE\tLOCATION\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN") for _, n := range nodes { - fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", - n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), + fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n", + n.ID, stateColumn(n), locationColumn(n), n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB), diskColumn(n.DiskFreeGB, n.DiskTotalGB), // What the node can run, which is the first thing to look // at when a model will not place and the hardware looks @@ -239,6 +241,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { fmt.Printf("\n%s: %s", n.ID, w) said = true } + if n.Held != "" { + fmt.Printf("\n%s: %s (kept out of placement)", n.ID, n.Held) + said = true + } } if said { fmt.Println() @@ -251,9 +257,28 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)") c.Flags().StringArrayVar(&f.labels, "label", nil, "only nodes carrying key=value; repeatable, and key alone matches any value") + c.Flags().StringVar(&f.domain, "domain", "", "only nodes in this rack, row, room, site or PDU, as level:name (rack:r12-07)") return c } +// stateColumn is a node's state, with "held" when a held domain keeps it out +// of placement while its own state says ready. +func stateColumn(n adminapi.NodeView) string { + if n.Held != "" && n.State == "ready" { + return "ready, held" + } + return n.State +} + +// locationColumn is where a node is, in a few characters; a dash when it +// carries no location label. +func locationColumn(n adminapi.NodeView) string { + if n.Location == nil || n.Location.Empty() { + return "-" + } + return n.Location.Short() +} + // modelStatus says what is happening to a model, in the words that decide what // to do about it. // diff --git a/internal/cli/facility.go b/internal/cli/facility.go new file mode 100644 index 0000000..1261739 --- /dev/null +++ b/internal/cli/facility.go @@ -0,0 +1,212 @@ +package cli + +import ( + "fmt" + "io" + "maps" + "net/http" + "net/url" + "slices" + "strings" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" +) + +// newFacilityCmd shows the failure domains and holds them for maintenance. +func newFacilityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "facility", + Short: "Racks, PDUs and rooms: their power against their caps, and what is held", + Long: `Where nodes are comes from their labels: llmgw.io/site, room, row, rack +and pdu, set in agent.yaml or with 'llmgw ctl label'. Placement spreads a +model's replicas across racks and PDUs, places nothing in a held domain, and +refuses a replica that would take a domain past its power cap (facility.budgets +in server.yaml). + +USED is what counts against the cap: node by node, the larger of what the +replicas placed there are estimated to draw and what the node's sensors read.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var v adminapi.FacilityView + if err := client().get(cmd.Context(), "/api/v1/facility", &v); err != nil { + return err + } + if *asJSON { + return emitJSON(v) + } + printFacility(cmd.OutOrStdout(), v) + return nil + }, + } + cmd.AddCommand(newFacilityHoldCmd(client), newFacilityReleaseCmd(client)) + return cmd +} + +// printFacility is the facility view as a table, then what is held, then +// the nodes nobody has said the place of. +func printFacility(out io.Writer, v adminapi.FacilityView) { + if len(v.Domains) == 0 { + fmt.Fprintln(out, "No node says where it is. Label them with llmgw.io/rack (and site, room, row, pdu):\n"+ + " llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b") + return + } + w := tabTo(out) + fmt.Fprintln(w, "DOMAIN\tNODES\tUSED\tLIMIT\tRESERVED\tMEASURED\tHELD") + for _, d := range v.Domains { + limit, used := "-", facility.KW(d.UsedWatts) + if d.LimitWatts > 0 { + limit = facility.KW(d.LimitWatts) + used += fmt.Sprintf(" (%.0f%%)", 100*d.UsedWatts/d.LimitWatts) + } + held := "-" + if d.Held != nil { + held = "held" + if d.Held.Reason != "" { + held += ": " + d.Held.Reason + } + if d.Held.Drain { + held += " (drained)" + } + } + fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\n", d.Domain, len(d.Nodes), used, limit, + facility.KW(d.ReservedWatts), facility.KW(d.MeasuredWatts), held) + } + _ = w.Flush() + for _, h := range v.Holds { + fmt.Fprintf(out, "\n%s held by %s since %s", h.Domain, or(h.By, "someone"), h.At.Local().Format("2006-01-02 15:04")) + } + if len(v.Holds) > 0 { + fmt.Fprintln(out) + } + if len(v.Unlocated) > 0 { + fmt.Fprintf(out, "\nNo location: %s. Each is placed as a rack of its own.\n", strings.Join(v.Unlocated, ", ")) + } +} + +func newFacilityHoldCmd(client func() *ctlClient) *cobra.Command { + var req adminapi.HoldRequest + c := &cobra.Command{ + Use: "hold ", + Short: "Place nothing new in a rack, row, PDU or room; with --drain, move its work off", + Long: `For maintenance, a cooling fault, work on a PDU: like cordoning every node in +the domain, including nodes labelled into it later, and kept across restarts +when the control plane has a store. What runs there keeps running unless +--drain is given; then each of its nodes is drained, one replica at a time +with the replacement up elsewhere first.`, + Example: ` llmgw ctl facility hold rack:r12-07 --reason "PDU A replacement" + llmgw ctl facility hold pdu:r12-07-a --drain --reason "breaker work"`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + req.Domain = args[0] + if _, err := facility.ParseDomain(req.Domain); err != nil { + return err + } + var v adminapi.HoldView + if err := client().do(cmd.Context(), http.MethodPost, "/api/v1/facility/holds", req, &v); err != nil { + return err + } + fmt.Printf("%s is held: nothing new is placed in it.\n", v.Domain) + if len(v.Nodes) > 0 { + fmt.Printf("Draining %s; their replicas move one at a time.\n", strings.Join(v.Nodes, ", ")) + } + fmt.Printf("Release it with: llmgw ctl facility release %s\n", v.Domain) + return nil + }, + } + c.Flags().StringVar(&req.Reason, "reason", "", "why, as explain-placement and the node list will say") + c.Flags().BoolVar(&req.Drain, "drain", false, "also drain the domain's nodes") + return c +} + +func newFacilityReleaseCmd(client func() *ctlClient) *cobra.Command { + return &cobra.Command{ + Use: "release ", + Short: "Allow placement in a held domain again", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + d, err := facility.ParseDomain(args[0]) + if err != nil { + return err + } + var v adminapi.HoldView + if err := client().delete(cmd.Context(), "/api/v1/facility/holds/"+url.PathEscape(string(d)), &v); err != nil { + return err + } + fmt.Printf("%s is released.\n", v.Domain) + if len(v.Nodes) > 0 { + fmt.Printf("Ready again: %s.\n", strings.Join(v.Nodes, ", ")) + } + return nil + }, + } +} + +// newLabelCmd sets labels on a node through the control plane. +func newLabelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "label key=value... [key-...]", + Short: "Set or remove a node's labels, over those in its agent.yaml", + Long: `Labels set here are kept by the control plane and win over the agent's own; +key- removes one, which puts the agent's value for the key back if it has +one. The node need not have joined yet. + +The llmgw.io/ labels say where a node is, for facility-aware placement: +site, room, row, rack, and pdu (a comma separated list for a dual-fed rack). +Any other key is free for a profile's node_selector.`, + Example: ` llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b + llmgw ctl label gpu01 llmgw.io/row=12 llmgw.io/room=hall-b + llmgw ctl label gpu01 llmgw.io/rack-`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + req, err := parseLabelArgs(args[1:]) + if err != nil { + return err + } + var v adminapi.NodeLabelsView + if err := client().do(cmd.Context(), http.MethodPost, + "/api/v1/nodes/"+url.PathEscape(args[0])+"/labels", req, &v); err != nil { + return err + } + if *asJSON { + return emitJSON(v) + } + if v.Labels == nil { + fmt.Printf("%s has not joined yet; it takes these labels when it does.\n", v.ID) + } + keys := slices.Sorted(maps.Keys(v.OperatorLabels)) + if len(keys) == 0 { + fmt.Printf("%s carries no labels set here, only its agent's.\n", v.ID) + } + for _, k := range keys { + fmt.Printf("%s: %s=%s\n", v.ID, k, v.OperatorLabels[k]) + } + if v.Location != nil { + fmt.Printf("%s is at %s.\n", v.ID, v.Location.Short()) + } + return nil + }, + } +} + +// parseLabelArgs reads key=value to set and key- to remove. +func parseLabelArgs(args []string) (adminapi.NodeLabelsRequest, error) { + req := adminapi.NodeLabelsRequest{Set: map[string]string{}} + for _, a := range args { + if k, v, ok := strings.Cut(a, "="); ok { + if k == "" || v == "" { + return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a) + } + req.Set[k] = v + continue + } + if k, ok := strings.CutSuffix(a, "-"); ok && k != "" { + req.Remove = append(req.Remove, k) + continue + } + return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a) + } + return req, nil +} diff --git a/internal/cli/facility_test.go b/internal/cli/facility_test.go new file mode 100644 index 0000000..42a2b13 --- /dev/null +++ b/internal/cli/facility_test.go @@ -0,0 +1,75 @@ +package cli + +import ( + "bytes" + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" +) + +func TestLabelArgumentsSetWithEqualsAndRemoveWithADash(t *testing.T) { + req, err := parseLabelArgs([]string{"llmgw.io/rack=r1", "llmgw.io/pdu=p1,p2", "role-"}) + if err != nil { + t.Fatal(err) + } + if req.Set["llmgw.io/rack"] != "r1" || req.Set["llmgw.io/pdu"] != "p1,p2" || !slices.Equal(req.Remove, []string{"role"}) { + t.Errorf("got %+v", req) + } + for _, bad := range []string{"rack", "=r1", "rack=", "-"} { + if _, err := parseLabelArgs([]string{bad}); err == nil { + t.Errorf("%q was accepted", bad) + } + } +} + +func TestNodesAreFilteredByDomain(t *testing.T) { + nodes := fleet() + nodes[0].Location = &facility.Location{Rack: "r1", PDUs: []string{"p1"}} + nodes[1].Location = &facility.Location{Rack: "r2", PDUs: []string{"p1"}} + got, err := nodeFilter{domain: "pdu:p1"}.apply(nodes) + if err != nil || ids(got) != "n1,n2" { + t.Errorf("--domain pdu:p1 = %s, %v", ids(got), err) + } + if got, _ := (nodeFilter{domain: "rack:r2"}).apply(nodes); ids(got) != "n2" { + t.Errorf("--domain rack:r2 = %s", ids(got)) + } + if _, err := (nodeFilter{domain: "r2"}).apply(nodes); err == nil { + t.Error("a domain with no level was accepted") + } +} + +func TestTheLocationFormSetsWhatChangedAndRemovesWhatWasEmptied(t *testing.T) { + n := adminapi.NodeView{ + Labels: map[string]string{"llmgw.io/rack": "r1", "llmgw.io/row": "12"}, + OperatorLabels: map[string]string{"llmgw.io/row": "12"}, + } + // site, room, row, rack, pdu + req := locationRequest(n, []string{"", "hall-b", "", "r1", "p1"}) + if len(req.Set) != 2 || req.Set["llmgw.io/room"] != "hall-b" || req.Set["llmgw.io/pdu"] != "p1" { + t.Errorf("set %v: the unchanged rack is left alone", req.Set) + } + if !slices.Equal(req.Remove, []string{"llmgw.io/row"}) { + t.Errorf("remove %v, want the row the operator had set", req.Remove) + } +} + +func TestTheFacilityTableShowsUseAgainstTheLimitAndWhatIsHeld(t *testing.T) { + var b bytes.Buffer + printFacility(&b, adminapi.FacilityView{ + Domains: []adminapi.DomainView{ + {Domain: "rack:r1", Nodes: []string{"a"}, LimitWatts: 10000, UsedWatts: 7500, ReservedWatts: 7000, MeasuredWatts: 7500, + Held: &adminapi.HoldView{Domain: "rack:r1", Reason: "PDU swap"}}, + }, + Holds: []adminapi.HoldView{{Domain: "rack:r1", By: "ilker"}}, + Unlocated: []string{"b"}, + }) + out := b.String() + for _, want := range []string{"7.5 kW (75%)", "10.0 kW", "held: PDU swap", "rack:r1 held by ilker", "No location: b"} { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } +} diff --git a/internal/cli/filter.go b/internal/cli/filter.go index bab10fc..cc2ed5b 100644 --- a/internal/cli/filter.go +++ b/internal/cli/filter.go @@ -6,6 +6,7 @@ import ( "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" ) // nodeFilter narrows a node listing. @@ -20,6 +21,8 @@ type nodeFilter struct { class string state string labels []string + // domain is level:name, rack:r12-07. + domain string } // empty reports whether this filter would narrow anything. @@ -28,7 +31,7 @@ type nodeFilter struct { // different situations with different next steps, and "no nodes" for both // makes a mistyped fabric look like an outage. func (f nodeFilter) empty() bool { - return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 + return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 && f.domain == "" } func (f nodeFilter) describe() string { @@ -45,6 +48,9 @@ func (f nodeFilter) describe() string { for _, l := range f.labels { parts = append(parts, "label "+l) } + if f.domain != "" { + parts = append(parts, "domain "+f.domain) + } return strings.Join(parts, ", ") } @@ -62,11 +68,23 @@ func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) { want[k] = v } + var domain facility.Domain + if f.domain != "" { + d, err := facility.ParseDomain(f.domain) + if err != nil { + return nil, fmt.Errorf("--domain: %w", err) + } + domain = d + } + out := make([]adminapi.NodeView, 0, len(in)) for _, n := range in { if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) { continue } + if domain != "" && (n.Location == nil || !slices.Contains(n.Location.Domains(), domain)) { + continue + } if f.class != "" && !slices.Contains(n.Classes, f.class) { continue } diff --git a/internal/cli/server.go b/internal/cli/server.go index 79a572e..934787e 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -654,6 +654,10 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live if u, ok := sink.(control.UsageSource); ok { ctl.SetUsageSource(u) } + // Power caps and estimates; Validate has already refused a bad one. + if pol, err := cfg.Facility.Policy(); err == nil { + ctl.Facility().SetPolicy(pol) + } // With a shared store, two control planes can see each other, so this one // competes rather than assuming it is alone. Without one there is nobody // to compete with and it places unconditionally, exactly as before. @@ -676,6 +680,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live ctl.Reconciler().SetMeasurements(control.Measurements(st, log)) // The engine builds chosen per node on the cluster page. ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log)) + // Domains held for maintenance and labels set on nodes. + ctl.Facility().UseStore(ctx, st) log.Info("competing to be the control plane that places work", "owner", lead.Owner()) } diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index 0d88fbc..66dc17a 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "gitea.mynodes.xyz/ilker/llmgw/internal/version" @@ -464,6 +465,7 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog { return nil }}) } + items = append(items, m.locationItems(n)...) if n.GPUTotal > 0 { for _, engine := range []string{"vllm", "sglang"} { items = append(items, menuItem{engine + " build…", func(m *consoleModel) tea.Cmd { @@ -825,3 +827,72 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { } return items } + +// locationItems are the node menu's facility entries: where the node is, +// and holding or releasing its rack. +func (m consoleModel) locationItems(n adminapi.NodeView) []menuItem { + path := "/api/v1/nodes/" + url.PathEscape(n.ID) + "/labels" + var loc facility.Location + if n.Location != nil { + loc = *n.Location + } + items := []menuItem{{"Location…", func(m *consoleModel) tea.Cmd { + m.openForm(&formDialog{title: "Where " + n.ID + " is", + note: "Placement spreads a model's replicas across racks and PDUs, and keeps out of held ones. " + + "Empty leaves a level unknown; a PDU list is comma separated, for a dual-fed rack.", + fields: []formField{ + textF("site", loc.Site, "sto1"), textF("room", loc.Room, "hall-b"), + textF("row", loc.Row, "12"), textF("rack", loc.Rack, "r12-07"), + textF("pdu", strings.Join(loc.PDUs, ","), "r12-07-a,r12-07-b"), + }, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + req := locationRequest(n, v) + if len(req.Set) == 0 && len(req.Remove) == 0 { + return nil, fmt.Errorf("nothing changed") + } + return m.call("Location of "+n.ID, http.MethodPost, path, req, + func(json.RawMessage) string { return n.ID + "'s location is set." }), nil + }}) + return nil + }}} + switch { + case n.HeldDomain != "": + d := n.HeldDomain + items = append(items, menuItem{"Release " + d, func(m *consoleModel) tea.Cmd { + return m.call("Release "+d, http.MethodDelete, "/api/v1/facility/holds/"+url.PathEscape(d), nil, + func(json.RawMessage) string { return d + " is released." }) + }}) + case loc.Rack != "": + d := string(facility.Of(facility.Rack, loc.Rack)) + items = append(items, menuItem{"Hold rack " + loc.Rack + "…", func(m *consoleModel) tea.Cmd { + m.openForm(&formDialog{title: "Hold rack " + loc.Rack, + note: "Nothing new is placed in the rack. Drain also moves its replicas off, one at a time.", + fields: []formField{textF("reason", "", "PDU A replacement"), + choiceF("drain", []string{"no", "yes"}, "no")}, + submit: func(m *consoleModel, v []string) (tea.Cmd, error) { + return m.call("Hold "+d, http.MethodPost, "/api/v1/facility/holds", + adminapi.HoldRequest{Domain: d, Reason: v[0], Drain: v[1] == "yes"}, + func(json.RawMessage) string { return d + " is held." }), nil + }}) + return nil + }}) + } + return items +} + +// locationRequest turns the location form's values, site to pdu, into the +// labels to set and the ones to remove: a level emptied that was set through +// the control plane is removed, which leaves the agent's own value if any. +func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequest { + req := adminapi.NodeLabelsRequest{Set: map[string]string{}} + for i, lv := range facility.Levels { + k, v := lv.Label(), strings.TrimSpace(vals[i]) + switch { + case v != "" && n.Labels[k] != v: + req.Set[k] = v + case v == "" && n.OperatorLabels[k] != "": + req.Remove = append(req.Remove, k) + } + } + return req +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 5fc1d12..02bd929 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -1041,11 +1041,11 @@ func (m consoleModel) panel() (title, body, keys string) { if n.Load != nil { cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent) } - state := n.State + state := stateColumn(n) if state != "ready" { state = sWarn.Render(state) } - rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal), + rows = append(rows, []string{n.ID, state, locationColumn(n), cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal), strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo}) } t := "Nodes" @@ -1055,7 +1055,7 @@ func (m consoleModel) panel() (title, body, keys string) { if m.inPanel { keys = rowKeys } - return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"}, + return t, "\n" + table([]string{"NODE", "STATE", "WHERE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"}, rows, m.row, pw, m.inPanel), keys case cGPUs: if m.inPanel { @@ -1299,6 +1299,18 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string { kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) } b.WriteString("\n") kv("state", n.State) + if n.Location != nil { + where := []string{} + for _, d := range n.Location.Domains() { + where = append(where, d.Human()) + } + kv("location", strings.Join(where, " · ")) + } else { + kv("location", sDim.Render("not labelled: Enter, then Location…")) + } + if n.Held != "" { + kv("held", sWarn.Render(n.Held)) + } kv("address", n.MgmtAddr) kv("system", n.OS) kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago") diff --git a/internal/config/config.go b/internal/config/config.go index 20f5585..e264388 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,6 +18,7 @@ import ( "github.com/knadh/koanf/v2" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/notify" "gitea.mynodes.xyz/ilker/llmgw/internal/obs" @@ -174,12 +175,95 @@ type Server struct { Control Control `koanf:"control"` Models []catalog.Model `koanf:"models"` + // Facility is the power caps of racks, PDUs and other domains, and how + // a replica's draw is estimated against them. Where a node is comes from + // its llmgw.io/site, room, row, rack and pdu labels; holding a domain for + // maintenance is `llmgw ctl facility hold`, not configuration. + Facility Facility `koanf:"facility"` + // Endpoints is the phase-1 static replica list. It is only consulted when // no model declares a profile, which is what makes a router-only smoke test // against sim-engines possible without a control plane. Endpoints []Endpoint `koanf:"endpoints"` } +// Facility configures facility-aware placement: what each rack, PDU or other +// domain may draw, and the figures a replica's draw is estimated from when +// its profile declares none (power_watts). +// +// Nothing here is needed for placement to know where nodes are or to spread +// replicas across racks; that is the nodes' labels alone. A domain with no +// budget is not capped. +type Facility struct { + // Budgets are the domains with a power cap. + Budgets []FacilityBudget `koanf:"budgets"` + // Margin is the share of every cap kept free, 0.1 for ten percent. Zero + // keeps none, for caps already given derated. + Margin float64 `koanf:"margin"` + // GPUWatts is a card's draw when it does not report its power limit: + // an agent without nvidia-smi's power fields, or a node that has not + // sent a heartbeat yet. Zero counts such a card as drawing nothing. + GPUWatts float64 `koanf:"gpu_watts"` + // CPUCoreWatts is the draw of a core reserved by a replica with no GPU + // (cpu_cores). Zero counts such replicas as drawing nothing. + CPUCoreWatts float64 `koanf:"cpu_core_watts"` + // NodeWatts is what a node draws besides its cards and reserved cores — + // idle CPUs, memory, fans, disks — counted for every node in a capped + // domain, busy or not. + NodeWatts float64 `koanf:"node_watts"` + // MeasuredFresh is how old a node's power reading may be and still be + // counted against its domains. Zero takes two minutes. + MeasuredFresh time.Duration `koanf:"measured_fresh"` +} + +// FacilityBudget is one domain's power cap. +type FacilityBudget struct { + // Domain is level:name, as the nodes' labels give it: rack:r12-07, + // pdu:r12-07-a, row:12. + Domain string `koanf:"domain"` + // CapWatts is what the domain may draw. For a dual-fed rack it is what + // one feed carries alone, derated as the facility derates it (usually + // 80%), since that is what the rack has to survive. + CapWatts float64 `koanf:"cap_watts"` + // OtherWatts is drawn in the domain by what llmgw does not place on: + // switches, storage, machines outside the cluster. + OtherWatts float64 `koanf:"other_watts"` +} + +// Policy is the configuration as placement reads it. +func (f Facility) Policy() (facility.Policy, error) { + p := facility.Policy{ + Margin: f.Margin, GPUWatts: f.GPUWatts, CPUCoreWatts: f.CPUCoreWatts, + NodeWatts: f.NodeWatts, MeasuredFresh: f.MeasuredFresh, + } + if f.Margin < 0 || f.Margin >= 1 { + return p, fmt.Errorf("facility.margin %v is a share of the cap, from 0 to below 1: 0.1 keeps ten percent free", f.Margin) + } + if f.GPUWatts < 0 || f.CPUCoreWatts < 0 || f.NodeWatts < 0 { + return p, fmt.Errorf("facility: gpu_watts, cpu_core_watts and node_watts cannot be negative") + } + for i, b := range f.Budgets { + d, err := facility.ParseDomain(b.Domain) + if err != nil { + return p, fmt.Errorf("facility.budgets[%d]: %w", i, err) + } + if b.CapWatts <= 0 { + return p, fmt.Errorf("facility.budgets[%d] (%s): cap_watts must be above zero", i, d) + } + if b.OtherWatts < 0 { + return p, fmt.Errorf("facility.budgets[%d] (%s): other_watts cannot be negative", i, d) + } + if p.Budgets == nil { + p.Budgets = map[facility.Domain]facility.Budget{} + } + if _, dup := p.Budgets[d]; dup { + return p, fmt.Errorf("facility.budgets[%d]: %s is given twice", i, d) + } + p.Budgets[d] = facility.Budget{CapWatts: b.CapWatts, OtherWatts: b.OtherWatts} + } + return p, nil +} + // Monitoring points the dashboard at a Prometheus holding this cluster's // history. // @@ -740,6 +824,9 @@ func (s Server) Validate() error { if s.Listen == "" { return fmt.Errorf("listen is empty") } + if _, err := s.Facility.Policy(); err != nil { + return err + } if s.Control.Timezone != "" { if _, err := time.LoadLocation(s.Control.Timezone); err != nil { return fmt.Errorf("control.timezone %q: %w", s.Control.Timezone, err) diff --git a/internal/config/facility_test.go b/internal/config/facility_test.go new file mode 100644 index 0000000..75c102b --- /dev/null +++ b/internal/config/facility_test.go @@ -0,0 +1,52 @@ +package config + +import ( + "strings" + "testing" +) + +func TestFacilityBudgetsAreReadFromServerYAML(t *testing.T) { + p := write(t, ` +facility: + margin: 0.1 + gpu_watts: 700 + node_watts: 350 + budgets: + - domain: rack:r12-07 + cap_watts: 16000 + other_watts: 800 + - domain: pdu:r12-07-a + cap_watts: 7400 +`) + cfg, err := Load(p) + if err != nil { + t.Fatal(err) + } + pol, err := cfg.Facility.Policy() + if err != nil { + t.Fatal(err) + } + if b := pol.Budgets["rack:r12-07"]; b.CapWatts != 16000 || b.OtherWatts != 800 { + t.Errorf("rack budget %+v", b) + } + if pol.Limit("pdu:r12-07-a") != 7400*0.9 || pol.GPUWatts != 700 || pol.NodeWatts != 350 { + t.Errorf("policy %+v", pol) + } +} + +func TestAFacilityBudgetThatCannotBeUsedIsRefused(t *testing.T) { + for yaml, want := range map[string]string{ + "facility:\n budgets:\n - domain: r12-07\n cap_watts: 1000\n": "level:name", + "facility:\n budgets:\n - domain: rack:r1\n": "cap_watts must be above zero", + "facility:\n margin: 1.5\n": "facility.margin", + "facility:\n budgets:\n - {domain: rack:r1, cap_watts: 1}\n - {domain: rack:r1, cap_watts: 2}\n": "given twice", + } { + cfg, err := Load(write(t, yaml)) + if err == nil { + err = cfg.Validate() + } + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%q: got %v, want %q", yaml, err, want) + } + } +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 2a85e79..597b193 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -32,6 +32,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" @@ -153,6 +154,10 @@ func (a *API) Register(mux Mux) { mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState) mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode) mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine) + mux.HandleFunc("POST /api/v1/nodes/{id}/labels", a.setNodeLabels) + mux.HandleFunc("GET /api/v1/facility", a.facilityReport) + mux.HandleFunc("POST /api/v1/facility/holds", a.holdDomain) + mux.HandleFunc("DELETE /api/v1/facility/holds/{domain}", a.releaseDomain) mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode) mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) @@ -254,6 +259,18 @@ type NodeView struct { // answer when a node looks idle but nothing will schedule onto it. Unhealthy []string `json:"unhealthy_gpus,omitempty"` + // Location is where the node is in the building, from its llmgw.io/site, + // room, row, rack and pdu labels; absent when it carries none. + Location *facility.Location `json:"location,omitempty"` + // Held names the hold keeping the node out of placement, "rack r12-07 + // is held: PDU swap"; empty when none is. + Held string `json:"held,omitempty"` + // HeldDomain is the held domain, level:name, for releasing it. + HeldDomain string `json:"held_domain,omitempty"` + // OperatorLabels are the labels set through the control plane, which + // Labels already includes over the agent's own. + OperatorLabels map[string]string `json:"operator_labels,omitempty"` + LastSeenAgo string `json:"last_seen_ago"` AgentVersion string `json:"agent_version,omitempty"` // Features are the requests the agent answers beyond the usual: @@ -566,6 +583,7 @@ func (a *API) Nodes() []NodeView { } inv := a.ctl.Inventory() + pol := inv.Facility() out := make([]NodeView, 0) for _, n := range inv.Nodes() { v := NodeView{ @@ -576,6 +594,17 @@ func (a *API) Nodes() []NodeView { Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features, Replicas: replicasOn[n.ID], } + if loc := n.Location(); !loc.Empty() { + v.Location = &loc + } + if h, held := pol.HeldIn(n.Domains()); held { + v.HeldDomain = string(h.Domain) + v.Held = h.Domain.Human() + " is held" + if h.Reason != "" { + v.Held += ": " + h.Reason + } + } + v.OperatorLabels = inv.OperatorLabels(n.ID) if free, total := n.LocalDisk(); total > 0 { v.DiskFreeGB = int(free / (1 << 30)) v.DiskTotalGB = int(total / (1 << 30)) diff --git a/internal/control/adminapi/facility.go b/internal/control/adminapi/facility.go new file mode 100644 index 0000000..c57cb6d --- /dev/null +++ b/internal/control/adminapi/facility.go @@ -0,0 +1,203 @@ +package adminapi + +import ( + "encoding/json" + "io" + "maps" + "net/http" + "slices" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// FacilityView is the failure domains as placement sees them. +type FacilityView struct { + // Domains are every domain a node is in or a budget names, by level + // (site, room, row, rack, pdu) and then name. + Domains []DomainView `json:"domains"` + // Holds are every domain held, including one no node is in yet. + Holds []HoldView `json:"holds,omitempty"` + // Margin is the share of every cap kept free. + Margin float64 `json:"margin,omitempty"` + // Unlocated are the nodes with no location label, which placement + // treats as each in a rack of its own. + Unlocated []string `json:"unlocated,omitempty"` +} + +// DomainView is one domain: its nodes, its power and whether it is held. +type DomainView struct { + Domain string `json:"domain"` + Level string `json:"level"` + Name string `json:"name"` + Nodes []string `json:"nodes,omitempty"` + // CapWatts is the domain's cap, LimitWatts the cap less the margin; + // both zero when it has none. + CapWatts float64 `json:"cap_watts,omitempty"` + LimitWatts float64 `json:"limit_watts,omitempty"` + OtherWatts float64 `json:"other_watts,omitempty"` + // ReservedWatts is what is placed in it is estimated to draw; + // MeasuredWatts what its nodes' sensors read; UsedWatts what counts + // against the cap, the larger of the two node by node. + ReservedWatts float64 `json:"reserved_watts"` + MeasuredWatts float64 `json:"measured_watts"` + UsedWatts float64 `json:"used_watts"` + // Held is set when the domain is held. + Held *HoldView `json:"held,omitempty"` +} + +// HoldRequest holds a domain. +type HoldRequest struct { + // Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12. + Domain string `json:"domain"` + Reason string `json:"reason,omitempty"` + // Drain moves the work in the domain elsewhere too. + Drain bool `json:"drain,omitempty"` +} + +// HoldView is a hold, and the nodes holding or releasing it changed. +type HoldView struct { + Domain string `json:"domain"` + Reason string `json:"reason,omitempty"` + Drain bool `json:"drain,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + // Nodes are those this call drained, or made ready again on release. + Nodes []string `json:"nodes,omitempty"` +} + +// NodeLabelsRequest sets and removes labels on a node. +type NodeLabelsRequest struct { + Set map[string]string `json:"set,omitempty"` + // Remove are keys whose label set through the control plane goes; the + // agent's own value for the key, if it has one, is in force again. + Remove []string `json:"remove,omitempty"` +} + +// NodeLabelsView is a node's labels after a change. +type NodeLabelsView struct { + ID string `json:"id"` + // OperatorLabels are those set through the control plane. + OperatorLabels map[string]string `json:"operator_labels,omitempty"` + // Labels are what placement matches: the agent's with the operator's + // over them. Absent for a node that has not registered yet. + Labels map[string]string `json:"labels,omitempty"` + Location *facility.Location `json:"location,omitempty"` +} + +func holdView(h facility.Hold, nodes []string) HoldView { + return HoldView{Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At, Nodes: nodes} +} + +// Facility builds the facility view; exported for the console's sake, as +// Nodes is. +func (a *API) Facility() FacilityView { + inv := a.ctl.Inventory() + pol := inv.Facility() + use := inv.PowerUse(time.Now()) + v := FacilityView{Margin: pol.Margin} + for _, n := range inv.Nodes() { + if n.Location().Empty() { + v.Unlocated = append(v.Unlocated, n.ID) + } + } + holds := a.ctl.Facility().Holds() + for _, h := range holds { + v.Holds = append(v.Holds, holdView(h, nil)) + // Listed with the others even before a node is labelled into it. + if _, ok := use[h.Domain]; !ok { + use[h.Domain] = inventory.PowerUse{Domain: h.Domain} + } + } + keys := slices.Collect(maps.Keys(use)) + slices.SortFunc(keys, func(a, b facility.Domain) int { + if c := slices.Index(facility.Levels, a.Level()) - slices.Index(facility.Levels, b.Level()); c != 0 { + return c + } + return strings.Compare(a.Name(), b.Name()) + }) + for _, d := range keys { + u := use[d] + dv := DomainView{Domain: string(d), Level: string(d.Level()), Name: d.Name(), Nodes: u.Nodes, + CapWatts: u.Budget.CapWatts, LimitWatts: u.Limit, OtherWatts: u.Budget.OtherWatts, + ReservedWatts: u.Reserved, MeasuredWatts: u.Measured, UsedWatts: u.Used} + if h, ok := pol.Holds[d]; ok { + hv := holdView(h, nil) + dv.Held = &hv + } + v.Domains = append(v.Domains, dv) + } + return v +} + +func (a *API) facilityReport(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, a.Facility()) +} + +func (a *API) holdDomain(w http.ResponseWriter, r *http.Request) { + var req HoldRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + d, err := facility.ParseDomain(req.Domain) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + h := facility.Hold{Domain: d, Reason: strings.TrimSpace(req.Reason), Drain: req.Drain, By: actorOf(r), At: time.Now()} + drained, err := a.ctl.Facility().Hold(r.Context(), h) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + a.ctl.Reconciler().Notify() + writeJSON(w, http.StatusOK, holdView(h, drained)) +} + +func (a *API) releaseDomain(w http.ResponseWriter, r *http.Request) { + d, err := facility.ParseDomain(r.PathValue("domain")) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + h, readied, err := a.ctl.Facility().Release(r.Context(), d) + if err != nil { + writeError(w, http.StatusNotFound, err.Error()) + return + } + a.ctl.Reconciler().Notify() + writeJSON(w, http.StatusOK, holdView(h, readied)) +} + +func (a *API) setNodeLabels(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var req NodeLabelsRequest + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + if len(req.Set) == 0 && len(req.Remove) == 0 { + writeError(w, http.StatusBadRequest, "nothing to change: give set, remove or both") + return + } + // A node not registered yet is allowed: labelling the machines of a + // rack before their agents start is the order a site is built in. A + // mistyped name is the risk, and the answer says it is not registered. + ops, err := a.ctl.Facility().SetLabels(r.Context(), id, req.Set, req.Remove, actorOf(r)) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + a.ctl.Reconciler().Notify() + v := NodeLabelsView{ID: id, OperatorLabels: ops} + if n, ok := a.ctl.Inventory().Get(id); ok { + v.Labels = n.Labels + if loc := n.Location(); !loc.Empty() { + v.Location = &loc + } + } + writeJSON(w, http.StatusOK, v) +} diff --git a/internal/control/adminapi/facility_test.go b/internal/control/adminapi/facility_test.go new file mode 100644 index 0000000..b129ed7 --- /dev/null +++ b/internal/control/adminapi/facility_test.go @@ -0,0 +1,68 @@ +package adminapi + +import ( + "net/http" + "slices" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" +) + +func TestARackIsLabelledHeldShownAndReleasedThroughTheAPI(t *testing.T) { + a, ctl := setup(t, testModels(), gpuNode("gpu01", 8), gpuNode("gpu02", 8)) + + var lv NodeLabelsView + if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels", + `{"set":{"llmgw.io/rack":"r12-07","llmgw.io/pdu":"p1,p2"}}`, &lv); code != http.StatusOK { + t.Fatalf("labelling: status %d", code) + } + if lv.Location == nil || lv.Location.Rack != "r12-07" || len(lv.Location.PDUs) != 2 { + t.Fatalf("location after labelling: %+v", lv.Location) + } + if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels", `{"set":{"llmgw.io/rak":"x"}}`, nil); code != http.StatusBadRequest { + t.Errorf("a misspelt location label: status %d, want 400", code) + } + + var hv HoldView + if code := call(t, a, http.MethodPost, "/api/v1/facility/holds", + `{"domain":"rack:r12-07","reason":"PDU A replacement","drain":true}`, &hv); code != http.StatusOK { + t.Fatalf("holding: status %d", code) + } + if !slices.Equal(hv.Nodes, []string{"gpu01"}) { + t.Errorf("drained %v, want gpu01", hv.Nodes) + } + + var fv FacilityView + call(t, a, http.MethodGet, "/api/v1/facility", "", &fv) + var rack *DomainView + for i, d := range fv.Domains { + if d.Domain == "rack:r12-07" { + rack = &fv.Domains[i] + } + } + if rack == nil || rack.Held == nil || rack.Held.Reason != "PDU A replacement" || !slices.Equal(rack.Nodes, []string{"gpu01"}) { + t.Fatalf("the rack in the facility view: %+v", rack) + } + if !slices.Equal(fv.Unlocated, []string{"gpu02"}) { + t.Errorf("unlocated %v, want gpu02", fv.Unlocated) + } + + var nodes []NodeView + call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes) + if nodes[0].Held != "rack r12-07 is held: PDU A replacement" || nodes[0].HeldDomain != "rack:r12-07" { + t.Errorf("gpu01 in the node list: held %q (%q)", nodes[0].Held, nodes[0].HeldDomain) + } + + if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", &hv); code != http.StatusOK { + t.Fatalf("releasing: status %d", code) + } + if _, held := ctl.Inventory().Facility().HeldIn([]facility.Domain{"rack:r12-07"}); held { + t.Error("released, still held") + } + if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", nil); code != http.StatusNotFound { + t.Errorf("releasing again: status %d, want 404", code) + } + if code := call(t, a, http.MethodPost, "/api/v1/facility/holds", `{"domain":"r12-07"}`, nil); code != http.StatusBadRequest { + t.Errorf("a domain with no level: status %d, want 400", code) + } +} diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index b7b5107..b199ef5 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -62,6 +62,7 @@ var PathParams = map[string]string{ "runs/{id}": "The quality run's ID, as starting it answered.", "requests/{id}": "The request's ID: the x-request-id a client was answered with.", "fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.", + "holds/{domain}": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.", "quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.", } @@ -84,6 +85,17 @@ var Routes = []Route{ {Method: "POST", Path: "/api/v1/nodes/{id}/engines", Summary: "Choose an engine's build for a node", Description: "The node installs the build chosen. Build is auto, a build such as cu129, or none.", Request: NodeEngineRequest{}, Response: NodeEngineRequest{}}, + {Method: "POST", Path: "/api/v1/nodes/{id}/labels", Summary: "Set or remove labels on a node, over its agent's own", + Description: "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.", + Request: NodeLabelsRequest{}, Response: NodeLabelsView{}}, + {Method: "GET", Path: "/api/v1/facility", Summary: "The failure domains, their power against their caps, and what is held", + Response: FacilityView{}}, + {Method: "POST", Path: "/api/v1/facility/holds", Summary: "Hold a domain for maintenance: nothing new is placed in it", + Description: "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.", + Request: HoldRequest{}, Response: HoldView{}}, + {Method: "DELETE", Path: "/api/v1/facility/holds/{domain}", Summary: "Release a held domain", + Description: "Nodes a draining hold drained are made ready again.", + Response: HoldView{}}, {Method: "DELETE", Path: "/api/v1/nodes/{id}", Summary: "Forget a node that has gone", Description: "Only a node that is lost, or unreachable with nothing on it; its certificate is revoked.", Response: NodeRemoved{}}, diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index ded29dc..5ae255f 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -14,6 +14,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" ) @@ -226,6 +227,21 @@ type Profile struct { // Agents have reported labels since the beginning; this is what reads them. NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"` + // PowerWatts is what one replica of this profile draws, in watts, split + // evenly over its nodes, counted against the power cap of every rack, + // PDU or other domain its nodes are in (facility.budgets in server.yaml). + // Zero derives it: each card at the power limit it reports, each + // reserved core at facility.cpu_core_watts. Only matters where a domain + // has a cap. + PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"` + + // KeepWithin keeps a multi-node replica's nodes inside one rack, row, + // room or site, read from their llmgw.io/rack (and so on) labels: for a + // fabric whose leaves do not reach across racks, or cables that do not. + // Nodes without the label are passed over for such a profile. Empty is + // no constraint. + KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"` + // Weight is the router's capacity weight. It should come from ctl bench // rather than a guess (plan §6.2.1). Weight float64 `json:"weight,omitempty" koanf:"weight"` @@ -715,6 +731,14 @@ func (m Model) validate() error { return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err) } + if p.PowerWatts < 0 { + return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts) + } + if p.KeepWithin != "" { + if _, err := facility.ParseLevel(p.KeepWithin); err != nil { + return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err) + } + } if p.GPUClass == "" && !cpuCapable(p.Engine) { return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", m.Name, p.Name, p.Engine) @@ -844,6 +868,8 @@ func (m Model) PlacementProfiles() []placement.Profile { MinMemoryGB: p.MinMemoryGB, MinDiskGB: p.MinDiskGB, CPUCores: p.CPUCores, + PowerWatts: p.PowerWatts, + KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))), // The model's source, so placement can prefer nodes that already // hold the weights rather than one that must first move them. Source: m.Source, diff --git a/internal/control/control.go b/internal/control/control.go index b75bfa2..7fcb9b9 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -78,15 +78,17 @@ type Controller struct { // engineChoices are the engine builds chosen per node; nil without a // store to keep them in. engineChoices *EngineChoices - cfg Config - log *slog.Logger - inv *inventory.Store - cat *catalog.Live - rec *reconciler.Reconciler - lead *Leadership - disp reconciler.Dispatcher - sink EndpointSink - usage UsageSource + // facility is the holds, budgets and operator labels placement reads. + facility *Facility + cfg Config + log *slog.Logger + inv *inventory.Store + cat *catalog.Live + rec *reconciler.Reconciler + lead *Leadership + disp reconciler.Dispatcher + sink EndpointSink + usage UsageSource mu sync.Mutex last string // fingerprint of the last published list @@ -106,15 +108,16 @@ func New(cfg Config, cat *catalog.Live, disp reconciler.Dispatcher, sink Endpoin } inv := inventory.NewStore() return &Controller{ - cfg: cfg, - log: log, - inv: inv, - cat: cat, - rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log), - disp: disp, - sink: sink, - stop: make(chan struct{}), - done: make(chan struct{}), + facility: NewFacility(inv, log), + cfg: cfg, + log: log, + inv: inv, + cat: cat, + rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log), + disp: disp, + sink: sink, + stop: make(chan struct{}), + done: make(chan struct{}), } } @@ -415,6 +418,9 @@ func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse() // SetEngineChoices gives the controller the engine builds chosen per node. func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e } +// Facility is where domains are held and nodes labelled; never nil. +func (c *Controller) Facility() *Facility { return c.facility } + // EngineChoices are the engine builds chosen per node, or nil. func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices } diff --git a/internal/control/facility.go b/internal/control/facility.go new file mode 100644 index 0000000..3b977e8 --- /dev/null +++ b/internal/control/facility.go @@ -0,0 +1,231 @@ +package control + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// FacilityStore is where holds and operator labels are kept. +type FacilityStore interface { + FacilityHolds(ctx context.Context) ([]store.FacilityHold, error) + SetFacilityHold(ctx context.Context, h store.FacilityHold) error + DeleteFacilityHold(ctx context.Context, domain string) error + NodeLabels(ctx context.Context) (map[string]map[string]string, error) + SetNodeLabel(ctx context.Context, node, key, value, by string) error +} + +// Facility is the operator's side of facility-aware placement: the budgets +// from server.yaml, the domains held for maintenance, and the labels set on +// nodes through the control plane. It keeps them in the store when there is +// one, and puts them in force in the inventory, where every placement — +// the reconciler's, headroom's and preemption's dry runs — reads them. +// +// Without a store it still works, for the life of the process: a held rack +// is worth honouring even on a control plane that will forget it. +type Facility struct { + inv *inventory.Store + log *slog.Logger + + mu sync.Mutex + st FacilityStore + base facility.Policy + holds map[facility.Domain]facility.Hold +} + +// NewFacility starts with nothing held and nothing capped. +func NewFacility(inv *inventory.Store, log *slog.Logger) *Facility { + if log == nil { + log = slog.Default() + } + return &Facility{inv: inv, log: log, holds: map[facility.Domain]facility.Hold{}} +} + +// SetPolicy puts the budgets and estimates from configuration in force. Its +// holds are ignored: those are the operator's, kept here. +func (f *Facility) SetPolicy(p facility.Policy) { + f.mu.Lock() + defer f.mu.Unlock() + f.base = p.Clone() + f.applyLocked() +} + +// Policy is what placement reads now. +func (f *Facility) Policy() facility.Policy { return f.inv.Facility() } + +// UseStore keeps holds and labels in the store from now on, and puts in +// force what it already holds. +func (f *Facility) UseStore(ctx context.Context, st FacilityStore) { + f.mu.Lock() + defer f.mu.Unlock() + f.st = st + if holds, err := st.FacilityHolds(ctx); err != nil { + f.log.Error("could not read the domains held for maintenance; none is held until they are set again", "err", err) + } else { + for _, h := range holds { + d, err := facility.ParseDomain(h.Domain) + if err != nil { + continue + } + f.holds[d] = facility.Hold{Domain: d, Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At} + } + } + if labels, err := st.NodeLabels(ctx); err != nil { + f.log.Error("could not read the labels set on nodes; nodes carry only their agents' until they are set again", "err", err) + } else { + for node, l := range labels { + f.inv.SetOperatorLabels(node, l) + } + } + f.applyLocked() +} + +func (f *Facility) applyLocked() { + p := f.base.Clone() + p.Holds = maps.Clone(f.holds) + f.inv.SetFacility(p) +} + +// Holds are the domains held, oldest first. +func (f *Facility) Holds() []facility.Hold { + f.mu.Lock() + defer f.mu.Unlock() + out := slices.Collect(maps.Values(f.holds)) + slices.SortFunc(out, func(a, b facility.Hold) int { + if c := a.At.Compare(b.At); c != 0 { + return c + } + return strings.Compare(string(a.Domain), string(b.Domain)) + }) + return out +} + +// Hold takes a domain out of placement, and with drain moves the work in it +// elsewhere: each of its nodes is drained, one replica at a time with the +// replacement up first, as `ctl drain` does. It answers the nodes drained. +func (f *Facility) Hold(ctx context.Context, h facility.Hold) ([]string, error) { + if h.At.IsZero() { + h.At = time.Now() + } + f.mu.Lock() + defer f.mu.Unlock() + if f.st != nil { + if err := f.st.SetFacilityHold(ctx, store.FacilityHold{ + Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, At: h.At, By: h.By, + }); err != nil { + return nil, fmt.Errorf("keeping the hold: %w", err) + } + } + f.holds[h.Domain] = h + f.applyLocked() + var drained []string + if h.Drain { + for _, n := range f.inv.Nodes() { + if slices.Contains(n.Domains(), h.Domain) && (n.State == inventory.Ready || n.State == inventory.Cordoned) { + f.inv.SetState(n.ID, inventory.Draining) + drained = append(drained, n.ID) + } + } + } + f.log.Info("domain held: nothing new is placed in it", "domain", h.Domain, "reason", h.Reason, + "drain", h.Drain, "by", h.By, "drained", drained) + return drained, nil +} + +// Release puts a domain back into placement. Nodes a draining hold drained +// are made ready again, unless another hold still covers them; a node an +// operator drained by itself is not told apart from those, so it is readied +// too, which is what releasing its rack asks for. +func (f *Facility) Release(ctx context.Context, d facility.Domain) (facility.Hold, []string, error) { + f.mu.Lock() + defer f.mu.Unlock() + h, ok := f.holds[d] + if !ok { + return facility.Hold{}, nil, fmt.Errorf("%s is not held; `llmgw ctl facility` lists what is", d) + } + if f.st != nil { + if err := f.st.DeleteFacilityHold(ctx, string(d)); err != nil { + return facility.Hold{}, nil, fmt.Errorf("releasing the hold: %w", err) + } + } + delete(f.holds, d) + f.applyLocked() + var readied []string + if h.Drain { + pol := f.inv.Facility() + for _, n := range f.inv.Nodes() { + doms := n.Domains() + if !slices.Contains(doms, d) || n.State != inventory.Draining { + continue + } + if _, still := pol.HeldIn(doms); still { + continue + } + f.inv.SetState(n.ID, inventory.Ready) + readied = append(readied, n.ID) + } + } + f.log.Info("domain released", "domain", d, "readied", readied) + return h, readied, nil +} + +// SetLabels sets and removes labels on a node through the control plane, +// over what its agent reports, and answers the operator's labels as they +// now are. A node not registered yet takes them when it does. +func (f *Facility) SetLabels(ctx context.Context, node string, set map[string]string, remove []string, by string) (map[string]string, error) { + for k, v := range set { + if err := checkLabel(k, v); err != nil { + return nil, err + } + } + f.mu.Lock() + defer f.mu.Unlock() + now := f.inv.OperatorLabels(node) + if now == nil { + now = map[string]string{} + } + if f.st != nil { + for _, k := range slices.Sorted(maps.Keys(set)) { + if err := f.st.SetNodeLabel(ctx, node, k, set[k], by); err != nil { + return nil, fmt.Errorf("keeping label %s: %w", k, err) + } + } + for _, k := range remove { + if err := f.st.SetNodeLabel(ctx, node, k, "", by); err != nil { + return nil, fmt.Errorf("removing label %s: %w", k, err) + } + } + } + maps.Copy(now, set) + for _, k := range remove { + delete(now, k) + } + f.inv.SetOperatorLabels(node, now) + f.log.Info("node labels set", "node", node, "set", set, "removed", remove, "by", by) + return now, nil +} + +// checkLabel refuses a key no selector could match, and a misspelt location: +// llmgw.io/rak would be kept, shown, and never read by placement. +func checkLabel(k, v string) error { + switch { + case strings.TrimSpace(k) == "" || strings.ContainsAny(k, " =\t\n"): + return fmt.Errorf("label key %q is empty or has a space or = in it", k) + case strings.TrimSpace(v) == "": + return fmt.Errorf("label %s has no value; remove it with %s- instead", k, k) + case strings.HasPrefix(k, facility.LabelPrefix): + if _, err := facility.ParseLevel(strings.TrimPrefix(k, facility.LabelPrefix)); err != nil { + return fmt.Errorf("label %s: %s is reserved for where a node is: llmgw.io/site, room, row, rack or pdu", k, facility.LabelPrefix) + } + } + return nil +} diff --git a/internal/control/facility/facility.go b/internal/control/facility/facility.go new file mode 100644 index 0000000..6f6fd98 --- /dev/null +++ b/internal/control/facility/facility.go @@ -0,0 +1,268 @@ +// Package facility is where nodes are in the building, and what the building +// allows: the racks, rows, rooms and power distribution units a node belongs +// to, which of them an operator has taken out of service, and how much power +// each may draw. +// +// Placement reads it to keep replicas of one model off a single rack or PDU, +// to keep a replica's shards inside one domain when its profile asks, to place +// nothing in a domain held for maintenance, and to refuse a replica that would +// take a domain past its power cap. Everything here is optional: a node with +// no location labels and a cluster with no facility configuration place +// exactly as they did before. +package facility + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// Level is a kind of failure domain, from the largest to the smallest. +type Level string + +const ( + Site Level = "site" + Room Level = "room" + Row Level = "row" + Rack Level = "rack" + // PDU is a power distribution unit. A node may be on several, one per + // feed of a dual-fed rack, so its label takes a comma separated list. + PDU Level = "pdu" +) + +// Levels are every level, the building's tree first and power after it. +var Levels = []Level{Site, Room, Row, Rack, PDU} + +// LabelPrefix starts the node labels that say where a node is. +const LabelPrefix = "llmgw.io/" + +// Label is the node label a level is read from: llmgw.io/rack. +func (l Level) Label() string { return LabelPrefix + string(l) } + +// ParseLevel reads a level's name, as a profile or a command gives it. +func ParseLevel(s string) (Level, error) { + l := Level(strings.ToLower(strings.TrimSpace(s))) + if slices.Contains(Levels, l) { + return l, nil + } + return "", fmt.Errorf("%q is not a facility level: use site, room, row, rack or pdu", s) +} + +// Domain names one failure domain as level:name, rack:r12-07. Names are taken +// as the labels give them, so a rack is one domain across the whole cluster: +// two rooms that both have a rack r1 must label them apart. +type Domain string + +// Of builds a domain from its level and name. +func Of(l Level, name string) Domain { return Domain(string(l) + ":" + name) } + +// Level is the domain's level. +func (d Domain) Level() Level { + l, _, _ := strings.Cut(string(d), ":") + return Level(l) +} + +// Name is the domain's name within its level. +func (d Domain) Name() string { + _, n, _ := strings.Cut(string(d), ":") + return n +} + +// String is level:name; human text says "rack r12-07". +func (d Domain) String() string { return string(d) } + +// Human is the domain as a sentence says it: rack r12-07. +func (d Domain) Human() string { return string(d.Level()) + " " + d.Name() } + +// ParseDomain reads level:name, the way commands and configuration name a +// domain. +func ParseDomain(s string) (Domain, error) { + l, n, ok := strings.Cut(strings.TrimSpace(s), ":") + if !ok || strings.TrimSpace(n) == "" { + return "", fmt.Errorf("%q is not a domain: name one as level:name, like rack:r12-07", s) + } + lv, err := ParseLevel(l) + if err != nil { + return "", err + } + return Of(lv, strings.TrimSpace(n)), nil +} + +// Location is where a node is, read from its labels. Empty fields are not +// known, which places as it always did. +type Location struct { + Site string `json:"site,omitempty"` + Room string `json:"room,omitempty"` + Row string `json:"row,omitempty"` + Rack string `json:"rack,omitempty"` + PDUs []string `json:"pdus,omitempty"` +} + +// LocationOf reads a node's labels. +func LocationOf(labels map[string]string) Location { + get := func(l Level) string { return strings.TrimSpace(labels[l.Label()]) } + loc := Location{Site: get(Site), Room: get(Room), Row: get(Row), Rack: get(Rack)} + for _, p := range strings.Split(get(PDU), ",") { + if p = strings.TrimSpace(p); p != "" && !slices.Contains(loc.PDUs, p) { + loc.PDUs = append(loc.PDUs, p) + } + } + return loc +} + +// Empty reports whether nothing is known about where the node is. +func (l Location) Empty() bool { + return l.Site == "" && l.Room == "" && l.Row == "" && l.Rack == "" && len(l.PDUs) == 0 +} + +// Domains are every domain the location is in, largest first. +func (l Location) Domains() []Domain { + var out []Domain + add := func(lv Level, name string) { + if name != "" { + out = append(out, Of(lv, name)) + } + } + add(Site, l.Site) + add(Room, l.Room) + add(Row, l.Row) + add(Rack, l.Rack) + for _, p := range l.PDUs { + add(PDU, p) + } + return out +} + +// At is the domain at one level, or empty. For PDUs it is the first. +func (l Location) At(lv Level) Domain { + for _, d := range l.Domains() { + if d.Level() == lv { + return d + } + } + return "" +} + +// Short is the location in a few characters for a table: the rack, else the +// row, room or site, with the PDUs after a slash. +func (l Location) Short() string { + s := "" + switch { + case l.Rack != "": + s = l.Rack + case l.Row != "": + s = "row " + l.Row + case l.Room != "": + s = l.Room + case l.Site != "": + s = l.Site + } + if len(l.PDUs) > 0 { + if s != "" { + s += " " + } + s += "pdu " + strings.Join(l.PDUs, ",") + } + return s +} + +// Hold takes a domain out of placement: maintenance, a cooling fault, work on +// a PDU. What runs there keeps running unless Drain is set. +type Hold struct { + Domain Domain `json:"domain"` + Reason string `json:"reason,omitempty"` + Drain bool `json:"drain,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` +} + +// Budget is a domain's power cap. +type Budget struct { + // CapWatts is what the domain may draw. For a dual-fed rack it is what + // one feed can carry alone, since that is what the rack must survive. + CapWatts float64 + // OtherWatts is drawn in the domain by things llmgw does not place on: + // switches, storage, nodes outside the cluster. + OtherWatts float64 +} + +// Policy is the facility's rules as placement reads them. The zero value +// holds nothing and caps nothing. +type Policy struct { + // Holds are the domains taken out of placement. + Holds map[Domain]Hold + // Budgets are the domains with a power cap. + Budgets map[Domain]Budget + // Margin is the share of every cap kept free, 0.1 for ten percent. + Margin float64 + // GPUWatts is a card's draw when its own power limit is not known. + GPUWatts float64 + // CPUCoreWatts is what a core reserved by a CPU replica draws. + CPUCoreWatts float64 + // NodeWatts is what a node draws beyond the GPUs and the cores given to + // replicas: its idle draw, fans, memory and disks. Counted for every node + // in a capped domain, busy or not. + NodeWatts float64 + // MeasuredFresh is how old a node's power reading may be and still be + // counted. Zero is two minutes. + MeasuredFresh time.Duration +} + +// Clone is a copy whose maps are its own. +func (p Policy) Clone() Policy { + c := p + if p.Holds != nil { + c.Holds = make(map[Domain]Hold, len(p.Holds)) + for k, v := range p.Holds { + c.Holds[k] = v + } + } + if p.Budgets != nil { + c.Budgets = make(map[Domain]Budget, len(p.Budgets)) + for k, v := range p.Budgets { + c.Budgets[k] = v + } + } + return c +} + +// HeldIn is the first hold among a node's domains, largest first. +func (p Policy) HeldIn(domains []Domain) (Hold, bool) { + for _, d := range domains { + if h, ok := p.Holds[d]; ok { + return h, true + } + } + return Hold{}, false +} + +// Limit is the most a domain may draw, its cap less the margin; zero with +// no cap. +func (p Policy) Limit(d Domain) float64 { + b, ok := p.Budgets[d] + if !ok || b.CapWatts <= 0 { + return 0 + } + m := p.Margin + if m < 0 || m >= 1 { + m = 0 + } + return b.CapWatts * (1 - m) +} + +// Fresh is how old a power reading may be. +func (p Policy) Fresh() time.Duration { + if p.MeasuredFresh > 0 { + return p.MeasuredFresh + } + return 2 * time.Minute +} + +// KW renders watts as a sentence about a rack would. +func KW(w float64) string { + if w >= 1000 { + return fmt.Sprintf("%.1f kW", w/1000) + } + return fmt.Sprintf("%.0f W", w) +} diff --git a/internal/control/facility/facility_test.go b/internal/control/facility/facility_test.go new file mode 100644 index 0000000..5d2b1ce --- /dev/null +++ b/internal/control/facility/facility_test.go @@ -0,0 +1,67 @@ +package facility + +import ( + "slices" + "testing" +) + +func TestALocationIsReadFromTheWellKnownLabels(t *testing.T) { + loc := LocationOf(map[string]string{ + "llmgw.io/site": "sto1", "llmgw.io/room": "hall-b", "llmgw.io/row": "12", + "llmgw.io/rack": "r12-07", "llmgw.io/pdu": "r12-07-a, r12-07-b,r12-07-a", "role": "gpu", + }) + want := []Domain{"site:sto1", "room:hall-b", "row:12", "rack:r12-07", "pdu:r12-07-a", "pdu:r12-07-b"} + if got := loc.Domains(); !slices.Equal(got, want) { + t.Errorf("domains %v, want %v", got, want) + } + if loc.Short() != "r12-07 pdu r12-07-a,r12-07-b" { + t.Errorf("short %q", loc.Short()) + } + if loc.At(Rack) != "rack:r12-07" || loc.At(PDU) != "pdu:r12-07-a" { + t.Errorf("at rack %q, at pdu %q", loc.At(Rack), loc.At(PDU)) + } +} + +func TestANodeWithoutLabelsIsNowhere(t *testing.T) { + loc := LocationOf(map[string]string{"role": "gpu"}) + if !loc.Empty() || len(loc.Domains()) != 0 || loc.Short() != "" { + t.Errorf("an unlabelled node is somewhere: %+v", loc) + } +} + +func TestADomainIsNamedAsLevelColonName(t *testing.T) { + d, err := ParseDomain(" Rack:r12-07 ") + if err != nil || d != "rack:r12-07" || d.Human() != "rack r12-07" { + t.Errorf("got %q, %v", d, err) + } + for _, bad := range []string{"r12-07", "rack:", "shelf:3"} { + if _, err := ParseDomain(bad); err == nil { + t.Errorf("%q was accepted as a domain", bad) + } + } +} + +func TestTheLimitIsTheCapLessTheMargin(t *testing.T) { + p := Policy{Margin: 0.1, Budgets: map[Domain]Budget{"rack:r1": {CapWatts: 16000}}} + if got := p.Limit("rack:r1"); got != 14400 { + t.Errorf("limit %v, want 14400", got) + } + if p.Limit("rack:r2") != 0 { + t.Error("a domain with no budget has a limit") + } +} + +func TestTheFirstHoldAmongANodesDomainsIsFound(t *testing.T) { + p := Policy{Holds: map[Domain]Hold{"pdu:p2": {Domain: "pdu:p2", Reason: "breaker"}}} + if h, ok := p.HeldIn([]Domain{"rack:r1", "pdu:p1", "pdu:p2"}); !ok || h.Reason != "breaker" { + t.Errorf("got %+v, %v", h, ok) + } + if _, ok := p.HeldIn([]Domain{"rack:r1"}); ok { + t.Error("an unheld rack is held") + } + c := p.Clone() + delete(c.Holds, "pdu:p2") + if _, ok := p.Holds["pdu:p2"]; !ok { + t.Error("changing a clone changed the original") + } +} diff --git a/internal/control/facility_test.go b/internal/control/facility_test.go new file mode 100644 index 0000000..95d55c5 --- /dev/null +++ b/internal/control/facility_test.go @@ -0,0 +1,132 @@ +package control + +import ( + "context" + "log/slog" + "maps" + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// memFacilityStore is the store's two tables in memory. +type memFacilityStore struct { + holds map[string]store.FacilityHold + labels map[string]map[string]string +} + +func newMemFacilityStore() *memFacilityStore { + return &memFacilityStore{holds: map[string]store.FacilityHold{}, labels: map[string]map[string]string{}} +} + +func (m *memFacilityStore) FacilityHolds(context.Context) ([]store.FacilityHold, error) { + return slices.Collect(maps.Values(m.holds)), nil +} +func (m *memFacilityStore) SetFacilityHold(_ context.Context, h store.FacilityHold) error { + m.holds[h.Domain] = h + return nil +} +func (m *memFacilityStore) DeleteFacilityHold(_ context.Context, d string) error { + delete(m.holds, d) + return nil +} +func (m *memFacilityStore) NodeLabels(context.Context) (map[string]map[string]string, error) { + return m.labels, nil +} +func (m *memFacilityStore) SetNodeLabel(_ context.Context, node, k, v, _ string) error { + if m.labels[node] == nil { + m.labels[node] = map[string]string{} + } + if v == "" { + delete(m.labels[node], k) + } else { + m.labels[node][k] = v + } + return nil +} + +func rackNode(id, rack string) inventory.Node { + return inventory.Node{ID: id, State: inventory.Ready, Labels: map[string]string{"llmgw.io/rack": rack}} +} + +func TestADrainingHoldDrainsTheRackAndReleasingItReadiesItAgain(t *testing.T) { + inv := inventory.NewStore() + inv.Upsert(rackNode("a1", "r1")) + inv.Upsert(rackNode("a2", "r1")) + inv.Upsert(rackNode("b1", "r2")) + f := NewFacility(inv, slog.New(slog.DiscardHandler)) + ctx := t.Context() + + drained, err := f.Hold(ctx, facility.Hold{Domain: "rack:r1", Reason: "PDU swap", Drain: true}) + if err != nil || !slices.Equal(drained, []string{"a1", "a2"}) { + t.Fatalf("drained %v, %v", drained, err) + } + if n, _ := inv.Get("b1"); n.State != inventory.Ready { + t.Errorf("b1, in another rack, is %s", n.State) + } + if _, held := inv.Facility().HeldIn([]facility.Domain{"rack:r1"}); !held { + t.Error("the hold is not in force for placement") + } + + _, readied, err := f.Release(ctx, "rack:r1") + if err != nil || !slices.Equal(readied, []string{"a1", "a2"}) { + t.Fatalf("readied %v, %v", readied, err) + } + if _, _, err := f.Release(ctx, "rack:r1"); err == nil { + t.Error("releasing a rack not held is not an error") + } +} + +func TestAHoldWithoutDrainLeavesTheNodesAsTheyAre(t *testing.T) { + inv := inventory.NewStore() + inv.Upsert(rackNode("a1", "r1")) + f := NewFacility(inv, nil) + if drained, _ := f.Hold(t.Context(), facility.Hold{Domain: "rack:r1"}); len(drained) != 0 { + t.Errorf("drained %v without being asked", drained) + } + if n, _ := inv.Get("a1"); n.State != inventory.Ready { + t.Errorf("a1 is %s", n.State) + } +} + +func TestHoldsAndLabelsAreKeptAndReadBackByTheNextControlPlane(t *testing.T) { + st := newMemFacilityStore() + inv := inventory.NewStore() + f := NewFacility(inv, nil) + f.UseStore(t.Context(), st) + if _, err := f.Hold(t.Context(), facility.Hold{Domain: "pdu:p1", Reason: "breaker", By: "ilker"}); err != nil { + t.Fatal(err) + } + if _, err := f.SetLabels(t.Context(), "gpu01", map[string]string{"llmgw.io/rack": "r9"}, nil, "ilker"); err != nil { + t.Fatal(err) + } + + // A restart: a new inventory and facility over the same store. + inv2 := inventory.NewStore() + f2 := NewFacility(inv2, nil) + f2.SetPolicy(facility.Policy{Margin: 0.1}) + f2.UseStore(t.Context(), st) + inv2.Upsert(inventory.Node{ID: "gpu01", State: inventory.Ready}) + if n, _ := inv2.Get("gpu01"); n.Labels["llmgw.io/rack"] != "r9" { + t.Errorf("the label set before the restart is gone: %v", n.Labels) + } + pol := inv2.Facility() + if h, ok := pol.Holds["pdu:p1"]; !ok || h.Reason != "breaker" || pol.Margin != 0.1 { + t.Errorf("after the restart: holds %v, margin %v", pol.Holds, pol.Margin) + } +} + +func TestAMisspeltLocationLabelIsRefused(t *testing.T) { + f := NewFacility(inventory.NewStore(), nil) + _, err := f.SetLabels(t.Context(), "n1", map[string]string{"llmgw.io/rak": "r1"}, nil, "") + if err == nil || !strings.Contains(err.Error(), "llmgw.io/site, room, row, rack or pdu") { + t.Errorf("got %v", err) + } + if _, err := f.SetLabels(t.Context(), "n1", map[string]string{"team": "nlp"}, nil, ""); err != nil { + t.Errorf("a free label was refused: %v", err) + } +} diff --git a/internal/control/inventory/facility.go b/internal/control/inventory/facility.go new file mode 100644 index 0000000..87eda43 --- /dev/null +++ b/internal/control/inventory/facility.go @@ -0,0 +1,176 @@ +package inventory + +import ( + "maps" + "slices" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" +) + +// Location is where the node is, from its labels. +func (n Node) Location() facility.Location { return facility.LocationOf(n.Labels) } + +// Domains are the failure domains the node is in, largest first. +func (n Node) Domains() []facility.Domain { return n.Location().Domains() } + +// mergeLabels lays the operator's labels over the agent's. A copy, so +// neither is written through. +func mergeLabels(agent, operator map[string]string) map[string]string { + if len(operator) == 0 { + return maps.Clone(agent) + } + out := make(map[string]string, len(agent)+len(operator)) + maps.Copy(out, agent) + maps.Copy(out, operator) + return out +} + +// SetFacility puts the facility's holds and power budgets in force for +// every placement from now on. +func (s *Store) SetFacility(p facility.Policy) { + s.mu.Lock() + defer s.mu.Unlock() + s.facility = p.Clone() +} + +// Facility is the policy in force; a copy, so a caller may change it. +func (s *Store) Facility() facility.Policy { + s.mu.RLock() + defer s.mu.RUnlock() + return s.facility.Clone() +} + +// SetOperatorLabels replaces the labels set through the control plane for a +// node, which may not have registered yet: they are laid over its agent's +// when it does. +func (s *Store) SetOperatorLabels(id string, labels map[string]string) { + s.mu.Lock() + defer s.mu.Unlock() + if s.operatorLabels == nil { + s.operatorLabels = map[string]map[string]string{} + } + if len(labels) == 0 { + delete(s.operatorLabels, id) + } else { + s.operatorLabels[id] = maps.Clone(labels) + } + if n, ok := s.nodes[id]; ok { + n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[id]) + } +} + +// OperatorLabels are the labels set through the control plane for a node. +func (s *Store) OperatorLabels(id string) map[string]string { + s.mu.RLock() + defer s.mu.RUnlock() + return maps.Clone(s.operatorLabels[id]) +} + +// PowerUse is one domain's draw as placement counts it. +type PowerUse struct { + Domain facility.Domain + // Nodes are the domain's nodes, sorted. + Nodes []string + // Reserved is what the replicas placed in the domain are estimated to + // draw, with each node's own draw and the domain's other equipment. + Reserved float64 + // Measured is what the nodes' sensors read, with the same additions, + // for the nodes that read anything; the others count what is reserved. + Measured float64 + // Used is what placement counts against the cap: per node, the larger + // of reserved and measured, so an estimate that was too low is + // corrected by the room it stops offering rather than by a breaker. + Used float64 + // Limit is the cap less the margin; zero is no cap. + Limit float64 + Budget facility.Budget +} + +// PowerUse is every domain any node is in, or that has a budget, by domain. +func (s *Store) PowerUse(now time.Time) map[facility.Domain]PowerUse { + s.mu.RLock() + defer s.mu.RUnlock() + pol := s.facility + out := map[facility.Domain]PowerUse{} + for d, b := range pol.Budgets { + out[d] = PowerUse{Domain: d, Budget: b, Limit: pol.Limit(d), + Reserved: b.OtherWatts, Measured: b.OtherWatts, Used: b.OtherWatts} + } + ids := slices.Sorted(maps.Keys(s.nodes)) + for _, id := range ids { + n := s.nodes[id] + reserved := 0.0 + for _, a := range s.allocs[id] { + reserved += AllocationWatts(*n, a, pol) + } + measured, known := MeasuredWatts(*n, now, pol.Fresh()) + used := reserved + if known && measured > used { + used = measured + } + if !known { + measured = reserved + } + for _, d := range n.Domains() { + u, ok := out[d] + if !ok { + u = PowerUse{Domain: d} + } + u.Nodes = append(u.Nodes, id) + u.Reserved += pol.NodeWatts + reserved + u.Measured += pol.NodeWatts + measured + u.Used += pol.NodeWatts + used + out[d] = u + } + } + return out +} + +// AllocationWatts is what one shard is estimated to draw: what its profile +// declared, or else each of its cards at the power limit the card reports +// (the policy's figure for a card that reports none) and each of its cores +// at the policy's figure. +func AllocationWatts(n Node, a Allocation, pol facility.Policy) float64 { + if a.PowerW > 0 { + return a.PowerW + } + w := float64(a.CPUCores) * pol.CPUCoreWatts + for _, i := range a.GPUIndices { + w += GPUWatts(n, i, pol) + } + return w +} + +// GPUWatts is one card's estimated draw: the power limit it reports, else +// the policy's figure for a card. +func GPUWatts(n Node, index int, pol facility.Policy) float64 { + if n.Load != nil { + for _, g := range n.Load.GPUs { + if g.Index == index && g.PowerLimitW > 0 { + return g.PowerLimitW + } + } + } + return pol.GPUWatts +} + +// MeasuredWatts is what the node's sensors read at its last heartbeat: its +// cards and its CPU packages. Known is false when the reading is older than +// fresh, or the node has no power sensor at all, which a VM never has. +func MeasuredWatts(n Node, now time.Time, fresh time.Duration) (w float64, known bool) { + if n.Load == nil || now.Sub(n.Load.At) > fresh { + return 0, false + } + for _, g := range n.Load.GPUs { + if g.PowerW > 0 { + w += g.PowerW + known = true + } + } + if n.Load.CPUPowerW > 0 { + w += n.Load.CPUPowerW + known = true + } + return w, known +} diff --git a/internal/control/inventory/facility_test.go b/internal/control/inventory/facility_test.go new file mode 100644 index 0000000..83d8703 --- /dev/null +++ b/internal/control/inventory/facility_test.go @@ -0,0 +1,79 @@ +package inventory + +import ( + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" +) + +func TestOperatorLabelsWinOverTheAgentsAndSurviveItsHeartbeats(t *testing.T) { + s := NewStore() + // Set before the node registers: labelling a rack's machines comes first. + s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"}) + s.Upsert(Node{ID: "n1", Labels: map[string]string{"llmgw.io/rack": "r1", "role": "gpu"}}) + + n, _ := s.Get("n1") + if n.Labels["llmgw.io/rack"] != "r2" || n.Labels["role"] != "gpu" { + t.Fatalf("labels %v: the operator's rack over the agent's, the agent's role kept", n.Labels) + } + // A heartbeat hands back the node it read. + s.Upsert(n) + s.Upsert(n) + if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r2" || n.AgentLabels["llmgw.io/rack"] != "r1" { + t.Fatalf("after heartbeats: labels %v, agent's %v", n.Labels, n.AgentLabels) + } + + s.SetOperatorLabels("n1", nil) + if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r1" { + t.Errorf("the operator's label removed, rack is %q; want the agent's r1 back", n.Labels["llmgw.io/rack"]) + } +} + +func TestAnOperatorLabelOnANodeWithNoAgentLabelsIsNotTakenForTheAgents(t *testing.T) { + s := NewStore() + s.Upsert(Node{ID: "n1"}) + s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"}) + n, _ := s.Get("n1") + s.Upsert(n) + s.SetOperatorLabels("n1", nil) + if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "" { + t.Errorf("removed, but the node still carries rack %q", n.Labels["llmgw.io/rack"]) + } +} + +func TestADomainsPowerIsTheLargerOfReservedAndMeasuredNodeByNode(t *testing.T) { + s := NewStore() + gpus := []discovery.GPU{{Index: 0}, {Index: 1}} + rack := map[string]string{"llmgw.io/rack": "r1"} + now := time.Now() + // a: two cards placed at a 700 W limit, drawing 300 each: 1400 reserved. + s.Upsert(Node{ID: "a", Labels: rack, GPUs: gpus}) + s.Upsert(Node{ID: "b", Labels: rack, GPUs: gpus}) + setLoad(s, "a", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerLimitW: 700, PowerW: 300}, {Index: 1, PowerLimitW: 700, PowerW: 300}}}) + // b: nothing placed, but drawing 500 W. + setLoad(s, "b", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerW: 250}, {Index: 1, PowerW: 250}}}) + s.Allocate("a", Allocation{ShardID: "x", GPUIndices: []int{0, 1}}) + s.SetFacility(facility.Policy{NodeWatts: 100, Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000, OtherWatts: 50}}}) + + u := s.PowerUse(now)["rack:r1"] + if u.Reserved != 50+100+1400+100+0 { + t.Errorf("reserved %v", u.Reserved) + } + if u.Measured != 50+100+600+100+500 { + t.Errorf("measured %v", u.Measured) + } + if u.Used != 50+100+1400+100+500 { + t.Errorf("used %v: a's reservation and b's reading", u.Used) + } + if len(u.Nodes) != 2 || u.Limit != 5000 { + t.Errorf("nodes %v, limit %v", u.Nodes, u.Limit) + } +} + +func setLoad(s *Store, id string, l *Load) { + n, _ := s.Get(id) + n.Load = l + s.Upsert(n) +} diff --git a/internal/control/inventory/inventory.go b/internal/control/inventory/inventory.go index 1f86b0e..ebb41f9 100644 --- a/internal/control/inventory/inventory.go +++ b/internal/control/inventory/inventory.go @@ -11,6 +11,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" ) // NodeState is where a node stands. @@ -71,10 +72,17 @@ type FabricAttachment struct { // Node is one machine in the cluster. type Node struct { - ID string `json:"id"` - Hostname string `json:"hostname"` - MgmtAddr string `json:"mgmt_addr"` - Labels map[string]string `json:"labels,omitempty"` + ID string `json:"id"` + Hostname string `json:"hostname"` + MgmtAddr string `json:"mgmt_addr"` + // Labels are what placement matches: the agent's own, from agent.yaml, + // with any an operator set through the control plane over them. + Labels map[string]string `json:"labels,omitempty"` + // AgentLabels are the labels the agent reported, kept apart so an + // operator's label removed brings the agent's value back. Nil on a node + // built anywhere but a registration; Upsert then takes Labels as the + // agent's. + AgentLabels map[string]string `json:"agent_labels,omitempty"` // MemoryTotalMB is the machine's RAM. Zero means the agent did not report // it, which is treated as no constraint — the same as every node before @@ -461,6 +469,10 @@ type Allocation struct { // CPUCores is the cores this shard was given to compute on, for a shard // with no GPU; zero for one that shares whatever is free. CPUCores int + // PowerW is the draw this shard's profile declared; zero derives it from + // the node's cards and the cores, which is what most profiles leave to + // placement. + PowerW float64 } // Store holds the cluster inventory. It is safe for concurrent use. @@ -473,6 +485,11 @@ type Store struct { smChanges []SMChange // scans are the latest ibnetdiscover of each physical fabric, by key. scans map[string]FabricScan + // facility is the holds and power budgets placement reads. + facility facility.Policy + // operatorLabels are labels set through the control plane, by node, + // laid over what each agent reports. + operatorLabels map[string]map[string]string } // NewStore returns an empty inventory. @@ -494,6 +511,7 @@ func (s *Store) Clone() *Store { for id, list := range s.allocs { c.allocs[id] = slices.Clone(list) } + c.facility = s.facility.Clone() return c } @@ -564,6 +582,17 @@ func (s *Store) Upsert(n Node) { n.State = prev.State } } + // A registration carries the agent's labels in Labels; a heartbeat hands + // back the node it read, whose Labels already have the operator's in + // them, and its AgentLabels set. Never nil after this, so the second case + // is always told from the first. + if n.AgentLabels == nil { + n.AgentLabels = maps.Clone(n.Labels) + if n.AgentLabels == nil { + n.AgentLabels = map[string]string{} + } + } + n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[n.ID]) n.LastSeen = time.Now() s.nodes[n.ID] = &n } diff --git a/internal/control/placement/facility.go b/internal/control/placement/facility.go new file mode 100644 index 0000000..fb8e1ad --- /dev/null +++ b/internal/control/placement/facility.go @@ -0,0 +1,193 @@ +package placement + +import ( + "fmt" + "slices" + "sort" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// declaredShardWatts is the profile's declared draw for one node's shard. +func (p Profile) declaredShardWatts() float64 { + if p.PowerWatts <= 0 { + return 0 + } + return p.PowerWatts / float64(max(p.Nodes, 1)) +} + +// shardWatts is what one shard of this profile is estimated to draw on a +// candidate: the declared figure, or its cards at the highest power limit +// among the node's free ones of the class (the cards are not chosen yet, and +// guessing low is the mistake that trips a breaker), or its cores. +func (s *Scheduler) shardWatts(p Profile, c candidate) float64 { + if w := p.declaredShardWatts(); w > 0 { + return w + } + pol := s.inv.Facility() + if p.CPUOnly() { + return float64(p.CPUCores) * pol.CPUCoreWatts + } + per := 0.0 + for _, g := range c.free { + per = max(per, inventory.GPUWatts(c.node, g.Index, pol)) + } + if per == 0 { + per = pol.GPUWatts + } + return per * float64(p.GPUsPerNode) +} + +// budget is the power budgets as one placement decision spends them: the +// domains' use when it began, and what the shards it has chosen so far add. +type budget struct { + pol facility.Policy + use map[facility.Domain]inventory.PowerUse + added map[facility.Domain]float64 +} + +// newBudget reads the domains' use, once per decision. Without budgets there +// is nothing to read and every check passes. +func (s *Scheduler) newBudget() *budget { + pol := s.inv.Facility() + b := &budget{pol: pol, added: map[facility.Domain]float64{}} + if len(pol.Budgets) > 0 { + b.use = s.inv.PowerUse(s.clock()) + } + return b +} + +// check says why a shard drawing need watts on a node would take one of the +// node's domains past its limit; empty when it fits them all. A shard whose +// draw is not known (no declared figure, no card limit, no policy figure) +// needs nothing: missing data never blocks placement. +func (b *budget) check(n inventory.Node, need float64) string { + if b.use == nil { + return "" + } + for _, d := range n.Domains() { + limit := b.pol.Limit(d) + if limit <= 0 { + continue + } + used := b.use[d].Used + b.added[d] + if used+need > limit { + capW := b.pol.Budgets[d].CapWatts + of := facility.KW(limit) + " cap" + if limit < capW { + of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW) + } + return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s", + d.Human(), facility.KW(used), of, facility.KW(need)) + } + } + return "" +} + +// add counts a chosen shard against the node's domains. +func (b *budget) add(n inventory.Node, need float64) { + for _, d := range n.Domains() { + b.added[d] += need + } +} + +// clock is the scheduler's time, for the freshness of power readings. +func (s *Scheduler) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +// heldReason is a hold as a rejection says it. +func heldReason(h facility.Hold) string { + why := h.Domain.Human() + " is held" + if h.Reason != "" { + why += ": " + h.Reason + } + return why +} + +// spreadLevels are the domains replicas of one model are kept apart in: +// the rack, which a top-of-rack switch or a door left open takes out whole, +// and the PDU, which a breaker does. +var spreadLevels = []facility.Level{facility.Rack, facility.PDU} + +// spreadAcrossDomains moves to the front the candidates sharing the fewest +// racks and PDUs with the nodes the model already runs on, keeping the order +// given among equals. +// +// A preference and never a filter, as spreading across nodes is: a rack +// holding a model's only free cards still takes its second replica, which +// survives an engine crash if not a breaker. Nodes with no location share +// nothing and lose nothing, so a cluster without labels places as before. +func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) { + if len(avoid) == 0 { + return + } + taken := map[facility.Domain]bool{} + for _, id := range avoid { + if n, ok := s.inv.Get(id); ok { + for _, d := range n.Domains() { + if slices.Contains(spreadLevels, d.Level()) { + taken[d] = true + } + } + } + } + if len(taken) == 0 { + return + } + shared := func(c candidate) int { + k := 0 + for _, d := range c.node.Domains() { + if taken[d] { + k++ + } + } + return k + } + sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) }) +} + +// withinGroups divides a multi-node replica's candidates by the domain its +// profile keeps it within, largest first, and rejects the nodes that do not +// say which they are in. Without the constraint it is all of them, as one. +func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate { + if p.KeepWithin == "" { + return [][]candidate{cands} + } + by := map[facility.Domain][]candidate{} + for _, c := range cands { + d := c.node.Location().At(p.KeepWithin) + if d == "" { + *rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID, + Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s", + p.KeepWithin.Label(), p.KeepWithin)}) + continue + } + by[d] = append(by[d], c) + } + keys := make([]facility.Domain, 0, len(by)) + for d := range by { + keys = append(keys, d) + } + sort.Slice(keys, func(i, j int) bool { + if len(by[keys[i]]) != len(by[keys[j]]) { + return len(by[keys[i]]) > len(by[keys[j]]) + } + return keys[i] < keys[j] + }) + out := make([][]candidate, 0, len(keys)) + for _, d := range keys { + if len(by[d]) < p.Nodes { + *rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, + Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))}) + continue + } + out = append(out, by[d]) + } + return out +} diff --git a/internal/control/placement/facility_test.go b/internal/control/placement/facility_test.go new file mode 100644 index 0000000..5365fbb --- /dev/null +++ b/internal/control/placement/facility_test.go @@ -0,0 +1,325 @@ +package placement + +import ( + "fmt" + "math/rand/v2" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// at puts a node in a rack, and on PDUs when given. +func at(n inventory.Node, rack string, pdus ...string) inventory.Node { + if n.Labels == nil { + n.Labels = map[string]string{} + } + n.Labels[facility.Rack.Label()] = rack + if len(pdus) > 0 { + n.Labels[facility.PDU.Label()] = strings.Join(pdus, ",") + } + return n +} + +// withPowerLimit has a node's cards report a power limit, and draw, as a +// heartbeat would. +func withPowerLimit(n inventory.Node, limit, draw float64) inventory.Node { + l := &inventory.Load{At: time.Now()} + for _, g := range n.GPUs { + l.GPUs = append(l.GPUs, inventory.GPUSensor{Index: g.Index, PowerLimitW: limit, PowerW: draw}) + } + n.Load = l + return n +} + +func onePerNode(name string, gpus int) Profile { + return Profile{Name: name, GPUClass: "h100-80g", GPUsPerNode: gpus, Nodes: 1} +} + +func TestNothingIsPlacedInAHeldRackAndTheRejectionSaysWhy(t *testing.T) { + st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("b1", "h100-80g", 8), "r2")) + st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{ + "rack:r2": {Domain: "rack:r2", Reason: "PDU A replacement"}, + }}) + st.Allocate("a1", inventory.Allocation{ShardID: "x", GPUIndices: []int{0, 1, 2, 3, 4, 5, 6, 7}}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil) + if res.Placement != nil { + t.Fatalf("placed on %s, in a held rack", res.Placement.Nodes[0].NodeID) + } + if !strings.Contains(res.Explain(), "b1: rack r2 is held: PDU A replacement") { + t.Errorf("the rejection does not name the hold:\n%s", res.Explain()) + } +} + +func TestAHeldPDUHoldsEveryRackOnIt(t *testing.T) { + st := store(at(node("a1", "h100-80g", 8), "r1", "p1", "p2"), at(node("b1", "h100-80g", 8), "r2", "p3")) + st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{"pdu:p2": {Domain: "pdu:p2"}}}) + + res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" { + t.Fatalf("want b1, the node on no held PDU:\n%s", res.Explain()) + } +} + +func TestASecondReplicaGoesToAnotherRackBeforeAFullerNodeInTheSameOne(t *testing.T) { + // Bin packing alone would choose a2, the partly used node beside the + // first replica; a rack apart is worth an emptier node. + st := store( + at(node("a1", "h100-80g", 8), "r1"), + at(node("a2", "h100-80g", 8), "r1"), + at(node("b1", "h100-80g", 8), "r2"), + ) + st.Allocate("a2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"}) + if res.Placement == nil { + t.Fatalf("not placed:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "b1" { + t.Errorf("placed on %s, in the rack of the first replica; want b1 in another", got) + } +} + +func TestReplicasAreSpreadAcrossPDUsAsWellAsRacks(t *testing.T) { + st := store( + at(node("a1", "h100-80g", 8), "r1", "p1"), + at(node("b1", "h100-80g", 8), "r2", "p1"), + at(node("c1", "h100-80g", 8), "r3", "p2"), + ) + st.Allocate("b1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"}) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "c1" { + t.Fatalf("want c1, on another rack and another PDU:\n%s", res.Explain()) + } +} + +func TestSpreadingAcrossRacksNeverLeavesAReplicaUnplaced(t *testing.T) { + st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("a2", "h100-80g", 8), "r1")) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"}) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "a2" { + t.Fatalf("the rack's other node should still be used:\n%s", res.Explain()) + } +} + +func TestNodesWithoutALocationPlaceAsTheyAlwaysDid(t *testing.T) { + // Bin packing picks the partly used node; nothing about racks changes it. + st := store(node("n1", "h100-80g", 8), node("n2", "h100-80g", 8)) + st.Allocate("n2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1}}) + st.SetFacility(facility.Policy{GPUWatts: 700, Margin: 0.1}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "n2" { + t.Fatalf("want n2, the bin-packed choice:\n%s", res.Explain()) + } +} + +func TestAReplicaThatWouldTakeARackPastItsCapGoesToAnotherRack(t *testing.T) { + // Rack r1: 8 cards at 700 W, four of them placed: 2.8 kW used of a + // 4.5 kW limit (5 kW less 10%). Four more need 2.8 kW, which does not fit. + st := store( + withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0), + withPowerLimit(at(node("b1", "h100-80g", 8), "r2"), 700, 0), + ) + st.Allocate("a1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}}) + st.SetFacility(facility.Policy{Margin: 0.1, Budgets: map[facility.Domain]facility.Budget{ + "rack:r1": {CapWatts: 5000}, + }}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil) + if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" { + t.Fatalf("want b1, in the rack with room:\n%s", res.Explain()) + } + want := "a1: rack r1: 2.8 kW of a 4.5 kW limit (5.0 kW cap less 10%) in use, the shard needs ~2.8 kW" + if !strings.Contains(res.Explain(), want) { + t.Errorf("the rejection should give the figures, %q:\n%s", want, res.Explain()) + } +} + +func TestAProfilesDeclaredDrawIsWhatIsCounted(t *testing.T) { + st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0)) + st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 2000}}}) + + derived := onePerNode("tp4", 4) // 4 × 700 W: too much for 2 kW + if res := New(st).Place([]Profile{derived}, nil); res.Placement != nil { + t.Fatalf("2.8 kW was placed under a 2 kW cap") + } + declared := derived + declared.PowerWatts = 1500 + res := New(st).Place([]Profile{declared}, nil) + if res.Placement == nil { + t.Fatalf("a profile declaring 1.5 kW was refused under a 2 kW cap:\n%s", res.Explain()) + } + if res.Placement.Nodes[0].PowerW != 1500 { + t.Errorf("the shard carries %v W to record, want the declared 1500", res.Placement.Nodes[0].PowerW) + } +} + +func TestWhatTheSensorsReadCountsWhenItIsMoreThanWasReserved(t *testing.T) { + // Nothing placed, but the cards draw 400 W each: 3.2 kW measured, and + // another 2.8 kW would pass a 5 kW cap only on the reservations. + st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400)) + st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}}) + + res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil) + if res.Placement != nil { + t.Fatalf("placed against reservations alone, with 3.2 kW measured") + } + if !strings.Contains(res.Explain(), "3.2 kW of a 5.0 kW cap in use") { + t.Errorf("the measured draw should be what is in use:\n%s", res.Explain()) + } +} + +func TestAStaleReadingIsNotCounted(t *testing.T) { + n := withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400) + n.Load.At = time.Now().Add(-10 * time.Minute) + st := store(n) + st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}}) + + if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement == nil { + t.Fatalf("a ten-minute-old reading kept a replica out:\n%s", res.Explain()) + } +} + +func TestADrawNobodyKnowsNeverBlocksPlacement(t *testing.T) { + // A cap, but no card reports a limit and no figure is configured: the + // shard's draw is unknown, which is not a reason to refuse it. + st := store(at(node("a1", "h100-80g", 8), "r1")) + st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 100}}}) + + if res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil); res.Placement == nil { + t.Fatalf("an unknown draw was refused:\n%s", res.Explain()) + } +} + +func TestTheOtherEquipmentInARackCountsAgainstItsCap(t *testing.T) { + st := store(at(node("a1", "h100-80g", 8), "r1")) + st.SetFacility(facility.Policy{GPUWatts: 500, Budgets: map[facility.Domain]facility.Budget{ + "rack:r1": {CapWatts: 5000, OtherWatts: 3500}, + }}) + + if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement != nil { + t.Fatal("2 kW was placed where switches already take 3.5 of 5") + } +} + +func TestAMultiNodeReplicasShardsAreCountedTogetherAgainstTheirRack(t *testing.T) { + // Each node alone fits r1's budget; two together do not, so the second + // shard goes to the rack with room — on the same fabric, as always. + st := store( + withPowerLimit(at(node("a1", "h100-80g", 8, "ib"), "r1"), 500, 0), + withPowerLimit(at(node("a2", "h100-80g", 8, "ib"), "r1"), 500, 0), + withPowerLimit(at(node("b1", "h100-80g", 8, "ib"), "r2"), 500, 0), + ) + st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}}) + + res := New(st).Place([]Profile{{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil) + if res.Placement == nil { + t.Fatalf("not placed:\n%s", res.Explain()) + } + racks := map[string]int{} + for _, a := range res.Placement.Nodes { + n, _ := st.Get(a.NodeID) + racks[n.Location().Rack]++ + } + if racks["r1"] != 1 || racks["r2"] != 1 { + t.Errorf("shards by rack %v: two in r1 draw 8 kW against a 6 kW cap", racks) + } + if !strings.Contains(res.Explain(), "with this replica's other shards") { + t.Errorf("the node passed over should say why:\n%s", res.Explain()) + } +} + +func TestAProfileKeptWithinARackNeverSpansTwo(t *testing.T) { + // The emptiest-first order would take a1 and b1; the profile wants one + // rack, and only r2 has two nodes. + st := store( + at(node("a1", "h100-80g", 8, "ib"), "r1"), + at(node("b1", "h100-80g", 8, "ib"), "r2"), + at(node("b2", "h100-80g", 8, "ib"), "r2"), + node("x1", "h100-80g", 8, "ib"), + ) + p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack} + + res := New(st).Place([]Profile{p}, nil) + if res.Placement == nil { + t.Fatalf("not placed:\n%s", res.Explain()) + } + for _, a := range res.Placement.Nodes { + if a.NodeID != "b1" && a.NodeID != "b2" { + t.Errorf("a shard on %s, outside rack r2", a.NodeID) + } + } + explain := res.Explain() + if !strings.Contains(explain, "x1: has no llmgw.io/rack label, and the profile keeps its nodes within one rack") { + t.Errorf("the unlabelled node should be named:\n%s", explain) + } + if !strings.Contains(explain, "needs 2 nodes within one rack, rack r1 has 1") { + t.Errorf("the rack too small should be named:\n%s", explain) + } +} + +func TestAProfileKeptWithinARackIsNotPlacedAcrossRacks(t *testing.T) { + st := store(at(node("a1", "h100-80g", 8, "ib"), "r1"), at(node("b1", "h100-80g", 8, "ib"), "r2")) + p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack} + if res := New(st).Place([]Profile{p}, nil); res.Placement != nil { + t.Fatal("a replica kept within one rack was placed across two") + } +} + +// TestNoCapIsEverExceeded fills random clusters until nothing more fits, and +// checks every capped domain stayed within its limit — and that a cluster +// whose draw nobody knows is filled as if it had no caps at all. +func TestNoCapIsEverExceeded(t *testing.T) { + rng := rand.New(rand.NewPCG(3, 4)) + for iter := range 200 { + st := inventory.NewStore() + known := rng.IntN(4) != 0 + cards := 0 + for i := range 2 + rng.IntN(6) { + n := at(node(fmt.Sprintf("n%d", i), "h100-80g", 1+rng.IntN(8), "ib"), + fmt.Sprintf("r%d", rng.IntN(3)), fmt.Sprintf("p%d", rng.IntN(2))) + if known { + n = withPowerLimit(n, float64(300+rng.IntN(500)), 0) + } + cards += len(n.GPUs) + st.Upsert(n) + } + pol := facility.Policy{Margin: float64(rng.IntN(20)) / 100, Budgets: map[facility.Domain]facility.Budget{}} + for _, d := range []facility.Domain{"rack:r0", "rack:r1", "rack:r2", "pdu:p0", "pdu:p1"} { + if rng.IntN(2) == 0 { + pol.Budgets[d] = facility.Budget{CapWatts: float64(1000 + rng.IntN(8000))} + } + } + st.SetFacility(pol) + + for k := range 60 { + p := Profile{Name: "gen", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1 + rng.IntN(2)} + res := New(st).Place([]Profile{p}, nil) + if res.Placement == nil { + continue + } + for _, a := range res.Placement.Nodes { + st.Allocate(a.NodeID, inventory.Allocation{ShardID: fmt.Sprintf("s%d-%d", k, a.Rank), GPUIndices: a.GPUIndices}) + } + } + for d, u := range st.PowerUse(time.Now()) { + if u.Limit > 0 && u.Used > u.Limit+1e-6 { + t.Fatalf("iteration %d: %s uses %.0f W of a %.0f W limit", iter, d, u.Used, u.Limit) + } + } + if !known { + free := 0 + for _, n := range st.Nodes() { + free += len(st.FreeGPUs(n.ID)) + } + if free != 0 { + t.Fatalf("iteration %d: %d of %d cards left free with no draw known", iter, free, cards) + } + } + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 6a57db1..945b8e6 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -16,8 +16,10 @@ import ( "slices" "sort" "strings" + "time" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" ) @@ -45,6 +47,13 @@ type Profile struct { // Source is the model's weights, as the engine would be given it. Used to // prefer nodes that already hold them; empty asks for no preference. Source string + // PowerWatts is what a replica of this profile draws, split evenly over + // its nodes. Zero derives it from the cards' power limits and the cores. + PowerWatts float64 + // KeepWithin keeps a multi-node replica's nodes inside one domain of + // this level, rack or row, where the fabric or the cabling needs them + // close. Empty is no constraint. + KeepWithin facility.Level } // CPUOnly reports whether this profile needs no GPUs. The catalog only lets a @@ -81,6 +90,9 @@ type NodeAssignment struct { GPUIndices []int // HCAs are the adapters of the chosen fabric only. HCAs []string + // PowerW is the draw the profile declared for this shard, to be recorded + // with its allocation; zero when the profile declared none. + PowerW float64 } // Placement is a decision: which profile, which fabric, which nodes. @@ -139,6 +151,9 @@ func (r Result) Explain() string { // Scheduler places replicas onto an inventory. type Scheduler struct { inv *inventory.Store + // now is the clock power readings are judged fresh by; nil is the wall + // clock. + now func() time.Time } // New builds a scheduler over an inventory. @@ -148,6 +163,11 @@ func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} } // // avoid lists nodes already hosting this model, so replicas of one model spread // across failure domains instead of stacking on one machine. +// +// The rejections of the profile that was placed are kept with the placement, +// not only those of the profiles before it: a replica placed in the second +// rack because the first was held or at its power cap is placed somewhere +// nobody expected, and explain-placement is where the operator asks why. func (s *Scheduler) Place(profiles []Profile, avoid []string) Result { var res Result @@ -168,6 +188,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result { if p.Nodes == 1 { if pl, rej := s.placeSingleNode(p, avoid); pl != nil { res.Placement = pl + res.Rejections = append(res.Rejections, rej...) return res } else { res.Rejections = append(res.Rejections, rej...) @@ -176,6 +197,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result { } if pl, rej := s.placeMultiNode(p, avoid); pl != nil { res.Placement = pl + res.Rejections = append(res.Rejections, rej...) return res } else { res.Rejections = append(res.Rejections, rej...) @@ -212,17 +234,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re len(s.inv.Allocations(candidates[j].node.ID)) }) preferNodesHoldingWeights(p.Source, candidates) + s.spreadAcrossDomains(candidates, avoid) return &Placement{ Profile: p, Pool: "none/cpu", - Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}}, + Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0, PowerW: p.declaredShardWatts()}}, }, rejections } - // Last, so it takes precedence: moving tens of gigabytes costs more than an - // imperfectly packed node, and with isolated fabrics it may cost a - // cross-fabric copy over the management network. + // After packing, so it takes precedence: moving tens of gigabytes costs + // more than an imperfectly packed node, and with isolated fabrics it may + // cost a cross-fabric copy over the management network. preferNodesHoldingWeights(p.Source, candidates) + // And racks over weights: a second replica in the rack of the first is + // one breaker away from none, which no copy saved is worth. + s.spreadAcrossDomains(candidates, avoid) best := candidates[0] gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil) @@ -237,7 +263,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re return &Placement{ Profile: p, Pool: "none/" + p.classLabel(), - Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}}, + Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus, PowerW: p.declaredShardWatts()}}, }, rejections } @@ -273,8 +299,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej if group.key != "" && group.split { pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel() } - if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil { - return pl, rejections + for _, within := range withinGroups(p, pool, group.nodes, &rejections) { + if pl := s.placeOnFabric(p, fabric, pool, within, &rejections); pl != nil { + return pl, rejections + } } } } @@ -325,8 +353,10 @@ func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejec }) for _, d := range keys { pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel() - if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil { - return pl + for _, within := range withinGroups(p, pool, by[d], rejections) { + if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, within, rejections); pl != nil { + return pl + } } } return nil @@ -416,8 +446,22 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c // together, a degraded cable last. candidates = byLeaf(candidates, hints(s.inv)) + // Each node passed the power budgets on its own; together, two shards in + // one rack may not. Nodes whose shard would take a domain past its cap + // with the shards already chosen are passed over for the next. + budget := s.newBudget() assignments := make([]NodeAssignment, 0, p.Nodes) - for rank, c := range candidates[:p.Nodes] { + for _, c := range candidates { + if len(assignments) == p.Nodes { + break + } + if why := budget.check(c.node, s.shardWatts(p, c)); why != "" { + *rejections = append(*rejections, Rejection{ + Profile: p.Name, Pool: pool, Node: c.node.ID, + Reason: why + ", with this replica's other shards", + }) + continue + } gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric)) if gpus == nil { *rejections = append(*rejections, Rejection{ @@ -426,14 +470,23 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c }) return nil } + budget.add(c.node, s.shardWatts(p, c)) assignments = append(assignments, NodeAssignment{ NodeID: c.node.ID, - Rank: rank, + Rank: len(assignments), GPUIndices: gpus, // Only this fabric's adapters, never the node's others. - HCAs: c.node.HCAsFor(fabric), + HCAs: c.node.HCAsFor(fabric), + PowerW: p.declaredShardWatts(), }) } + if len(assignments) < p.Nodes { + *rejections = append(*rejections, Rejection{ + Profile: p.Name, Pool: pool, + Reason: fmt.Sprintf("needs %d nodes, only %d fit within the power budgets together", p.Nodes, len(assignments)), + }) + return nil + } return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments} } @@ -508,6 +561,8 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje } pool += "/" + p.classLabel() + pol := s.inv.Facility() + budget := s.newBudget() var out []candidate for _, n := range s.inv.Nodes() { reject := func(reason string) { @@ -518,6 +573,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje reject("node is " + n.State.String()) continue } + // A domain held for maintenance takes nothing new, as a cordon of + // every node in it would, and says which hold and why. + if h, held := pol.HeldIn(n.Domains()); held { + reject(heldReason(h)) + continue + } if slices.Contains(avoid, n.ID) { reject("already hosts this model; spreading across failure domains") continue @@ -608,7 +669,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje if p.CPUOnly() { // Every ready node can run it; there is no card to be free. - out = append(out, candidate{node: n}) + c := candidate{node: n} + if why := budget.check(n, s.shardWatts(p, c)); why != "" { + reject(why) + continue + } + out = append(out, c) continue } @@ -624,7 +690,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode)) continue } - out = append(out, candidate{node: n, free: free}) + c := candidate{node: n, free: free} + if why := budget.check(n, s.shardWatts(p, c)); why != "" { + reject(why) + continue + } + out = append(out, c) } return out } diff --git a/internal/control/reconciler/adopt.go b/internal/control/reconciler/adopt.go index b5830e3..b74722c 100644 --- a/internal/control/reconciler/adopt.go +++ b/internal/control/reconciler/adopt.go @@ -145,6 +145,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) { ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices, MemoryGB: r.profileMemory(rep.Model, rep.Profile), CPUCores: r.adoptedCores(rep.Model, rep.Profile), + PowerW: r.profileWatts(rep.Model, rep.Profile), }) // The endpoint comes from the leader, which may be in any of the // groups — the rank 0 node is not necessarily the first to register. diff --git a/internal/control/reconciler/facility_test.go b/internal/control/reconciler/facility_test.go new file mode 100644 index 0000000..4329b5c --- /dev/null +++ b/internal/control/reconciler/facility_test.go @@ -0,0 +1,88 @@ +package reconciler + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +func inRack(n inventory.Node, rack string) inventory.Node { + n.Labels = map[string]string{"llmgw.io/rack": rack} + return n +} + +func runPasses(h *harness, n int) { + for range n { + h.r.Reconcile(context.Background()) + h.markReady() + h.advance(time.Second) + } +} + +func racksOf(h *harness) map[string]int { + out := map[string]int{} + for _, rep := range h.r.Replicas() { + for _, s := range rep.Shards { + n, _ := h.inv.Get(s.NodeID) + out[n.Location().Rack]++ + } + } + return out +} + +func TestAModelsReplicasLandInDifferentRacks(t *testing.T) { + // Two nodes in r1 and one in r2, each with room for one replica of + // four cards twice over: without racks the second replica's node is + // a coin toss between r1's other node and r2's. + m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4}) + h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"), inRack(gpuNode("2", 8), "r1"), inRack(gpuNode("3", 8), "r2")) + runPasses(h, 6) + + if got := racksOf(h); got["r1"] != 1 || got["r2"] != 1 { + t.Errorf("replicas by rack %v, want one in each", got) + } +} + +func TestNothingIsPlacedInAHeldRackAndExplainSaysSo(t *testing.T) { + m := testModel(1, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4}) + h := newHarness(t, m, inRack(gpuNode("1", 8), "r1")) + h.inv.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{ + "rack:r1": {Domain: "rack:r1", Reason: "cooling fault"}, + }}) + runPasses(h, 3) + + if n := len(h.r.Replicas()); n != 0 { + t.Fatalf("%d replicas placed in a held rack", n) + } + res, _ := h.r.Explain("llama") + if !strings.Contains(res.Explain(), "rack r1 is held: cooling fault") { + t.Errorf("explain-placement does not say the rack is held:\n%s", res.Explain()) + } + + h.inv.SetFacility(facility.Policy{}) + runPasses(h, 3) + if n := len(h.r.Replicas()); n != 1 { + t.Errorf("released, %d replicas; want 1", n) + } +} + +func TestADeclaredDrawIsRecordedSoTheNextReplicaSeesTheRackFull(t *testing.T) { + // 6 kW cap; each replica declares 4 kW. The first fits, the second would + // take the rack to 8 and is refused, though the node has the cards. + m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4, PowerWatts: 4000}) + h := newHarness(t, m, inRack(gpuNode("1", 8), "r1")) + h.inv.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}}) + runPasses(h, 6) + + if n := len(h.r.Replicas()); n != 1 { + t.Fatalf("%d replicas under a cap with room for one", n) + } + if u := h.inv.PowerUse(time.Now())["rack:r1"]; u.Reserved != 4000 { + t.Errorf("rack r1 has %v W reserved, want the declared 4000", u.Reserved) + } +} diff --git a/internal/control/reconciler/headroom.go b/internal/control/reconciler/headroom.go index 7290ecb..5554166 100644 --- a/internal/control/reconciler/headroom.go +++ b/internal/control/reconciler/headroom.go @@ -129,7 +129,7 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) { var nodes []string for _, a := range res.Placement.Nodes { inv.Allocate(a.NodeID, inventory.Allocation{ - ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, + ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW, }) nodes = append(nodes, a.NodeID) } diff --git a/internal/control/reconciler/reconciler.go b/internal/control/reconciler/reconciler.go index c3a0dff..2308a0a 100644 --- a/internal/control/reconciler/reconciler.go +++ b/internal/control/reconciler/reconciler.go @@ -1034,7 +1034,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac }) r.inv.Allocate(a.NodeID, inventory.Allocation{ ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices, - MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), + MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), PowerW: a.PowerW, }) } rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort) @@ -1361,6 +1361,17 @@ func (r *Reconciler) profileMemory(model, profile string) int { return p.MinMemoryGB } +// profileWatts is the draw a profile declared for each of its shards, so an +// adopted replica counts against its racks' power budgets as a placed one +// does; zero when it declared none. +func (r *Reconciler) profileWatts(model, profile string) float64 { + p, _ := r.profileOf(model, profile) + if p.PowerWatts <= 0 { + return 0 + } + return p.PowerWatts / float64(max(p.Nodes, 1)) +} + // profileOf is a replica's profile as the catalog holds it now. func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) { m, ok := r.cat.Load().Resolve(model) diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index abd4942..32be293 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -99,6 +99,19 @@ var Reference = []Block{ {Key: "model", Type: "string", Doc: "Model is what the provider calls it; requests are rewritten to it.", Nested: ""}, {Key: "api_key_file", Type: "string", Doc: "APIKeyFile holds the provider's key, on the control plane. A file\nrather than the key itself, so the key is in neither the catalog, the\nstore nor the audit log; empty for a provider that asks for none.", Nested: ""}, }}, + {Package: "config", Name: "Facility", Title: "Facility", Doc: "", Fields: []Field{ + {Key: "budgets", Type: "[]FacilityBudget", Doc: "Budgets are the domains with a power cap.", Nested: ""}, + {Key: "margin", Type: "float64", Doc: "Margin is the share of every cap kept free, 0.1 for ten percent. Zero\nkeeps none, for caps already given derated.", Nested: ""}, + {Key: "gpu_watts", Type: "float64", Doc: "GPUWatts is a card's draw when it does not report its power limit:\nan agent without nvidia-smi's power fields, or a node that has not\nsent a heartbeat yet. Zero counts such a card as drawing nothing.", Nested: ""}, + {Key: "cpu_core_watts", Type: "float64", Doc: "CPUCoreWatts is the draw of a core reserved by a replica with no GPU\n(cpu_cores). Zero counts such replicas as drawing nothing.", Nested: ""}, + {Key: "node_watts", Type: "float64", Doc: "NodeWatts is what a node draws besides its cards and reserved cores —\nidle CPUs, memory, fans, disks — counted for every node in a capped\ndomain, busy or not.", Nested: ""}, + {Key: "measured_fresh", Type: "time.Duration", Doc: "MeasuredFresh is how old a node's power reading may be and still be\ncounted against its domains. Zero takes two minutes.", Nested: ""}, + }}, + {Package: "config", Name: "FacilityBudget", Title: "FacilityBudget", Doc: "", Fields: []Field{ + {Key: "domain", Type: "string", Doc: "Domain is level:name, as the nodes' labels give it: rack:r12-07,\npdu:r12-07-a, row:12.", Nested: ""}, + {Key: "cap_watts", Type: "float64", Doc: "CapWatts is what the domain may draw. For a dual-fed rack it is what\none feed carries alone, derated as the facility derates it (usually\n80%), since that is what the rack has to survive.", Nested: ""}, + {Key: "other_watts", Type: "float64", Doc: "OtherWatts is drawn in the domain by what llmgw does not place on:\nswitches, storage, machines outside the cluster.", Nested: ""}, + }}, {Package: "catalog", Name: "GuardModel", Title: "GuardModel", Doc: "", Fields: []Field{ {Key: "format", Type: "string", Doc: "Format is llamaguard3 (Meta's Llama Guard 3: \"safe\", or \"unsafe\" and\nthe categories), shieldgemma (Google's ShieldGemma: Yes or No), or\njson ({\"safe\": false, \"categories\": [\"...\"]}).", Nested: ""}, {Key: "categories", Type: "[]string", Doc: "Categories the model is asked about, for shieldgemma — each a policy\nsentence — and the names reported for llamaguard3's S1, S2, ….", Nested: ""}, @@ -240,6 +253,8 @@ var Reference = []Block{ {Key: "cpu_cores", Type: "int", Doc: "CPUCores is the cores a replica of a profile with no GPU is given:\nreserved out of its node's CPUs by placement, as memory is, and what\nits engine computes on (llama.cpp's --threads).\n\nReserved because an engine takes every core it is told about, and two\non one node that each think they have all of them spin against each\nother: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second\nto under one each. Zero reserves nothing and leaves the thread count to\nthe node, which is every profile from before this.", Nested: ""}, {Key: "min_disk_gb", Type: "int", Doc: "MinDiskGB is the free space a node needs before it can hold these\nweights.\n\nOnly checked when the node does not already have them: a node holding\nthe file needs no room for it, and requiring the space twice would keep\na model off the one node best placed to run it.\n\nDeclared rather than derived, because the size of a model that has never\nbeen fetched anywhere is not knowable from here — and a wrong guess\nfails late, halfway through a download, which is the failure this exists\nto prevent. Zero means unconstrained, and a node that reports no storage\nis not excluded: unknown is not full.", Nested: ""}, {Key: "node_selector", Type: "map[string]string", Doc: "NodeSelector restricts this profile to nodes carrying every one of these\nlabels, with these values.\n\nA CPU profile otherwise fits everywhere, because there is no card to be\nfree — which is right for a 0.5B model and wrong for a 14B one. Nothing\nin the inventory says how much memory a node has, so the operator says\nwhich nodes are meant for a model instead of the scheduler inferring it.\nAgents have reported labels since the beginning; this is what reads them.", Nested: ""}, + {Key: "power_watts", Type: "float64", Doc: "PowerWatts is what one replica of this profile draws, in watts, split\nevenly over its nodes, counted against the power cap of every rack,\nPDU or other domain its nodes are in (facility.budgets in server.yaml).\nZero derives it: each card at the power limit it reports, each\nreserved core at facility.cpu_core_watts. Only matters where a domain\nhas a cap.", Nested: ""}, + {Key: "keep_within", Type: "string", Doc: "KeepWithin keeps a multi-node replica's nodes inside one rack, row,\nroom or site, read from their llmgw.io/rack (and so on) labels: for a\nfabric whose leaves do not reach across racks, or cables that do not.\nNodes without the label are passed over for such a profile. Empty is\nno constraint.", Nested: ""}, {Key: "weight", Type: "float64", Doc: "Weight is the router's capacity weight. It should come from ctl bench\nrather than a guess (plan §6.2.1).", Nested: ""}, {Key: "priority", Type: "int", Doc: "Priority orders profiles during placement; highest that fits wins.", Nested: ""}, {Key: "startup_timeout", Type: "time.Duration", Doc: "StartupTimeout bounds model loading. A 70B model across two nodes takes\ntens of minutes, so this is per profile rather than global.", Nested: ""}, @@ -337,6 +352,7 @@ var Reference = []Block{ {Key: "router", Type: "Router", Doc: "", Nested: "config.Router"}, {Key: "control", Type: "Control", Doc: "", Nested: "config.Control"}, {Key: "models", Type: "[]catalog.Model", Doc: "", Nested: ""}, + {Key: "facility", Type: "Facility", Doc: "Facility is the power caps of racks, PDUs and other domains, and how\na replica's draw is estimated against them. Where a node is comes from\nits llmgw.io/site, room, row, rack and pdu labels; holding a domain for\nmaintenance is `llmgw ctl facility hold`, not configuration.", Nested: "config.Facility"}, {Key: "endpoints", Type: "[]Endpoint", Doc: "Endpoints is the phase-1 static replica list. It is only consulted when\nno model declares a profile, which is what makes a router-only smoke test\nagainst sim-engines possible without a control plane.", Nested: ""}, }}, {Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{ diff --git a/internal/store/copy.go b/internal/store/copy.go index f56e9d5..81ce820 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -51,6 +51,9 @@ var copyOrder = []string{ // of them at the moment of the move. "cluster_secrets", "node_engines", + // Or a held rack would take work again after the move. + "facility_holds", + "node_labels", "control_planes", // Or a removed node's certificate would work again in the new store. "revoked_nodes", diff --git a/internal/store/facility.go b/internal/store/facility.go new file mode 100644 index 0000000..50a21d1 --- /dev/null +++ b/internal/store/facility.go @@ -0,0 +1,93 @@ +package store + +import ( + "context" + "time" +) + +// FacilityHold is a domain taken out of placement: rack:r12-07. +type FacilityHold struct { + Domain string + Reason string + Drain bool + At time.Time + By string +} + +// FacilityHolds are every domain held, oldest first. +func (s *DB) FacilityHolds(ctx context.Context) ([]FacilityHold, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT domain, reason, drain, held_at, held_by FROM facility_holds ORDER BY held_at, domain`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []FacilityHold + for rows.Next() { + var h FacilityHold + var drain, at int64 + if err := rows.Scan(&h.Domain, &h.Reason, &drain, &at, &h.By); err != nil { + return nil, err + } + h.Drain, h.At = drain != 0, time.UnixMilli(at) + out = append(out, h) + } + return out, rows.Err() +} + +// SetFacilityHold holds a domain, or changes the reason it is held for. +func (s *DB) SetFacilityHold(ctx context.Context, h FacilityHold) error { + drain := 0 + if h.Drain { + drain = 1 + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO facility_holds (domain, reason, drain, held_at, held_by) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (domain) DO UPDATE SET reason = excluded.reason, drain = excluded.drain, + held_at = excluded.held_at, held_by = excluded.held_by`, + h.Domain, h.Reason, drain, h.At.UnixMilli(), h.By) + return err +} + +// DeleteFacilityHold releases a domain; releasing one not held is no error. +func (s *DB) DeleteFacilityHold(ctx context.Context, domain string) error { + _, err := s.db.ExecContext(ctx, `DELETE FROM facility_holds WHERE domain = ?`, domain) + return err +} + +// NodeLabels are the labels set through the control plane: node id, then +// key, then value. +func (s *DB) NodeLabels(ctx context.Context) (map[string]map[string]string, error) { + rows, err := s.db.QueryContext(ctx, `SELECT node_id, key, value FROM node_labels`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]map[string]string{} + for rows.Next() { + var node, key, value string + if err := rows.Scan(&node, &key, &value); err != nil { + return nil, err + } + if out[node] == nil { + out[node] = map[string]string{} + } + out[node][key] = value + } + return out, rows.Err() +} + +// SetNodeLabel sets one label on a node; an empty value removes it, which +// leaves the agent's own value for the key, if it has one, in force again. +func (s *DB) SetNodeLabel(ctx context.Context, node, key, value, by string) error { + if value == "" { + _, err := s.db.ExecContext(ctx, `DELETE FROM node_labels WHERE node_id = ? AND key = ?`, node, key) + return err + } + _, err := s.db.ExecContext(ctx, + `INSERT INTO node_labels (node_id, key, value, updated_at, updated_by) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (node_id, key) DO UPDATE SET value = excluded.value, + updated_at = excluded.updated_at, updated_by = excluded.updated_by`, + node, key, value, time.Now().UnixMilli(), by) + return err +} diff --git a/internal/store/facility_test.go b/internal/store/facility_test.go new file mode 100644 index 0000000..9113f13 --- /dev/null +++ b/internal/store/facility_test.go @@ -0,0 +1,52 @@ +package store + +import ( + "testing" + "time" +) + +func TestAHeldDomainIsKeptUntilReleasedAndHoldingAgainChangesTheReason(t *testing.T) { + db := capacityStore(t) + ctx := t.Context() + at := time.UnixMilli(1_700_000_000_000) + if err := db.SetFacilityHold(ctx, FacilityHold{Domain: "rack:r1", Reason: "PDU swap", At: at, By: "ilker"}); err != nil { + t.Fatal(err) + } + if err := db.SetFacilityHold(ctx, FacilityHold{Domain: "rack:r1", Reason: "cooling", Drain: true, At: at, By: "ilker"}); err != nil { + t.Fatal(err) + } + got, err := db.FacilityHolds(ctx) + if err != nil || len(got) != 1 { + t.Fatalf("got %v, %v; one hold per domain", got, err) + } + if h := got[0]; h.Reason != "cooling" || !h.Drain || !h.At.Equal(at) || h.By != "ilker" { + t.Errorf("the second hold did not replace the first: %+v", h) + } + if err := db.DeleteFacilityHold(ctx, "rack:r1"); err != nil { + t.Fatal(err) + } + if got, _ := db.FacilityHolds(ctx); len(got) != 0 { + t.Errorf("released, still %v", got) + } +} + +func TestANodeLabelSetThroughTheControlPlaneIsKeptAndCanBeRemoved(t *testing.T) { + db := capacityStore(t) + ctx := t.Context() + if err := db.SetNodeLabel(ctx, "gpu07", "llmgw.io/rack", "r12-07", "ilker"); err != nil { + t.Fatal(err) + } + if err := db.SetNodeLabel(ctx, "gpu07", "llmgw.io/rack", "r12-08", "ilker"); err != nil { + t.Fatal(err) + } + got, err := db.NodeLabels(ctx) + if err != nil || got["gpu07"]["llmgw.io/rack"] != "r12-08" { + t.Fatalf("got %v, %v", got, err) + } + if err := db.SetNodeLabel(ctx, "gpu07", "llmgw.io/rack", "", "ilker"); err != nil { + t.Fatal(err) + } + if got, _ := db.NodeLabels(ctx); len(got["gpu07"]) != 0 { + t.Errorf("removed, still %v", got) + } +} diff --git a/internal/store/migrations/postgres/0031_facility.sql b/internal/store/migrations/postgres/0031_facility.sql new file mode 100644 index 0000000..820dd0a --- /dev/null +++ b/internal/store/migrations/postgres/0031_facility.sql @@ -0,0 +1,24 @@ +-- Ported from the SQLite migration of the same number. +-- Facility-aware placement. Domains held for maintenance: a rack, row, PDU +-- or other domain taken out of placement by `llmgw ctl facility hold`, +-- kept until released so a restart does not quietly put work back on a rack +-- somebody has the doors open on. +CREATE TABLE facility_holds ( + domain TEXT PRIMARY KEY, + reason TEXT NOT NULL, + drain BIGINT NOT NULL, + held_at BIGINT NOT NULL, + held_by TEXT NOT NULL +); + +-- Node labels set through the control plane (`llmgw ctl label`), laid over +-- what each node's agent.yaml says: where a node is in the building is +-- usually known to whoever runs the facility, not to the machine. +CREATE TABLE node_labels ( + node_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + updated_at BIGINT NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (node_id, key) +); diff --git a/internal/store/migrations/sqlite/0031_facility.sql b/internal/store/migrations/sqlite/0031_facility.sql new file mode 100644 index 0000000..3886d01 --- /dev/null +++ b/internal/store/migrations/sqlite/0031_facility.sql @@ -0,0 +1,23 @@ +-- Facility-aware placement. Domains held for maintenance: a rack, row, PDU +-- or other domain taken out of placement by `llmgw ctl facility hold`, +-- kept until released so a restart does not quietly put work back on a rack +-- somebody has the doors open on. +CREATE TABLE facility_holds ( + domain TEXT PRIMARY KEY, + reason TEXT NOT NULL, + drain INTEGER NOT NULL, + held_at INTEGER NOT NULL, + held_by TEXT NOT NULL +); + +-- Node labels set through the control plane (`llmgw ctl label`), laid over +-- what each node's agent.yaml says: where a node is in the building is +-- usually known to whoever runs the facility, not to the machine. +CREATE TABLE node_labels ( + node_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL, + updated_by TEXT NOT NULL, + PRIMARY KEY (node_id, key) +); diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go index c78b7e2..94e5980 100644 --- a/internal/store/postgres_test.go +++ b/internal/store/postgres_test.go @@ -44,7 +44,7 @@ func openPG(t *testing.T) *DB { `TRUNCATE thread_messages, threads, memory_notes, usage, usage_5m, usage_1h, usage_1d, api_keys, memberships, key_claims, limits, models, sessions, users, projects, teams, organizations, leadership, control_planes, files, batches, batch_results, request_bodies, - eval_runs, eval_cases RESTART IDENTITY CASCADE`) + eval_runs, eval_cases, facility_holds, node_labels RESTART IDENTITY CASCADE`) if err != nil { t.Fatalf("clearing the test database: %v", err) } diff --git a/notes/c24-facility.md b/notes/c24-facility.md new file mode 100644 index 0000000..531bd57 --- /dev/null +++ b/notes/c24-facility.md @@ -0,0 +1,40 @@ +## CHANGELOG + +### Facility-aware placement: racks, PDUs, holds and power caps + +- **Where a node is**, from five labels: `llmgw.io/site`, `room`, `row`, + `rack` and `pdu` (a list, for a dual-fed rack), in `agent.yaml` or set from + the control plane with **`llmgw ctl label NODE key=value… [key-…]`**, which + is kept in the store (migration 0031, `node_labels`), wins over the agent's + labels, works before the node joins, and refuses a misspelt `llmgw.io/` key. + A node with no location places as before. +- **Replicas spread across racks and PDUs.** A model's next replica goes to a + node sharing no rack or PDU with its others when there is one: an ordering + above weights already on disk and bin packing, never a filter. +- **`keep_within: rack`** (or row, room, site) on a profile keeps a + multi-node replica's nodes in one such domain. +- **Holds.** **`llmgw ctl facility hold rack:r12-07 --reason … [--drain]`** + places nothing new in a rack, row, room, site or PDU; `--drain` also drains + its nodes. **`llmgw ctl facility release`** undoes both. Kept across + restarts (migration 0031, `facility_holds`). +- **Power caps.** `facility.budgets` in `server.yaml` gives a domain's + `cap_watts` and `other_watts`, with a `margin`; a replica's draw is the + profile's `power_watts`, or its cards at the power limit they report + (`gpu_watts` otherwise) and its cores at `cpu_core_watts`, with `node_watts` + per node. Use is the larger of reserved and measured, node by node; a shard + that would take a domain past its cap less the margin is refused, and a + multi-node replica's shards are counted together. Unknown draw never blocks. +- **Seen everywhere it matters.** `ctl explain-placement` names the held + domain or the kilowatts, and keeps the rejections of a replica that was + placed, so one in an unexpected rack says why. `ctl nodes` has LOCATION, + *ready, held* and `--domain`. **`llmgw ctl facility`** lists every domain + with nodes, use, limit, reserved, measured and holds. The console's Nodes + view has WHERE; a node's menu has **Location…** and **Hold rack…/Release**. + Admin API: `GET /api/v1/facility`, `POST /api/v1/facility/holds`, + `DELETE /api/v1/facility/holds/{domain}`, `POST /api/v1/nodes/{id}/labels`. +- Docs: `docs/facility-aware-placement.md` says what is built and where it + departs from the design; `docs/install.md` and `docs/console.md`. + +## BACKLOG + +- [x] 24 (facility) — built: location labels (`ctl label`, agent.yaml), spreading across racks/PDUs, `keep_within`, held domains with drain (`ctl facility hold/release`, kept in the store), power caps from `facility.budgets` against declared/derived/measured draw, shown in explain-placement, `ctl nodes`, `ctl facility`, the console and the API. Left: temperature conditions with hysteresis and their enforcement (steps 3–4), a Prometheus/DCIM source, power headroom as a rack-levelling score, spreading multi-node replicas across racks, `llmgw_domain_*` metrics and alerts (near cap, over cap, stale data), shedding and tokens per watt (step 5), and a drained node's state surviving a restart.