Release v0.11.1

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.gitea/workflows/release.yml

@@ -93,6 +93,24 @@ jobs:
           ./dist/llmgw-${version}-linux-amd64 --version
           cat dist/checksums.txt
 
+      # Signed when the repository has a signing key, so a machine that trusts
+      # it installs nothing the server alone could have put there. Without the
+      # secret nothing is signed and releases are checked on their checksums,
+      # as before; a build that trusts a key refuses those, so the secret has
+      # to be in place before trusted.pub is filled in.
+      - name: Sign the checksums
+        env:
+          RELEASE_SIGNING_KEY: ${{ secrets.RELEASE_SIGNING_KEY }}
+        run: |
+          set -euo pipefail
+          version="${GITHUB_REF_NAME:-$(git describe --tags --always)}"
+          if [ -z "${RELEASE_SIGNING_KEY:-}" ]; then
+            echo "no RELEASE_SIGNING_KEY secret; releases are not signed"
+            exit 0
+          fi
+          ./dist/llmgw-${version}-linux-amd64 release sign dist/checksums.txt
+          echo "signed: dist/checksums.txt.sig"
+
       - name: Attach to the release
         env:
           TOKEN: ${{ secrets.GITEA_TOKEN }}

api/proto/llmgw/v1/agent.proto

@@ -25,6 +25,8 @@ message AgentMessage {
     ShardStatus shard = 3;
     LogsReply logs = 4;
     FabricScanReply fabric_scan = 5;
+    FabricTestReply fabric_test = 6;
+    UpgradeReply upgrade = 7;
   }
 }
 
@@ -39,9 +41,20 @@ message ControlMessage {
     Provision provision = 6;
     LogsRequest logs = 7;
     FabricScanRequest fabric_scan = 8;
+    FabricTestRequest fabric_test = 9;
+    UpgradeAgent upgrade = 10;
+    ControlPlanes control_planes = 11;
   }
 }
 
+// ControlPlanes is every control plane an agent may dial, sent by the leader
+// when an agent registers and whenever the list changes. The agent keeps it
+// beside its configuration, so a second control plane added after the node
+// was onboarded is one it fails over to without anybody editing the node.
+message ControlPlanes {
+  repeated string servers = 1;
+}
+
 // Register is the first message on every stream. It carries the node's full
 // hardware inventory, so the control plane needs no separate discovery call and
 // a reconnecting agent automatically refreshes what the scheduler knows.
@@ -116,6 +129,10 @@ message Register {
   // meant "an agent too old to say" and constrained nothing — so a freshly
   // built node with no engine installed yet was a candidate for every model.
   bool engines_probed = 14;
+  // cpu_count is the CPUs the agent may run on. Placement reserves cores
+  // for CPU replicas out of it, the way it reserves memory out of
+  // memory_total_mb; zero, from an older agent, reserves nothing.
+  int32 cpu_count = 18;
   // imex is the service that lets a multi-node NVLink domain share memory.
   IMEX imex = 15;
 
@@ -125,6 +142,12 @@ message Register {
   // vLLM, it answered "unsupported", and every node of a cluster half way
   // through an upgrade showed as failing.
   repeated string installable_engines = 16;
+
+  // features are the requests this agent answers beyond the ones every agent
+  // does: "fabric-test". An agent that does not know a request ignores it, so
+  // without this the control plane would wait minutes for an answer that is
+  // never coming.
+  repeated string features = 17;
 }
 
 // Host describes the operating system and processor of a node.
@@ -271,6 +294,54 @@ message NodeLoad {
   int64 memory_used_bytes = 6;
   int64 memory_available_bytes = 7;
   repeated ShardLoad shards = 8;
+  // What the machine's own sensors say; zero is not measured. cpu_temp_c is
+  // the hottest CPU sensor, cpu_power_w the CPU packages' draw from RAPL,
+  // inlet_temp_c the board's inlet or ambient sensor where it has one.
+  double cpu_temp_c = 9;
+  double cpu_power_w = 10;
+  double inlet_temp_c = 11;
+  repeated GPUSensor gpus = 12;
+}
+
+// GPUSensor is one card's heat, power, and what is holding it back.
+message GPUSensor {
+  int32 index = 1;
+  double temp_c = 2;
+  double memory_temp_c = 3;
+  double power_w = 4;
+  double power_limit_w = 5;
+  // throttle names what is holding the clocks down now: sw_power_cap,
+  // hw_slowdown, sw_thermal, hw_thermal, power_brake.
+  repeated string throttle = 6;
+  // How busy the card is, sampled with the rest, rather than read once at
+  // registration. -1 is not reported by the driver. memory_total_mb is set
+  // by every agent that sends these, so zero there is an older agent.
+  double utilization_percent = 7;
+  double memory_utilization_percent = 8;
+  double memory_used_mb = 9;
+  double memory_total_mb = 10;
+  double sm_clock_mhz = 11;
+  // ecc_uncorrected counts uncorrected ECC errors since the driver loaded.
+  double ecc_uncorrected = 12;
+  // What wears the card out, read every minute; has_wear says these are
+  // present. Counts of -1 are not reported for this card.
+  bool has_wear = 13;
+  int64 pcie_replays = 14;
+  int64 retired_pages_sbe = 15;
+  int64 retired_pages_dbe = 16;
+  bool retirement_pending = 17;
+  int64 remapped_rows_correctable = 18;
+  int64 remapped_rows_uncorrectable = 19;
+  bool remap_pending = 20;
+  bool remap_failure = 21;
+  int64 nvlink_crc_errors = 22;
+  int64 nvlink_replay_errors = 23;
+  int64 nvlink_recovery_errors = 24;
+  // xid_errors are the driver's error events since the agent started.
+  int64 xid_errors = 25;
+  int32 last_xid = 26;
+  string last_xid_text = 27;
+  int64 last_xid_unix = 28;
 }
 
 // ShardLoad is one engine process group's use.
@@ -410,6 +481,17 @@ message Heartbeat {
   // domain, and the node has to stop being offered work that spans it.
   repeated GPUFabricState gpu_fabric = 16;
   IMEX imex = 17;
+  // cpu_count as in Register, for a VM given more CPUs while it runs.
+  int32 cpu_count = 18;
+
+  // gpu_foreign_memory_mb, indexed by GPU, is what processes that are not
+  // this agent's shards hold on each card, read afresh every heartbeat. What
+  // keeps a card out of placement for another tenant is decided from this
+  // rather than from registration: a card that had something else on it when
+  // the agent started is not taken forever, and one that gains a tenant later
+  // is not handed out. Empty from an agent that does not measure it, or from
+  // one that could not this time.
+  repeated int32 gpu_foreign_memory_mb = 19;
 }
 
 // Provision is the software a node should have: which engines, at which
@@ -430,6 +512,9 @@ message Provision {
 message EngineWant {
   string engine = 1;  // "llamacpp"
   string version = 2; // "b10991"
+  // variant is the build to install, when one is chosen for the node rather
+  // than left to its hardware: "cu13", "cu129". Empty is the agent's choice.
+  string variant = 3;
 }
 
 // ProvisionStatus is where one engine's installation stands on a node.
@@ -516,6 +601,13 @@ message StartShard {
   // profile_fp identifies the version of the profile, so the agent can say
   // which one a running shard was started from. The agent never interprets it.
   string profile_fp = 15;
+  // api_key is what the engine demands of every request, derived by the
+  // control plane from the replica id. The router sends it; nothing else
+  // that can reach the node's ports has it.
+  string api_key = 16;
+  // cpu_threads is how many CPU threads a shard with no GPU computes on:
+  // the cores placement reserved for it. Zero leaves it to the agent.
+  int32 cpu_threads = 17;
 }
 
 message EngineProfile {
@@ -530,6 +622,9 @@ message EngineProfile {
   map<string, string> env = 9;
   string model_path = 10;
   string served_model_name = 11;
+  // adapters are LoRA adapters loaded beside the model, in order: the index
+  // is the id a request applies one by.
+  repeated LoraAdapter adapters = 12;
 }
 
 // RemoveWeights deletes cached model files a node no longer needs.
@@ -605,3 +700,67 @@ message FabricScanReply {
   string error = 3;
   int64 took_ms = 4;
 }
+
+// FabricTestRequest asks a node for one side of an RDMA bandwidth test
+// between two nodes: ib_write_bw, from perftest, over one adapter. The
+// server side is asked first and answers when the test is over; the client
+// dials it. What it proves is what a replica spanning nodes needs and nothing
+// on one node can show: that these two adapters reach each other through the
+// fabric, and how fast.
+message FabricTestRequest {
+  string request_id = 1;
+  // role is "server" or "client".
+  string role = 2;
+  string hca = 3;
+  int32 port = 4;
+  // peer is the server's management address, for the client. perftest
+  // exchanges its connection details over TCP before any RDMA.
+  string peer = 5;
+  // tcp_port is that exchange's port, the same on both sides.
+  int32 tcp_port = 6;
+  int32 seconds = 7;
+}
+
+// FabricTestReply is what one side measured.
+message FabricTestReply {
+  string request_id = 1;
+  // gbps is the average bandwidth, in gigabits a second.
+  double gbps = 2;
+  // output is the end of the tool's output, for when the number is not
+  // enough.
+  string output = 3;
+  string error = 4;
+  int64 took_ms = 5;
+}
+
+// UpgradeAgent tells an agent to replace its own binary with a release and
+// restart.
+//
+// The agent fetches the release for its own platform and checks it against
+// the release's checksums before replacing anything, keeps the old binary
+// beside the new, then exits so systemd starts the new one. Engines keep
+// running throughout, as for any agent restart, and are adopted on the way
+// back. Over the stream the agent already holds, so upgrading a fleet needs
+// no ssh into it.
+message UpgradeAgent {
+  string request_id = 1;
+  // version must be a release: v0.10.4 or v0.10.4-rc.2.
+  string version = 2;
+  // releases is the base URL release files are downloaded from.
+  string releases = 3;
+}
+
+// UpgradeReply says whether the new binary is in place. Sent before the
+// agent exits; an empty error means it will be back on the new version.
+message UpgradeReply {
+  string request_id = 1;
+  string version = 2;
+  string error = 3;
+}
+
+// LoraAdapter is one adapter: the name clients ask for it by, and its source,
+// which the agent fetches as it does the model's weights.
+message LoraAdapter {
+  string name = 1;
+  string source = 2;
+}

deploy/bootstrap/onboard-node.sh

@@ -122,9 +122,10 @@ TMP="$(mktemp -d)"
 trap 'rm -rf "$TMP"' EXIT
 "$BINARY" ctl cert issue-client "$NODE" --dir "$PKI" --out "$TMP" >/dev/null
 
-# 2 and 3. Identity, and trusting the root so future tooling on that machine
-#    does not treat our certificates as strangers.
-say "installing the certificate and trusting the root"
+# 2 and 3. Identity. The root is trusted by the agent's configuration only,
+#    never system-wide: it signs whatever the control plane asks, and every
+#    TLS client on the machine would take it for any site.
+say "installing the certificate"
 scp -q $SSHOPTS "$PKI/ca.pem" "$TMP/$NODE.pem" "$TMP/$NODE-key.pem" "$SSH_USER@$ADDR:/tmp/"
 $SSH "set -e
   sudo install -d -m 0755 /etc/llmgw
@@ -132,8 +133,10 @@ $SSH "set -e
   sudo install -m 0644 /tmp/$NODE.pem /etc/llmgw/agent.pem
   sudo install -m 0600 /tmp/$NODE-key.pem /etc/llmgw/agent-key.pem
   sudo chown root:root /etc/llmgw/*
-  sudo install -m 0644 /tmp/ca.pem /usr/local/share/ca-certificates/llmgw-ca.crt
-  sudo update-ca-certificates >/dev/null 2>&1 || true
+  if [ -f /usr/local/share/ca-certificates/llmgw-ca.crt ]; then
+    sudo rm -f /usr/local/share/ca-certificates/llmgw-ca.crt
+    sudo update-ca-certificates --fresh >/dev/null 2>&1 || true
+  fi
   rm -f /tmp/ca.pem /tmp/$NODE.pem /tmp/$NODE-key.pem"
 
 # 4. The binary, the configuration and the unit.
@@ -143,7 +146,7 @@ $SSH "set -e
 # that is not a release (a developer's, with commits past the tag) has no such
 # file, and is copied as before.
 TAG="$("$BINARY" --version 2>/dev/null | awk '{print $NF}')"
-if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
+if printf '%s' "$TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$'; then
   say "installing llmgw $TAG from the release"
   $SSH "set -e
     arch=\$(dpkg --print-architecture)

deploy/examples/server.yaml

@@ -92,6 +92,11 @@ agent_listen: ":7443"
 #   flush_interval: 200ms
 #   batch_size: 500
 #   buffer_size: 8192
+#
+#   # How long the record of each request is kept (what `ctl request` reads).
+#   # Charts, totals and budgets come from five-minute sums that are kept, so
+#   # this only bounds the table that grows by a row a request. 0 keeps all.
+#   usage_retention: 2160h   # 90 days
 
 router:
   scrape_interval: 2s

deploy/ha/README.md

@@ -85,3 +85,63 @@ auth:
 
 See [docs/install.md](../../docs/install.md) for moving an existing
 installation onto PostgreSQL without reissuing every key.
+
+## The store itself: PostgreSQL under Patroni
+
+Two control planes on one PostgreSQL have moved the single point of failure,
+not removed it. Patroni keeps PostgreSQL on two machines, one streaming to the
+other, and promotes the standby when the primary is lost. It keeps its leader
+lock in etcd, and etcd decides only with a majority — so three members, not
+two: two machines that lose sight of each other must not both decide they are
+the primary. The third member can be any machine that is up; it holds a vote,
+not data.
+
+On the reference cluster:
+
+| machine | etcd | PostgreSQL |
+|---|---|---|
+| llmgw-ctl (control plane) | member | — |
+| llmgw-pg01 | member | Patroni |
+| llmgw-ctl02 (second control plane) | member | Patroni |
+
+`llmgw server init --patroni user@host --patroni user@host` sets all of it up
+over ssh and moves the store in — see [install.md](../../docs/install.md).
+It looks before it acts at every step, so it is also how an existing cluster
+is checked: run against one that is up, it changes nothing. What it writes
+is in `etcd.default.example` (one member's `/etc/default/etcd`) and
+`patroni.yml.example` (one node's `/etc/patroni/config.yml`), for anyone
+doing it by hand; passwords are generated into `/root/llmgw-pg` on each
+database machine.
+
+**Give llmgw every member, and ask for the writable one:**
+
+```
+postgres://llmgw:…@10.0.0.149:5432,10.0.0.150:5432/llmgw?sslmode=disable&target_session_attrs=read-write
+```
+
+The driver connects to whichever member accepts writes, and reconnects to the
+new one after a failover. No floating address or HAProxy is needed for that.
+`llmgw server backup` reads the same address and dumps from a standby when
+there is one.
+
+Measured on the reference cluster, with the gateway serving throughout:
+
+- **Planned switchover** (`patronictl switchover`): the control plane logged
+  one failed lease renewal, reconnected to the new primary, and kept its
+  leadership; nothing else noticed.
+- **The primary's machine switched off**: the standby was promoted 24 seconds
+  later — the 30-second TTL in `patroni.yml` — and the control plane rode it
+  out without losing leadership or a replica. Started again, the old primary
+  rejoined as a standby by itself.
+
+**Watching it.** For a store of several hosts the control plane reads the
+cluster from each member's Patroni (port 8008) and publishes
+`llmgw_store_member{member,host,role,state}`, `llmgw_store_replica_lag_bytes`
+and `llmgw_store_patroni_up`; a host of the address that the cluster no longer
+lists is a member in state `missing`. `llmgw ctl status` prints it as one
+line, and `deploy/monitoring/llmgw-rules.yml` alerts on no leader, a member
+not streaming or missing, and a standby too far behind to be promoted.
+
+Not done here, and worth doing in production: a watchdog (`softdog`) so a
+primary that has lost the lock but not died cannot keep accepting writes, TLS
+on etcd and between PostgreSQL members, and more than one standby.

deploy/ha/etcd.default.example

@@ -0,0 +1,10 @@
+# llmgw: one member of the three that hold Patroni's leader lock (140, 149, 150).
+ETCD_NAME=llmgw-pg01
+ETCD_DATA_DIR=/var/lib/etcd/default
+ETCD_LISTEN_PEER_URLS=http://10.0.0.149:2380
+ETCD_LISTEN_CLIENT_URLS=http://10.0.0.149:2379,http://127.0.0.1:2379
+ETCD_INITIAL_ADVERTISE_PEER_URLS=http://10.0.0.149:2380
+ETCD_ADVERTISE_CLIENT_URLS=http://10.0.0.149:2379
+ETCD_INITIAL_CLUSTER=llmgw-ctl=http://10.0.0.140:2380,llmgw-pg01=http://10.0.0.149:2380,llmgw-ctl02=http://10.0.0.150:2380
+ETCD_INITIAL_CLUSTER_STATE=new
+ETCD_INITIAL_CLUSTER_TOKEN=llmgw-patroni

deploy/ha/patroni.yml.example

@@ -0,0 +1,68 @@
+# llmgw: the control planes' store, kept by Patroni on llmgw-pg01 and
+# llmgw-ctl02, with the leader lock in etcd on 140, 149 and 150.
+scope: llmgw
+namespace: /service/
+name: llmgw-pg01
+
+restapi:
+  listen: 10.0.0.149:8008
+  connect_address: 10.0.0.149:8008
+
+etcd3:
+  hosts: 10.0.0.140:2379,10.0.0.149:2379,10.0.0.150:2379
+
+bootstrap:
+  dcs:
+    ttl: 30
+    loop_wait: 10
+    retry_timeout: 10
+    maximum_lag_on_failover: 1048576
+    postgresql:
+      use_pg_rewind: true
+      use_slots: true
+      parameters:
+        wal_level: replica
+        hot_standby: "on"
+        max_wal_senders: 10
+        max_replication_slots: 10
+        wal_keep_size: 1GB
+        wal_log_hints: "on"
+        password_encryption: scram-sha-256
+  initdb:
+    - encoding: UTF8
+    - data-checksums
+
+postgresql:
+  listen: 127.0.0.1,10.0.0.149:5432
+  connect_address: 10.0.0.149:5432
+  data_dir: /var/lib/postgresql/16/llmgw
+  bin_dir: /usr/lib/postgresql/16/bin
+  pgpass: /var/lib/postgresql/.pgpass_patroni
+  authentication:
+    superuser:
+      username: postgres
+      password: "<from /root/llmgw-pg>"
+    replication:
+      username: replicator
+      password: "<from /root/llmgw-pg>"
+    rewind:
+      username: postgres
+      password: "<from /root/llmgw-pg>"
+  parameters:
+    unix_socket_directories: /var/run/postgresql
+  pg_hba:
+    - local all all peer
+    - host all all 127.0.0.1/32 scram-sha-256
+    - host replication replicator 10.0.0.149/32 scram-sha-256
+    - host replication replicator 10.0.0.150/32 scram-sha-256
+    - host all postgres 10.0.0.149/32 scram-sha-256
+    - host all postgres 10.0.0.150/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.140/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.149/32 scram-sha-256
+    - host llmgw llmgw 10.0.0.150/32 scram-sha-256
+
+tags:
+  nofailover: false
+  noloadbalance: false
+  clonefrom: false
+  nosync: false

deploy/monitoring/README.md

@@ -46,6 +46,19 @@ apt-get install prometheus prometheus-alertmanager
 install -m 0644 prometheus.yml            /etc/prometheus/prometheus.yml
 install -m 0644 llmgw-rules.yml           /etc/prometheus/rules/llmgw.yml
 install -m 0644 alertmanager.yml          /etc/prometheus/alertmanager.yml
+# Alertmanager answers on loopback only. Its API takes silences from anyone who
+# can reach it, with no login, and silencing BackupTooOld or ModelParked is how
+# something that should be noticed is not. Prometheus beside it is its only
+# client; one Alertmanager needs no cluster port.
+echo 'ARGS="--web.listen-address=127.0.0.1:9093 --cluster.listen-address="' \
+    > /etc/default/prometheus-alertmanager
+# Prometheus itself is read-only as installed: its admin and lifecycle APIs
+# stay off (no --web.enable-admin-api, no --web.enable-lifecycle). It still
+# answers any query from anyone who reaches :9090, which says which models and
+# nodes there are and how busy each is. Its readers are the control plane
+# (monitoring.prometheus) and Grafana beside it; to keep it to those, allow
+# the control plane and drop the rest, e.g. with nftables:
+#   nft add rule inet filter input tcp dport 9090 ip saddr != { 127.0.0.1, CONTROL_PLANE_IP } drop
 promtool check config /etc/prometheus/prometheus.yml
 systemctl restart prometheus prometheus-alertmanager
 
@@ -53,6 +66,8 @@ install -m 0644 grafana/datasource.yaml   /etc/grafana/provisioning/datasources/
 install -m 0644 grafana/dashboards.yaml   /etc/grafana/provisioning/dashboards/llmgw.yaml
 install -m 0644 -o grafana -g grafana grafana/llmgw-overview.json \
     /var/lib/grafana/dashboards/llmgw.json
+install -m 0644 -o grafana -g grafana grafana/llmgw-tenants.json \
+    /var/lib/grafana/dashboards/llmgw-tenants.json
 systemctl restart grafana-server
 ```
 
@@ -75,7 +90,8 @@ Alertmanager groups and holds alerts and has no receiver.
 Where an alert should go — an address, a Slack workspace, a webhook — is a
 decision with credentials attached, and a monitoring system that quietly starts
 mailing people is worse than one that is visibly not wired up. Alerts are
-visible at `:9093` and in Prometheus; add a receiver to `alertmanager.yml` to
+visible in Prometheus, and at `:9093` on the monitoring host itself (`ssh -L
+9093:127.0.0.1:9093 <host>` to open its page); add a receiver to `alertmanager.yml` to
 make them travel. `critical` already routes separately so it can be pointed
 somewhere else without touching the rest.
 
@@ -86,7 +102,7 @@ An alerting system nobody has seen fire is an assumption.
 ```
 llmgw ctl cordon <node>     # work already there keeps running
 # ~2 minutes later
-curl -s http://<host>:9093/api/v2/alerts | jq '.[].labels'
+ssh <host> curl -s http://127.0.0.1:9093/api/v2/alerts | jq '.[].labels'
 llmgw ctl uncordon <node>   # the alert resolves on its own
 ```
 

deploy/monitoring/grafana/llmgw-overview.json

@@ -487,6 +487,427 @@
         "defaults": {},
         "overrides": []
       }
+    },
+    {
+      "type": "row",
+      "title": "GPUs",
+      "id": 14,
+      "gridPos": {
+        "x": 0,
+        "y": 30,
+        "w": 24,
+        "h": 1
+      },
+      "collapsed": false,
+      "panels": []
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU busy, by card",
+      "description": "Share of each sample period a kernel was running, sampled every heartbeat.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 31,
+        "w": 12,
+        "h": 7
+      },
+      "id": 15,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_utilization_percent",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU memory in use, by card",
+      "description": "Framebuffer in use. Engines reserve most of it at start.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 31,
+        "w": 12,
+        "h": 7
+      },
+      "id": 16,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_memory_used_bytes / llmgw_gpu_memory_total_bytes",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU memory bandwidth busy, by card",
+      "description": "Decode is bound by memory: high here with compute low is a card waiting on memory.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 12,
+        "h": 7
+      },
+      "id": 17,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_memory_utilization_percent",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "min": 0,
+          "max": 100
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU temperature",
+      "description": "Throttling usually starts in the high 80s.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 38,
+        "w": 12,
+        "h": 7
+      },
+      "id": 18,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_temperature_celsius",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "celsius"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU power against its limit",
+      "description": "1 is the enforced limit; a card held there is power-capped.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 45,
+        "w": 12,
+        "h": 7
+      },
+      "id": 19,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_power_watts / llmgw_gpu_power_limit_watts",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1.1
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU clock",
+      "description": "A clock held low under load is the card throttled; see the next panel for why.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 45,
+        "w": 12,
+        "h": 7
+      },
+      "id": 20,
+      "targets": [
+        {
+          "expr": "llmgw_gpu_sm_clock_mhz",
+          "legendFormat": "{{node}} gpu {{gpu}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "rotmhz"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "What holds GPUs back",
+      "description": "Cards throttled now, by reason, and uncorrected ECC errors since the driver loaded.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 52,
+        "w": 24,
+        "h": 7
+      },
+      "id": 21,
+      "targets": [
+        {
+          "expr": "sum by (reason) (llmgw_gpu_throttled)",
+          "legendFormat": "throttled: {{reason}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (node, gpu) (llmgw_gpu_ecc_errors)",
+          "legendFormat": "ECC errors {{node}} gpu {{gpu}}",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Output tokens/s per GPU, by model",
+      "description": "What each model gets out of each card it holds.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 59,
+        "w": 12,
+        "h": 7
+      },
+      "id": 22,
+      "targets": [
+        {
+          "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) / count by (model) (llmgw_gpu_assigned)",
+          "legendFormat": "{{model}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "Output tokens per watt-hour, by model",
+      "description": "Tokens for the energy the model's cards draw.",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 59,
+        "w": 12,
+        "h": 7
+      },
+      "id": 23,
+      "targets": [
+        {
+          "expr": "sum by (model) (rate(llmgw_tokens_total{direction=\"completion\"}[5m])) * 3600 / sum by (model) (llmgw_gpu_power_watts * on (node, gpu) group_left (model) llmgw_gpu_assigned)",
+          "legendFormat": "{{model}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "type": "timeseries",
+      "title": "GPU wear",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "id": 24,
+      "description": "What wears cards out: XID errors in the last 15 minutes, memory rows remapped, PCIe retries an hour. Anything above zero here is worth a look on that card's node page.",
+      "gridPos": {
+        "x": 0,
+        "y": 66,
+        "w": 24,
+        "h": 7
+      },
+      "targets": [
+        {
+          "expr": "increase(llmgw_gpu_xid_errors[15m]) > 0",
+          "legendFormat": "XID {{node}} gpu {{gpu}}",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (node, gpu) (llmgw_gpu_remapped_rows) > 0",
+          "legendFormat": "rows remapped {{node}} gpu {{gpu}}",
+          "refId": "B"
+        },
+        {
+          "expr": "increase(llmgw_gpu_pcie_replays[1h]) > 0",
+          "legendFormat": "PCIe retries/h {{node}} gpu {{gpu}}",
+          "refId": "C"
+        },
+        {
+          "expr": "increase(llmgw_gpu_nvlink_errors[1h]) > 0",
+          "legendFormat": "NVLink {{kind}}/h {{node}} gpu {{gpu}}",
+          "refId": "D"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
     }
   ]
 }

deploy/monitoring/grafana/llmgw-tenants.json

@@ -0,0 +1,650 @@
+{
+  "uid": "llmgw-tenants",
+  "title": "llmgw \u2014 tenants",
+  "schemaVersion": 39,
+  "version": 1,
+  "editable": true,
+  "time": {
+    "from": "now-24h",
+    "to": "now"
+  },
+  "refresh": "1m",
+  "tags": [
+    "llmgw"
+  ],
+  "description": "Requests, tokens, failures, refusals and first-token time per tenant \u2014 the level router.tenant_label names \u2014 and every budget's share spent.",
+  "templating": {
+    "list": [
+      {
+        "name": "tenant",
+        "label": "Tenant",
+        "type": "query",
+        "datasource": {
+          "type": "prometheus",
+          "uid": "llmgw-prom"
+        },
+        "query": {
+          "query": "label_values(llmgw_tenant_requests_total, tenant)",
+          "refId": "tenant"
+        },
+        "definition": "label_values(llmgw_tenant_requests_total, tenant)",
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "current": {
+          "text": "All",
+          "value": "$__all"
+        },
+        "refresh": 2,
+        "sort": 1
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "title": "Requests a second",
+      "type": "stat",
+      "description": "Across the tenants chosen above.",
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 2,
+      "title": "Failing",
+      "type": "stat",
+      "description": "Requests that failed, not counting refusals by a limit.",
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.02
+              },
+              {
+                "color": "red",
+                "value": 0.05
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 3,
+      "title": "Refused by a limit",
+      "type": "stat",
+      "description": "Rate, token and budget limits; 429 to the caller.",
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum(rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.02
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 4,
+      "title": "First token, p95",
+      "type": "stat",
+      "description": "What people wait for before anything appears.",
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[5m])))",
+          "legendFormat": "",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 3
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "textMode": "auto",
+        "colorMode": "value"
+      }
+    },
+    {
+      "id": 5,
+      "title": "Requests by tenant",
+      "type": "timeseries",
+      "description": "Stacked, so the top edge is the whole of it.",
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": {
+            "stacking": {
+              "mode": "normal"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 6,
+      "title": "Tokens by tenant",
+      "type": "timeseries",
+      "description": "Prompt and completion together: what was consumed.",
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 12,
+        "h": 9
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": {
+            "stacking": {
+              "mode": "normal"
+            },
+            "fillOpacity": 30
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 7,
+      "title": "Refused by a limit, by tenant",
+      "type": "timeseries",
+      "description": "The share of each tenant's requests a limit turned away.",
+      "gridPos": {
+        "x": 0,
+        "y": 13,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 8,
+      "title": "Failing, by tenant",
+      "type": "timeseries",
+      "description": "One tenant alone failing is usually what it sends; all of them together is a model.",
+      "gridPos": {
+        "x": 12,
+        "y": 13,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"error\"}[5m])) / sum by (tenant) (rate(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[5m]))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 9,
+      "title": "First token p95, by tenant",
+      "type": "timeseries",
+      "description": "",
+      "gridPos": {
+        "x": 0,
+        "y": 21,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "histogram_quantile(0.95, sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket{tenant=~\"$tenant\"}[10m])))",
+          "legendFormat": "{{tenant}}",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right",
+          "calcs": [
+            "mean",
+            "max"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      }
+    },
+    {
+      "id": 10,
+      "title": "Budgets spent this period",
+      "type": "bargauge",
+      "description": "Every organization, team, project and person with a budget. Requests are refused at 100%.",
+      "gridPos": {
+        "x": 12,
+        "y": 21,
+        "w": 12,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sort_desc(llmgw_budget_spent_ratio)",
+          "legendFormat": "{{scope}} {{name}} ({{period}})",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          }
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 0.8
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ]
+        },
+        "showUnfilled": true
+      }
+    },
+    {
+      "id": 11,
+      "title": "Who used most, last 24 hours",
+      "type": "table",
+      "description": "Spend in money is on the people page's usage tab, which reads it from the store.",
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 24,
+        "h": 8
+      },
+      "datasource": {
+        "type": "prometheus",
+        "uid": "llmgw-prom"
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sort_desc(sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\"}[24h])))",
+          "legendFormat": "requests",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        },
+        {
+          "refId": "B",
+          "expr": "sum by (tenant) (increase(llmgw_tenant_tokens_total{tenant=~\"$tenant\"}[24h]))",
+          "legendFormat": "tokens",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        },
+        {
+          "refId": "C",
+          "expr": "sum by (tenant) (increase(llmgw_tenant_requests_total{tenant=~\"$tenant\",outcome=\"limited\"}[24h]))",
+          "legendFormat": "limited",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "llmgw-prom"
+          },
+          "format": "table",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true
+      },
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true
+            },
+            "renameByName": {
+              "Value #A": "Requests",
+              "Value #B": "Tokens",
+              "Value #C": "Refused by a limit"
+            }
+          }
+        }
+      ]
+    }
+  ]
+}

deploy/monitoring/llmgw-rules.yml

@@ -42,10 +42,119 @@ groups:
         annotations:
           summary: "{{ $labels.model }} has no replica ready"
           description: >-
-            The model is placed and nothing is serving it. Ten minutes, because
-            a large model on CPU legitimately takes minutes to load. A model
-            that is deliberately stopped publishes no series at all, so this
-            cannot fire for one nobody asked for.
+            The model should be running and nothing is serving it. Ten minutes,
+            because a large model on CPU legitimately takes minutes to load. The
+            series is published at 0 for every model that should be running —
+            before v0.10.4 it was published only while a replica was ready, so
+            this could not fire at all. A model deliberately stopped, or an
+            on-demand one nobody is asking for, publishes nothing.
+
+      - alert: ModelParked
+        expr: llmgw_model_parked == 1
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.model }} failed every time it started and is no longer retried"
+          description: >-
+            Five replicas in a row failed at start. Before this existed a model
+            whose weights its engine could not load was started 155 times over
+            eight hours. `llmgw ctl status` has the last error; once it is dealt
+            with, `llmgw ctl retry <model>`. Changing the model's definition
+            also retries it.
+
+      - alert: ModelFailingToStart
+        expr: llmgw_model_start_failures >= 2
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} has failed to start {{ $value }} times in a row"
+          description: >-
+            On its way to being parked. The engine's own words are in
+            `llmgw ctl logs <node> --source <shard>` for the next attempt, or in
+            the control plane's log for the last one.
+
+      - alert: ModelCannotBePlaced
+        expr: llmgw_model_unplaceable == 1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.model }} is wanted and fits on no node"
+          description: >-
+            Short of replicas, and the last attempt found nowhere with the
+            memory, cards or engine it needs. `llmgw ctl explain-placement
+            <model>` says what each node lacks. For an on-demand model, requests
+            for it are told this rather than asked to retry.
+
+      - alert: ModelPreempted
+        expr: increase(llmgw_model_preempted_total[15m]) > 0
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.model }} ({{ $labels.class }}) gave up replicas to a higher class"
+          description: >-
+            Capacity ran short and a model of a higher class took this one's
+            room. Expected now and then; if it keeps firing for the same model,
+            the cluster is too small for what is asked of it at these classes.
+            `llmgw ctl models` shows who it gave way to.
+
+      - alert: CapacityAtLimit
+        expr: llmgw_capacity_holds >= on(scope, resource) llmgw_capacity_limit
+        for: 15m
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.scope }} holds all the {{ $labels.resource }} it may"
+          description: >-
+            Its models get no further replicas, under load or not, until
+            something of theirs stops or the limit is raised: `llmgw ctl quota
+            list`, `llmgw ctl quota set`.
+
+      - alert: EngineInstallFailed
+        expr: llmgw_node_engine_install{state="failed"} == 1
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.engine }} {{ $labels.version }} will not install on {{ $labels.node }}"
+          description: >-
+            The node takes no model needing it until it does. The agent retries
+            on its own; `llmgw ctl nodes` and the node's logs on the cluster
+            page say why it failed.
+
+      - alert: ControlPlaneLeadership
+        expr: sum(llmgw_control_plane_leader) != 1
+        for: 2m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $value }} control planes believe they are placing work"
+          description: >-
+            None means nothing is placed or replaced; two means both are, each
+            into GPUs the other believes free. The lease is in the store: check
+            it is reachable from every control plane.
+
+      - alert: CertificateExpiresSoon
+        expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 30 * 86400
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
+          description: >-
+            Certificates last a year and nothing renews them. When the agent
+            server's runs out every agent refuses to connect at once. Reissue
+            with `llmgw ctl cert` and restart the control plane.
+
+      - alert: CertificateExpiresVerySoon
+        expr: (llmgw_certificate_expiry_timestamp_seconds - time()) < 7 * 86400
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.cert }} ({{ $labels.subject }}) expires in {{ $value | humanizeDuration }}"
+          description: See CertificateExpiresSoon; a week is not long.
+
+      - alert: BackupTooOld
+        expr: time() - llmgw_backup_last_success_timestamp_seconds > 26 * 3600
+        labels: { severity: warning }
+        annotations:
+          summary: "The newest llmgw backup is {{ $value | humanizeDuration }} old"
+          description: >-
+            llmgw-backup.timer runs nightly. Also fires when there has never
+            been a backup (the series is 0 then). `journalctl -u
+            llmgw-backup.service` on the control plane says why the last one
+            failed; docs/install.md, "Backing up", says how to set it up.
 
       - alert: PoolSaturated
         expr: llmgw_pool_saturation > 1
@@ -91,6 +200,94 @@ groups:
             is skipped by placement rather than failing, so this is quiet until
             somebody wonders why a model will not land there.
 
+  - name: llmgw-gpus
+    rules:
+      - alert: GPUThermalThrottling
+        expr: llmgw_gpu_throttled{reason=~"sw_thermal|hw_thermal|hw_slowdown"} == 1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} is slowed by heat ({{ $labels.reason }})"
+          description: >-
+            Ten minutes held back by temperature: every replica on the card is
+            slower than its measurements say. Usually airflow, a fan, or the
+            room — the node page shows the card's temperature over time.
+
+      - alert: GPUUncorrectedECCErrors
+        expr: llmgw_gpu_ecc_errors > 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has {{ $value }} uncorrected ECC errors"
+          description: >-
+            Uncorrected memory errors since the driver loaded: what runs on the
+            card may be computing on corrupted memory. Drain the node
+            (llmgw ctl drain) and have the card looked at.
+
+      - alert: GPUXidError
+        expr: increase(llmgw_gpu_xid_errors[15m]) > 0
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} logged an XID error"
+          description: >-
+            The driver reported an error on the card (its code is
+            llmgw_gpu_last_xid, and the node page says it in words). 79 is a
+            card fallen off the bus, 48 and 63/64 memory errors, 74 NVLink,
+            94/95 contained and uncontained ECC errors: those need the card
+            looked at. Others are often an engine's own fault, and pass.
+
+      - alert: GPUOutOfSpareRows
+        expr: llmgw_gpu_remap_failure == 1
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has no spare memory rows left"
+          description: >-
+            The card cannot remap another failing row. Drain the node and
+            have the card replaced.
+
+      - alert: GPUNeedsReset
+        expr: llmgw_gpu_remap_pending == 1 or llmgw_gpu_retirement_pending == 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} has memory repairs waiting for a reset"
+          description: >-
+            Rows remapped or pages retired take effect only after the card is
+            reset. Drain the node (llmgw ctl drain), reset or reboot, and bring
+            it back.
+
+      - alert: GPUPCIeRetriesClimbing
+        expr: increase(llmgw_gpu_pcie_replays[1h]) > 100
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: {{ $value }} PCIe retries in an hour"
+          description: >-
+            The card's link to the host is retrying. A riser, the slot or a
+            cable is the usual cause, and transfers to the card slow down.
+
+      - alert: GPUNVLinkErrorsClimbing
+        expr: increase(llmgw_gpu_nvlink_errors{kind="crc"}[1h]) > 10
+        for: 0m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }}: NVLink CRC errors climbing"
+          description: >-
+            Errors on the card's NVLink links: tensor-parallel replicas on it
+            slow down, and a link that keeps failing takes a replica with it.
+
+      - alert: GPUHot
+        expr: llmgw_gpu_temperature_celsius > 87
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.node }} GPU {{ $labels.gpu }} at {{ $value }} °C"
+          description: >-
+            Ten minutes above 87 °C, where most data-centre cards begin to
+            throttle.
+
   - name: monitoring-host
     rules:
       - alert: MonitoringDiskFilling
@@ -153,3 +350,130 @@ groups:
             Working as designed when the cluster is full: the lowest band is
             held first. Worth knowing about because the callers being refused
             do not know why.
+
+  # Per tenant: the level router.tenant_label names (a team by default), and
+  # every organization, team, project and person given a budget.
+  - name: llmgw-tenants
+    rules:
+      - alert: BudgetMostlySpent
+        expr: llmgw_budget_spent_ratio >= 0.8 and llmgw_budget_spent_ratio < 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.scope }} {{ $labels.name }} has spent {{ $value | humanizePercentage }} of its {{ $labels.period }}ly budget"
+          description: >-
+            Its requests are refused once it reaches all of it, until the
+            period turns. Raise it on the people page, or tell them now
+            rather than when they are refused.
+
+      - alert: BudgetSpent
+        expr: llmgw_budget_spent_ratio >= 1
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "{{ $labels.scope }} {{ $labels.name }} has spent its {{ $labels.period }}ly budget"
+          description: >-
+            Every request under it is refused with 429 until the period
+            turns, unless a cheaper model is named as its fallback.
+
+      - alert: TenantHittingLimits
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="limited"}[10m]))
+          / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests are refused by a limit"
+          description: >-
+            A rate, token or budget limit is turning their traffic away. Either
+            the limit is too low for what they now do, or something of theirs
+            is looping.
+
+      - alert: TenantErrors
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total{outcome="error"}[10m]))
+          / sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05
+          and sum by (tenant) (rate(llmgw_tenant_requests_total[10m])) > 0.05
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $value | humanizePercentage }} of {{ $labels.tenant }}'s requests fail"
+          description: >-
+            Not counting refusals by a limit. If every tenant fails alike it
+            is a model (ModelFailing); one alone is usually what they send —
+            prompts past the context, a model they may not use.
+
+      - alert: TenantTrafficSpike
+        expr: >-
+          sum by (tenant) (rate(llmgw_tenant_requests_total[30m]))
+          > 3 * sum by (tenant) (rate(llmgw_tenant_requests_total[7d]))
+          and sum by (tenant) (rate(llmgw_tenant_requests_total[30m])) > 0.2
+        for: 15m
+        labels: { severity: info }
+        annotations:
+          summary: "{{ $labels.tenant }} is sending three times its weekly average"
+          description: >-
+            A new workload, a batch job, or a client retrying in a loop. Only
+            for a tenant sending more than one request every five seconds, so
+            a quiet team starting work in the morning does not page anybody.
+
+      - alert: TenantSlowFirstToken
+        expr: >-
+          histogram_quantile(0.95,
+            sum by (tenant, le) (rate(llmgw_tenant_ttft_seconds_bucket[15m]))) > 10
+          and sum by (tenant) (rate(llmgw_tenant_ttft_seconds_count[15m])) > 0.02
+        for: 15m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.tenant }} waits {{ $value | humanizeDuration }} for a first token at p95"
+          description: >-
+            What their people feel. Compare with SlowFirstToken per model: if
+            only this tenant is slow, it is the models it uses or its queue
+            band, not the cluster.
+  # The store, when it is a replicated cluster under Patroni (llmgw server
+  # init --patroni). Published only for a store of several hosts.
+  - name: llmgw-store
+    rules:
+      - alert: StoreHasNoLeader
+        expr: llmgw_store_patroni_up == 1 and on() count(llmgw_store_member{role="leader"}) == 0
+        for: 1m
+        labels: { severity: critical }
+        annotations:
+          summary: "The store's cluster has no primary"
+          description: >-
+            Every write fails — keys, usage, budgets, leadership. Patroni
+            promotes a standby within its 30-second TTL; a minute without one
+            means it cannot: etcd has lost its majority, or no standby is fit.
+            `patronictl -c /etc/patroni/config.yml list` on a member says which.
+
+      - alert: StorePatroniUnreachable
+        expr: llmgw_store_patroni_up == 0
+        for: 2m
+        labels: { severity: warning }
+        annotations:
+          summary: "No member of the store's cluster answers on its Patroni port"
+          description: >-
+            The control plane reads the cluster's state from port 8008 on each
+            member. The database may still be serving; nothing here can tell.
+
+      - alert: StoreReplicaNotStreaming
+        expr: llmgw_store_member{role!="leader", state!="streaming"} == 1
+        for: 5m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.member }} is {{ $labels.state }}, not streaming from the primary"
+          description: >-
+            A standby that is not streaming is not a standby: if the primary
+            is lost now, there is nothing to promote. "missing" is a host of
+            the store address the cluster no longer lists — usually a machine
+            that is down.
+
+      - alert: StoreReplicaBehind
+        expr: llmgw_store_replica_lag_bytes > 64 * 1024 * 1024 or llmgw_store_replica_lag_bytes == -1
+        for: 10m
+        labels: { severity: warning }
+        annotations:
+          summary: "{{ $labels.member }} is {{ $value | humanize1024 }}B behind the primary"
+          description: >-
+            Patroni will not promote a standby more than 1 MB behind, so a
+            failover now would find none fit. -1 is a lag Patroni cannot read.

deploy/systemd/llmgw-backup.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=llmgw backup: the store and /etc/llmgw, to /var/backups/llmgw-server
+After=llmgw-server.service postgresql.service
+
+[Service]
+Type=oneshot
+# Nothing it writes is for anyone else: the archive holds the CA's key and a
+# copy of the store.
+UMask=0077
+# Root, because the archive includes /etc/llmgw/secrets and the certificate
+# authority's key, which the llmgw user deliberately cannot read all of.
+ExecStart=/usr/local/bin/llmgw server backup --config /etc/llmgw/server.yaml --keep 14
+Nice=10
+IOSchedulingClass=idle

deploy/systemd/llmgw-backup.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=Back up llmgw every night
+
+[Timer]
+OnCalendar=*-*-* 03:15:00
+# A machine that was off at 03:15 backs up when it next starts, rather than
+# waiting another day with a gap nobody notices until the alert.
+Persistent=true
+RandomizedDelaySec=10m
+
+[Install]
+WantedBy=timers.target

docs/fabrics.md

@@ -121,6 +121,39 @@ rate and errors, and the NCCL environment a shard on each fabric would be given.
 `--json` writes every fact it read as well as what it concluded — send that
 file when the report is not enough.
 
+## Checking two nodes: `llmgw ctl fabric-test`
+
+Every check on one node can pass while two nodes still cannot reach each other
+— a cable into the wrong switch, a partition key, a RoCE network with no route
+— and a replica spanning them then hangs on its first all-reduce. Before
+placing one, on the control plane:
+
+```
+llmgw ctl fabric-test gpu01 gpu02
+```
+
+```
+gpu01 → gpu02 on ib-compute, 41s
+
+RAIL              LINK      MEASURED    VERDICT  DETAIL
+mlx5_0 → mlx5_0   400 Gb/s  388.2 Gb/s  ok       97% of 400 Gb/s
+mlx5_1 → mlx5_1   400 Gb/s  241.0 Gb/s  slow     60% of 400 Gb/s: check the cable, and the card's PCIe width and NUMA node
+mlx5_2 → mlx5_2   400 Gb/s  0.0 Gb/s    fail     gpu01: ib_write_bw: Couldn't connect to 10.1.0.22:18517
+```
+
+It runs `ib_write_bw` (perftest, installed on a node the first time) from each
+of the first node's adapters on the fabric to the same adapter on the second —
+mlx5_0 to mlx5_0, the way NCCL pairs them — one rail at a time, through the
+agents, so no SSH between nodes is needed. Each rail is graded against its
+link's rate: **ok** at 80% or more, **slow** at 50%, **fail** below that or when
+nothing got through. `--fabric` picks one when the nodes share several,
+`--seconds` the length of each rail's run. The traffic is real: run it when the
+fabric is quiet.
+
+It measures RDMA between adapters, not GPUDirect or NCCL itself: a pass says
+the fabric carries traffic between these two nodes at its rate, which is what
+a first all-reduce needs and what nothing on one node can show.
+
 ## Watching them
 
 - **Cluster page, Fabrics.** Each declared fabric and what it physically is;
@@ -172,6 +205,6 @@ workers headless. `launch_mode: ray` is there for a vLLM too old for that.
 | It | Look at |
 |---|---|
 | is not placed | `llmgw ctl explain-placement <model>`: a fabric split by identity, a node whose IMEX is down, a node refused because its adapters are cabled to two fabrics |
-| starts and hangs on its first collective | the cluster page's fabric problems; `doctor` on each node — a port down, the wrong interface, RoCE without a v2 GID |
+| starts and hangs on its first collective | `llmgw ctl fabric-test` between two of its nodes; the cluster page's fabric problems; `doctor` on each node — a port down, the wrong interface, RoCE without a v2 GID |
 | runs, slowly | `nvidia_peermem`; the map for a degraded cable; the node page's NVLink matrix for a group that crossed PCIe |
 | worked yesterday | the subnet manager table for a failover; `link_downed` on the node's ports |

docs/facility-aware-placement.md

@@ -0,0 +1,303 @@
+# Facility-aware placement: rack power, temperature and cooling
+
+**Status: step 0, reading the nodes' own sensors, is built; steps 1–5 are
+deferred** until the rack, PDU and power information of a real site is at
+hand — without it they would be designed against guesses.
+This is the design for step 5 of resource allocation, after reserved CPU cores
+and memory (1), capacity classes and preemption (2), tenant capacity quotas (3)
+and sizing by measured throughput (4), all of which shipped in v0.10.5. Of
+what follows, only the node sensors (rollout step 0) exist in the code: they
+are read, published and shown, and change no decision.
+
+## Why
+
+In a large installation the scarce thing is often not a GPU but the room it
+sits in. Three failures are invisible to placement today:
+
+- **A rack goes over its power budget.** Every node in it reports healthy;
+  placement sees free cards and fills them; the breaker, or the facility's
+  power cap, decides the rest. A rack with A and B feeds is worse: it runs
+  fine until one feed fails and the other cannot carry the load.
+- **Hot hardware is slow hardware.** GPUs throttle long before they fail. A
+  replica placed in a rack whose inlet air is too warm serves fewer tokens per
+  second than its measured capacity says, and nothing connects the two.
+- **Correlated failure.** Two replicas of a model on two nodes in the same rack
+  or on the same PDU are one failure away from zero. Placement spreads replicas
+  across nodes, not across racks.
+
+The goal is for placement to treat facility limits the way it already treats
+memory and cores: reserve against a budget, refuse what does not fit, say why,
+and never let missing data stop the cluster.
+
+## What it builds on
+
+| Already in place | Used for |
+|---|---|
+| Node labels, reported by agents, matched by `node_selector` | Where a node is: its rack, row, PDU |
+| `inventory.Allocation` carries GPUs, memory and cores per shard | Adding watts, summed per rack and PDU |
+| Placement rejections (`explain-placement`, headroom's "why not") | Naming the rack that is full or hot |
+| Headroom: the scheduler dry-run on a copy of the inventory | Room that respects power budgets, for free |
+| Preemption with classes, minimums and a dry run (step 2) | Shedding load from a hot rack in the right order |
+| Measured throughput per profile (step 4) | Tokens per watt as a placement preference |
+| Node sensors, read by every agent (step 0, built) | Heat and power with no facility integration: see below |
+| `internal/promq`, a Prometheus client | Reading facility metrics that DCIM exporters publish |
+
+## Concepts
+
+**Location.** Each node has a place in a tree: site, room, row, rack. Power has
+its own tree beside it: a rack draws from one or more PDUs, and each PDU from a
+feed. Both are read from node labels with well-known keys:
+
+```yaml
+# agent.yaml
+labels:
+  llmgw.io/site: sto1
+  llmgw.io/room: hall-b
+  llmgw.io/row: "12"
+  llmgw.io/rack: r12-07
+  llmgw.io/pdu: r12-07-a,r12-07-b   # both feeds, for a dual-fed rack
+```
+
+A node without these labels is its own rack of one: placement behaves as it
+does today.
+
+**Domain.** A rack, a row, a PDU, a room: anything with a budget or a
+condition. Domains are declared in a facility file (below) or discovered from
+node labels; a label naming a domain the file does not declare creates it with
+no budget and no condition.
+
+**Budget.** A domain may have a power cap in watts. Placement reserves each
+replica's estimated draw against every domain its node belongs to and refuses a
+placement that would take any of them past its cap less a safety margin. For a
+dual-fed rack the cap that matters is what one feed can carry alone, since that
+is what the rack must survive.
+
+**Condition.** A domain has a thermal and cooling condition: `ok`, `warm`, `hot`
+or `unknown`, derived from readings against thresholds with hysteresis.
+Conditions change what placement prefers and allows; they are not amounts to
+fill.
+
+**Reading.** A measured value with a time: inlet temperature, power draw, a
+cooling unit's alarm state. Readings older than their staleness limit are
+treated as absent.
+
+## Estimating a replica's draw
+
+What placement reserves has to be known before the replica runs, so it is an
+estimate first and a measurement later:
+
+1. **Declared.** A profile may say `power_watts`. The operator's figure wins.
+2. **Derived.** Otherwise GPU profiles take each card's power limit from NVML
+   (the agent already reads the cards; the power limit is one more field) plus
+   a per-node share for CPU, memory and fans; CPU profiles take their reserved
+   cores times a per-core figure for the node's CPU model.
+3. **Measured.** Once replicas run, the agent reports their cards' actual draw
+   (NVML) and, where the BMC allows, the node's (Redfish or IPMI DCMI). A
+   domain's headroom is computed from the larger of what is reserved and what
+   is measured, so an estimate that was too low is corrected by the room it
+   stops offering, not by an outage.
+
+## Decisions
+
+### Power
+
+- **Hard limit.** A placement that would take any domain past `cap × (1 −
+  margin)` is rejected, with the domain and the figures: *rack r12-07: 14.2 of
+  16.0 kW reserved, replica needs ~2.8, margin 10%*.
+- **Feeds.** For a domain with two feeds the cap is the smaller feed's rating
+  times its derating (typically 80%), not the sum.
+- **Already over.** A domain found over its cap, because the estimate was low
+  or the cap was lowered, gets no new placements, and an alert. It is not
+  emptied automatically; shedding power is the facility's decision, and
+  stopping inference to do it is ours only if asked (see shedding below).
+
+### Temperature and cooling
+
+Thresholds are per domain with defaults, and each condition needs to hold for a
+while before it changes, in both directions, so a single reading neither
+starts nor ends anything:
+
+| Condition | Enters when | Placement |
+|---|---|---|
+| `ok` | below the warm threshold for 5 min | normal |
+| `warm` | inlet above the warm threshold for 2 min, GPU temperatures near throttling, or any GPU thermally throttled | allowed, scored down: other domains first |
+| `hot` | inlet above the hot threshold for 2 min, a cooling unit in alarm, or a sustained share of the domain's GPUs thermally throttled | no new placements, as a cordon would, but automatic and reversible |
+| `unknown` | no fresh reading | as `ok`, and reported as unknown |
+
+**Shedding** (optional, off by default): a domain `hot` for longer than its
+shed delay has load moved off it using step 2's machinery. The lowest class goes
+first, and replicas above their model's minimum go before any at it. A critical
+minimum is never touched, and moves are rate limited to one replica per domain
+per interval. The replica is placed elsewhere first where there is room, so
+shedding moves work rather than dropping it. GPU power capping (`nvidia-smi
+-pl`) is a gentler alternative worth measuring against shedding: less
+throughput, no restart.
+
+### Spreading
+
+Replicas of one model are spread across racks, and across PDUs, the way they
+are spread across nodes today. This is a preference, dropped when it is the only
+thing standing in the way, as node spreading already is.
+
+### Scoring
+
+Placement today filters, then takes the first candidate in a fixed order
+(fewest free GPUs for GPU profiles, fewest replicas for CPU ones, nodes holding
+the weights first). Facility signals are better as a score than as more
+filters. Candidates that pass the hard limits are ordered by:
+
+1. condition: `ok` before `warm`;
+2. power headroom left in the rack after this replica, more first, so load
+   levels across racks;
+3. the existing order.
+
+Once measurements include power, step 4's profile preference can use tokens per
+watt instead of tokens per GPU where both are known.
+
+## Where the data comes from
+
+A `FacilitySource` interface, implemented several ways, chosen in
+`server.yaml`:
+
+```go
+type FacilitySource interface {
+    // Domains is what the source knows: budgets, thresholds, members.
+    Domains(ctx) ([]Domain, error)
+    // Readings is the latest value of each signal, per domain.
+    Readings(ctx) ([]Reading, error)
+}
+```
+
+- **Static file.** Domains, caps and thresholds, written by hand or generated
+  from a DCIM export. Enough for power budgets with no live data at all.
+- **Prometheus.** DCIM, PDU (SNMP exporter) and BMC (Redfish exporter)
+  readings usually end up in Prometheus. The source runs configured instant
+  queries and maps their labels onto domains:
+
+  ```yaml
+  facility:
+    source: prometheus
+    prometheus: http://monitoring:9090
+    signals:
+      inlet_temp_c:
+        query: max by (rack) (dcim_rack_inlet_temperature_celsius)
+        domain_label: rack
+        stale_after: 2m
+      power_watts:
+        query: sum by (pdu) (pdu_outlet_power_watts)
+        domain_label: pdu
+      cooling_alarm:
+        query: max by (row) (crac_alarm)
+        domain_label: row
+  ```
+- **Agents (built, step 0).** What each node reads about itself, every ten
+  seconds at most, sent with its heartbeat:
+  - **GPUs, from nvidia-smi:** temperature, memory (HBM) temperature where the
+    driver reports it, power draw, the enforced power limit, and what is
+    holding the clocks down now — `sw_power_cap`, `hw_slowdown`, `sw_thermal`,
+    `hw_thermal`, `power_brake`.
+  - **CPUs, from hwmon** (what lm_sensors reads, without it installed): the
+    package temperature, or the hottest core; and the board's inlet or ambient
+    sensor where it has one (`SYSTIN`, `Inlet`, `Ambient`).
+  - **CPU power, from RAPL:** the packages' energy counters over time,
+    subdomains not counted twice, counter wrap handled.
+
+  A machine without a sensor reports nothing for it, never a zero; a VM has
+  none of them. Published as `llmgw_gpu_temperature_celsius`,
+  `llmgw_gpu_memory_temperature_celsius`, `llmgw_gpu_power_watts`,
+  `llmgw_gpu_power_limit_watts`, `llmgw_gpu_throttled{reason}`,
+  `llmgw_node_cpu_temperature_celsius`, `llmgw_node_cpu_power_watts` and
+  `llmgw_node_inlet_temperature_celsius`, and on each node's page.
+
+  What they can and cannot say:
+  - **Throttling is the strongest signal there is.** A temperature is a proxy;
+    a card holding its clocks down is the fact that invalidates its measured
+    capacity, and it needs no threshold to be chosen.
+  - **Measured power** is GPU draw plus CPU packages; memory, disks, fans and
+    power-supply losses are not in it, so a node's draw is that plus an
+    estimated remainder. A rack's is its llmgw nodes' plus a declared figure
+    for everything else in it: switches and storage report nothing here.
+  - **CPU and GPU temperatures follow the load, not the room.** A hot rack is
+    inferred from many nodes running hot at comparable load, or from
+    throttling, not from one node's CPU. Only a board inlet sensor reads the
+    air, and most boards have none.
+  - **No budget comes from a sensor.** A reading is a draw; the cap it is
+    measured against still comes from the facility.
+- **Simulated.** Readings from a script, for tests and for rehearsing a
+  thermal event on a cluster without facility data.
+
+## Safety
+
+- **Missing data never blocks.** A source that is down, a reading that is stale,
+  a node without location labels: placement behaves as it does today, and says
+  the data is missing (a metric, a line on the cluster page). Facility
+  awareness can make placement more careful, never impossible.
+- **Hysteresis and hold times** on every condition, so a noisy sensor does not
+  flap a rack between allowed and forbidden.
+- **Observe-only first.** A mode where every decision is computed, logged and
+  shown ("would have refused: rack r12-07 hot") but nothing is enforced, run
+  for long enough to trust the thresholds before turning enforcement on.
+- **Manual override.** `llmgw ctl facility hold RACK` and `release RACK`, for the
+  facility team to mark a domain unavailable, or to clear a condition they know
+  to be a sensor fault.
+- **Shedding is opt-in**, per domain or cluster-wide, and rate limited.
+
+## What people see
+
+- `explain-placement` and headroom name the domain: *rack r12-07 is hot (inlet
+  31°C for 6 min)*, *pdu r12-07-a: 7.1 of 7.4 kW*.
+- `llmgw ctl facility` lists domains with budget, reserved, measured and
+  condition. The cluster page gets a rack summary beside the nodes table.
+- Metrics: `llmgw_domain_power_budget_watts`, `_reserved_watts`,
+  `_measured_watts`, `llmgw_domain_condition{condition}`,
+  `llmgw_domain_reading_age_seconds`; alerts for a domain near its cap, a
+  domain hot, a domain over its cap, and facility data gone stale.
+
+## Rollout
+
+Each step ships on its own and is useful without the next:
+
+0. **Node sensors, observe-only (built).** GPU heat, power and throttling,
+   CPU heat and power, inlet air where the board has a sensor; published and
+   shown, no decision changed.
+1. **Location and spreading.** Well-known labels, domains from labels, replicas
+   spread across racks and PDUs, locations on the node views. No facility data
+   needed.
+2. **Power budgets from a static file.** Caps and margins, estimated draw per
+   replica (declared or derived), hard limit in placement and headroom.
+3. **Readings, observe-only.** The Prometheus source beside the agents',
+   conditions with hysteresis (throttling first among them), measured power
+   folded into headroom; decisions shown, not enforced.
+4. **Thermal enforcement.** `warm` scored down, `hot` refused, manual hold and
+   release.
+5. **Shedding and tokens per watt.** Opt-in shedding through step 2's
+   preemption; measured power in profile preference.
+
+## Testing
+
+- The simulated source drives unit tests of every condition transition, the
+  hysteresis, staleness, and the placement outcomes, with mutation checks as
+  for steps 1–4.
+- Property tests: whatever the readings, placement never exceeds a declared
+  cap with fresh data, and never refuses everything because data is missing.
+- On the PoC cluster, with no facility data: step 1 with labelled VMs, step 2
+  with a static file of made-up caps, steps 3–4 with the simulated source
+  replaying a thermal event.
+- Before enforcement anywhere real: a period in observe-only mode against the
+  site's actual data.
+
+## Open questions
+
+- **What the sites have.** Which DCIM, which PDUs, which BMCs, and whether any
+  of it already reaches a Prometheus. The source design assumes Prometheus is
+  the common ground; a site without it needs another source.
+- **Who owns the numbers.** Caps, deratings and thresholds belong to the
+  facility team. Whether they edit llmgw's file, or llmgw reads theirs.
+- **Phase balance.** Three-phase PDUs have per-phase limits; whether placement
+  should see phases or only the PDU total.
+- **Cooling granularity.** Cooling is often per row or per aisle, not per rack;
+  the domain tree allows either, but the defaults should match what sites
+  actually measure.
+- **Power capping instead of shedding.** Whether to lower GPU power limits in a
+  warm rack rather than move replicas, and how that interacts with measured
+  capacity from step 4, which would no longer hold at the lower limit.

docs/install.md

@@ -87,7 +87,24 @@ CGO_ENABLED=0 go build -o llmgw ./cmd/llmgw
 
 ## 2. The control plane
 
-On the machine that will be the control plane, as root:
+**The short way is the screens.** On the machine that will be the control
+plane, run `sudo llmgw` with nothing after it. With nothing installed it asks
+whether this is one machine or three for high availability; for three, it asks
+for the other two — address, user, and an ssh key, the ssh agent, or a
+password, and the sudo password if sudo asks for one; or, for a machine
+nothing may log into, a join code to run there with `sudo llmgw join <code>`,
+which dials out to this one on port 7444 — checks all three before
+installing anything (system, room, clock, ports, internet, whether they reach
+one another), shows the plan, and installs. Passwords are held in memory for
+the run only; a sudo password lets the user sudo without one until the install
+finishes, and is then taken back. At the end it shows the administrator's
+password and the command that does the same from a script.
+
+Run again later, `sudo llmgw` is the console: the cluster's state, refreshed,
+and what to do next — add a node, replicate the store. A setup that stopped
+halfway opens the setup screens again, which finish it.
+
+The rest of this section is the same by command, as root:
 
 ```
 sudo llmgw server init --admin you@example.com
@@ -121,6 +138,20 @@ Add `--postgres` to keep accounts and keys in PostgreSQL rather than SQLite:
 init installs it and sets it up, and on a machine already running on SQLite it
 moves everything across. See [PostgreSQL](#postgresql-when-sqlite-is-no-longer-the-right-shape).
 
+Or `--patroni`, given two machines (or more), to keep it in PostgreSQL
+replicated across them, so losing one loses nothing:
+
+```
+sudo --preserve-env=SSH_AUTH_SOCK llmgw server init --patroni ubuntu@10.0.0.149 --patroni ubuntu@10.0.0.150
+```
+
+Over ssh with passwordless sudo, as `node add` works, it installs PostgreSQL,
+Patroni and etcd there — this machine is etcd's third vote — and moves the
+store already in use, SQLite or PostgreSQL, into the cluster. A standby is
+promoted within about thirty seconds of the primary being lost, and the
+control plane follows it by itself. `llmgw ctl status` says who leads; see
+[deploy/ha](../deploy/ha/README.md) for what it sets up.
+
 Every step looks before it acts. Running it again finishes an install that
 stopped halfway and changes nothing on one that did not; it never overwrites
 `server.yaml`, the authority, or an existing account's password.
@@ -367,7 +398,7 @@ Nothing to do: the agent installs them once the node joins, under
 
 | Engine | On | How |
 |---|---|---|
-| llama.cpp | every node | the release build for its processor, CUDA where there is a GPU, checked against GitHub's checksum |
+| llama.cpp | every node | the release build for its processor and GPU — CUDA 13.3, CUDA 12.8 (x64), or Vulkan for GPUs and drivers the CUDA builds do not cover; the CPU build only where there is no NVIDIA GPU — checked against GitHub's checksum and seen to list the GPU |
 | vLLM | nodes with an NVIDIA GPU it can run on | a Python environment built by uv from PyPI, about 8 GB |
 | SGLang | only when you ask for it | the same way as vLLM |
 
@@ -376,8 +407,17 @@ node gets the same dependencies whenever it is built, and installed with
 `--require-hashes` against the sha256 PyPI publishes. The resolved list stays in
 the environment as `llmgw-requirements.txt`. It needs:
 
-- **driver 580 or newer and a Turing GPU or newer** (compute capability 7.5):
-  vLLM is built for CUDA 13. A V100 cannot run it;
+- **the build its hardware runs**, chosen by the node. Driver 580 and newer:
+  the CUDA 13 build from PyPI. Driver 525 to 579: the same version built for
+  CUDA 12.9, from vLLM's GitHub release (checked against the digest GitHub
+  publishes) with PyTorch's CUDA 12.9 torch. Both have kernels for Turing
+  (7.5) and newer, 8.0 and newer on aarch64. **A V100** (Volta, 7.0) gets
+  **vLLM 0.19.1**, the newest built for it — no build from 0.20 on has its
+  kernels — and the cluster page shows it running that version; run `llmgw
+  ctl engine-check vllm` there once, since llmgw's flags are checked against
+  the current one. Pascal and older run no vLLM. Whatever is chosen, the
+  install ends by computing something on the GPU; a build that cannot is
+  passed over for the next;
 - **about 25 GB free** under the engines directory while it installs;
 - **gcc**, which Triton builds its kernel launcher with. The agent installs it.
 
@@ -394,8 +434,51 @@ engines:
 ```
 
 A changed version reaches every node when it next connects. To use an engine
-you installed yourself instead, set its version to `""` and name its path in
-the node's `agent.yaml` (`engines.vllm.bin`, `engines.sglang.python`).
+you installed yourself on a node, name its path in that node's `agent.yaml`
+(`engines.llamacpp.bin`, `engines.vllm.bin`, `engines.sglang.python`): the agent
+then installs none of its own beside it.
+
+#### Choosing a node's vLLM build
+
+Which build a GPU node installs is chosen on the cluster page, in the **vLLM
+build** column — *auto* (what its hardware runs, the default), *CUDA 13*,
+*CUDA 12.9*, or *none* — or from the command line:
+
+```
+llmgw ctl engine-build gpu07 vllm cu129
+```
+
+The choice is kept by the control plane and the node is told at once; it
+installs the build in the background, and the column shows how that stands.
+
+#### Different GPUs, different engines
+
+A cluster of two GPU generations may need two builds of an engine, or one on
+only some nodes. `engine_overrides` picks nodes — by a GPU class they have, by
+id, or by labels, every one given having to match — and lays engines over the
+cluster's for them, in order:
+
+```yaml
+engines:
+  vllm: "0.30.0"              # CUDA 13: Turing and newer
+engine_overrides:
+  - gpu_class: a10g           # pin a class to a version you have tested
+    engines: { vllm: "0.29.0" }
+  - labels: { role: gguf }    # nodes meant for llama.cpp alone
+    engines: { vllm: "" }     # none
+```
+
+A node told nothing it can run of an engine joins without it: `llmgw doctor`
+and `node add` warn, and the node serves what it does run. To keep an engine
+off a node for good, whatever the control plane says, add the node with
+`llmgw node add --disable-engine vllm`, which writes `engines.disabled: [vllm]`
+into its `agent.yaml` and leaves vLLM out of doctor's checks. That is narrower
+than `--ignore-checks`, which passes over every other problem as well.
+
+A llama.cpp CUDA build is checked for the GPU it is meant for: installing it
+asks it which devices it sees, and a replica placed on GPUs is not ready until
+its process holds memory on each of them. A build missing a CUDA library fails
+there, naming the library, rather than serving on the CPU.
 
 After changing a version, check it still takes the flags llmgw builds for it,
 which drift between releases:
@@ -671,6 +754,74 @@ llmgw ctl explain-placement qwen2.5-coder-3b-instruct
 which names every candidate and why each was rejected, rather than leaving you
 to infer it.
 
+### LoRA adapters
+
+A llama.cpp model can carry LoRA adapters, each asked for by its own name as
+though it were a model:
+
+```yaml
+- name: qwen2.5-coder-7b-instruct
+  source: http://mirror:8081/qwen2.5-coder-7b-instruct/5d567d26/model.gguf
+  adapters:
+    - name: coder-sql
+      source: http://mirror:8081/coder-sql/v1/adapter.gguf
+      scale: 1.0          # optional; 1 when left out
+  profiles: [...]
+```
+
+Every replica of the model fetches every adapter and loads it at start, applied
+to nothing. A request for `coder-sql` goes to the model's replicas with the
+adapter applied to it alone; a request for the model gets the model. Adapter
+names share the namespace of models and aliases, appear in `/v1/models`, and
+follow key allowlists on their own: a key allowed `coder-sql` reaches that
+adapter and not the model under it, while a key allowed the model reaches its
+adapters too. Adding or removing an adapter is a change to the model; running
+replicas load the new set when restarted (`llmgw ctl restart --model M`).
+Besides the mirror page's model card, adapters are attached from the command
+line, which reads the file first and refuses one that is not an adapter for
+this model's architecture:
+
+    llmgw ctl model adapters coder --add coder-sql=http://mirror:8081/sql-lora/v1/sql.gguf@0.8
+    llmgw ctl model adapters coder --remove coder-sql
+
+The adapter file must be a GGUF LoRA made for this model — llama.cpp's
+`convert_lora_to_gguf.py` makes one from a PEFT adapter — and not a model with
+the adapter already merged in, which is what most "LoRA GGUF" repositories hold.
+
+### Models served outside the cluster
+
+A model can be an OpenAI-compatible API elsewhere — a hosted provider, or
+another gateway — rather than replicas here. Nothing is placed for it; the
+router sends its requests on, with the provider's key and the provider's name
+for the model:
+
+```yaml
+- name: cloud-mini
+  source: external
+  external:
+    url: https://api.openai.com          # without /v1
+    model: gpt-4o-mini                   # what the provider calls it
+    api_key_file: /etc/llmgw/secrets/openai.key
+```
+
+Named in another model's `fallbacks`, it is used only when the cluster has no
+capacity left for that model:
+
+```yaml
+- name: qwen2.5-coder-7b-instruct
+  fallbacks: [cloud-mini]
+  ...
+```
+
+The key is read from its file on the control plane, never stored in the
+catalog, and a key rotated in the file is used within seconds. A model whose key
+file cannot be read is not offered. Everything else applies as to any model:
+prices and budgets, rate limits, usage records, and key allowlists.
+
+**Its prompts leave the cluster.** Decide who may send them: a key or project
+whose allowlist leaves the external model out cannot reach it, as a fallback or
+by name. The cluster page and `ctl models` mark it `external`.
+
 ### How many replicas, and how much each one carries
 
 Replica count is changed while the cluster runs, without editing the
@@ -909,88 +1060,165 @@ refuses every request.
 
 ## Upgrading
 
-One binary on every machine, so an upgrade is a copy and a restart. The part
-worth care is the copy, not the order.
+On the control plane:
+
+```
+sudo llmgw ctl upgrade v0.10.4
+```
+
+It upgrades this machine first — fetching the release for its platform,
+checking it against the release's checksums, keeping the old binary as
+`/usr/local/bin/llmgw.bak-pre-<version>`, restarting `llmgw-server` and waiting
+for every node to reconnect — and then each node in turn. A node's agent is told
+the version over the stream it already holds, fetches and checks the release
+itself, swaps it in and restarts; no ssh into the nodes is needed. Release
+candidates (`v0.10.4-rc.2`) work the same way.
+
+A node counts as done when it is back on the new version, ready, with as many
+ready replicas as before. One that is not within `--timeout` (three minutes)
+stops the upgrade there, so a bad release reaches one node rather than all of
+them; `llmgw ctl logs <node>` says why, and the node keeps its previous binary
+beside the new one. Nodes already on the version are skipped, so running the
+command again resumes where it stopped. `--nodes a,b` limits it to some nodes,
+and `--skip-control-plane` leaves this machine alone.
 
 **Nothing stops serving.** Engines are started with `KillMode=process`, so they
 keep running when the agent restarts, and the agent reports them back on its
 next connection. The control plane holds no replica state across its own
-restart either — it rebuilds the fleet from what the agents report. A rolling
-upgrade of the whole cluster interrupts no request that is not in flight at the
-moment of the restart.
+restart either — it rebuilds the fleet from what the agents report.
 
 **A mixed fleet is fine while you roll.** The agent protocol only ever gains
 fields, and both sides treat an absent one as unknown rather than as a default.
-Order does not matter; finish the roll the same day anyway, because
-`ctl nodes` reports each agent's version and a fleet that disagrees with itself
-is one more thing to hold in your head.
+Agents older than v0.10.4 cannot upgrade themselves: move those by hand once,
+installing the release binary and restarting `llmgw-agent`, and every upgrade
+after that is the command above.
+
+**The mirror is upgraded on its own host**, since it is not an agent and its
+service cannot replace its own binary:
 
 ```
-VERSION=v0.9.0
-# copy to every machine first, install nowhere yet
-for h in 10.0.0.140 10.0.0.141 10.0.0.142 10.0.0.146; do
-  ssh $h 'cat > /tmp/llmgw-new; stat -c %s /tmp/llmgw-new' < llmgw-$VERSION-linux-amd64
-done
+sudo llmgw upgrade v0.10.5 --service llmgw-mirror
 ```
 
-**Check every size before installing anything.** This is not defensive writing:
-on a host whose login shell prints anything, `scp` fails outright, and the
-obvious workaround — `ssh -n host 'cat > file'` — points stdin at `/dev/null`
-and writes a *zero-byte file* while reporting success. Installing that on a
-fleet takes the fleet down. Compare the number on each machine with the local
-one, then:
-
-```
-sudo install -m0755 /tmp/llmgw-new /usr/local/bin/llmgw
-sudo systemctl restart llmgw-agent     # or llmgw-server, or llmgw-mirror
-llmgw --version
-```
+The same checks — the release's checksums, the old binary kept beside the new —
+then a restart of the unit, waiting for it to stay up. A mirror older than
+v0.10.5 does not have the command: install the release binary by hand once.
 
 Then confirm what you have, rather than assuming:
 
 ```
-llmgw ctl nodes       # every agent back, LAST SEEN in seconds
+llmgw ctl nodes       # every agent on the new version, LAST SEEN in seconds
 llmgw ctl replicas    # same replicas, adopted rather than replaced
 ```
 
 A replica may come back marked **`ready (stale profile)`**. That means it is
 running a profile that has since been edited — the reconciler leaves a serving
 replica alone on purpose, because churning one costs more than the difference.
-Restart it when you want the new profile to take effect; the flag is there so
-the choice is yours rather than silent.
+`llmgw ctl restart <replica>` replaces it with one on the new profile, starting
+the new one before stopping the old. Where the old one is all that stands in
+the new one's way — a single GPU — it is stopped first, at once, and the model
+is unserved while the new one loads; where the new profile fits nowhere even
+then, the restart is refused and the old one keeps serving, which `ctl
+replicas` shows as `restart refused:` with the reason. `--in-place` asks for
+stop-then-start whatever room there is.
+
+### Signed releases
+
+Every install above checks a release against its published checksums, which
+proves the download is what the server holds — not that the server holds what
+was built. Signing closes that: CI signs the checksums, and a binary that
+trusts a key installs only releases signed with it. Off until a key exists;
+turning it on, once:
+
+```
+llmgw release keygen
+```
+
+1. Put the **private** key in the repository's Actions secret
+   `RELEASE_SIGNING_KEY`. From the next tag, releases carry
+   `checksums.txt.sig`.
+2. Put the **public** key in `internal/release/trusted.pub` and release. Every
+   binary built from then on carries it and refuses releases that are not
+   signed with it — `ctl upgrade`, agents upgrading themselves, and
+   `llmgw upgrade` alike.
+
+In that order: a build that trusts a key refuses unsigned releases, so the
+secret has to be signing before the key is trusted. The first trusting release
+is itself installed by a binary that trusts nothing; every one after is checked
+by the one before. `LLMGW_RELEASE_KEY` overrides the built-in key, for rotating
+it. The onboarding script fetches with `curl` and checks checksums only; a node
+is checked from its first upgrade on.
 
 ## Backing up
 
-Three things, in descending order of how badly you want them.
+`llmgw server backup` writes one archive holding everything a control plane
+cannot be rebuilt without:
 
-**The certificate authority.** `ca-key.pem`, `/etc/llmgw/pki` unless you kept it elsewhere (step 2).
-Losing it does not stop the cluster, and it means every node identity has to be
-reissued the next time you touch one. Nothing in llmgw can reconstruct it.
+- **the store** — users, API keys, the tenancy tree, limits and budgets, usage
+  history, models added at runtime, measured capacity, and the secret every
+  engine key is derived from (`pg_dump` in its custom format for PostgreSQL, a
+  consistent `VACUUM INTO` snapshot for SQLite; both are safe against a running
+  gateway);
+- **`/etc/llmgw`** — the configuration, the store's address, and the
+  certificate authority's key (`pki/`). Nothing in llmgw can reconstruct that
+  key; losing it means reissuing every node's identity.
 
-**The store.** Users, API keys, organisations, teams, projects, every limit and
-budget, all usage history, models added at runtime, and measured capacity.
+Archives go to `/var/backups/llmgw-server`, readable by root only because of the CA
+key, and the newest 14 are kept. Run it nightly with the timer:
 
 ```
-# SQLite
-sqlite3 /var/lib/llmgw/llmgw.db ".backup '/backup/llmgw-$(date +%F).db'"
-
-# PostgreSQL
-pg_dump --format=custom llmgw > /backup/llmgw-$(date +%F).dump
+sudo install -m 0644 deploy/systemd/llmgw-backup.service deploy/systemd/llmgw-backup.timer /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now llmgw-backup.timer
+sudo systemctl start llmgw-backup.service     # one now, to see it work
+ls -l /var/backups/llmgw-server
 ```
 
-`.backup` and `pg_dump` are both safe against a running gateway. Copying the
-SQLite file with `cp` while it is being written is not.
+The control plane publishes when the newest archive was written as
+`llmgw_backup_last_success_timestamp_seconds`, and **BackupTooOld** fires when
+it is more than 26 hours old — including when there has never been one.
+
+**Copy the archives off the machine.** They sit on the disk they back up and do
+not survive losing it. Anything that copies files will do — rsync from the
+mirror, a NAS mount, an offsite bucket — as long as it keeps the archives
+readable by root only.
 
 **Keys are stored hashed and cannot be recovered from a backup or from
 anywhere else.** Restoring an old store restores which keys exist and who owns
 them, not their secrets. A key created after the backup is gone and has to be
-reissued — which is why the store is worth backing up on a schedule rather than
-before upgrades.
+reissued.
+
+### Restoring
+
+On a machine with llmgw installed (the same version or newer) and, for
+PostgreSQL, an empty database and role as `server init --postgres` would create
+them:
+
+```
+sudo systemctl stop llmgw-server
+mkdir /tmp/restore && sudo tar -xzf /var/backups/llmgw-server/llmgw-backup-<host>-<time>.tar.gz -C /tmp/restore
+cat /tmp/restore/MANIFEST                     # version and time it was taken
+
+# /etc/llmgw: configuration, secrets, certificates and the CA
+sudo cp -a /tmp/restore/etc/llmgw/. /etc/llmgw/
+
+# the store — PostgreSQL
+sudo -u postgres pg_restore --clean --if-exists --no-owner --role=llmgw -d llmgw /tmp/restore/store.pgdump
+# the store — SQLite
+sudo install -o llmgw -g llmgw -m 0600 /tmp/restore/store.sqlite /var/lib/llmgw/llmgw.db
+
+sudo systemctl start llmgw-server
+sudo rm -rf /tmp/restore
+```
+
+Running engines keep serving throughout: agents reconnect to the restored
+control plane and their replicas are adopted, and because the engine-key secret
+came back with the store, the router can still reach them.
 
 **The mirror's directory** is the one thing you can rebuild, by downloading it
-all again. Worth remembering that a pinned version may no longer be upstream to
-download: that is exactly what pinning it protected you from, and a backup is
-what makes the protection outlive the disk.
+all again. A pinned version may no longer be upstream to download: that is
+exactly what pinning it protected you from, and a copy of the directory is what
+makes the protection outlive the disk.
 
 ## Reclaiming disk
 
@@ -1074,11 +1302,12 @@ requests. On a slow model, set `timeouts.generation` higher.
   can see each other, seeding between them is the obvious next thing to build.
 - **A budget without a price never binds.** Set `default_price` or `pricing`
   (step 2) or every request costs zero, and a budget is something the dashboard
-  displays rather than something that stops anything. The quotas page says so
+  displays rather than something that stops anything. The Limits tab says so
   where it can, which is not the same as being told before you rely on it.
-- **vLLM needs CUDA 13.** The version the agent installs needs driver 580
-  and a Turing GPU or newer. On older hardware, set `engines.vllm` to `""` and
-  install a vLLM built for your CUDA yourself.
+- **A V100 runs vLLM 0.19.1**, not the version the rest of the cluster runs:
+  no build from 0.20 on has Volta kernels, and the node picks the newest that
+  has. llmgw's flags are checked against the current version, so run `llmgw
+  ctl engine-check vllm` on a V100 node once.
 - **Two control planes have not been run for long.** The leader election and
   agent failover work and there is a pattern in `deploy/ha/`, but every cluster
   this has been run on in anger has had one. A single control plane is the
@@ -1089,3 +1318,11 @@ requests. On a slow model, set `timeouts.generation` higher.
   has them. Single-node placement on real GPUs is the well-trodden path; if you
   are the first to run a TP group across real cards, run `llmgw doctor --json`
   on each node first and keep the reports.
+- **Two engine setups are weaker than the rest.** An engine's key is passed
+  in its environment, where only root and its own user can read it, except
+  SGLang's, which takes it only as a flag: other users on the node can see it
+  in the process list. Mount `/proc` with `hidepid=2` on nodes where that
+  matters. vLLM's `launch_mode: ray` starts a Ray head without
+  authentication, so anyone who can reach its port can run code on the
+  node. The dashboard is off, but firewall the port to the other nodes in the
+  replica, or use the default `mp` launch mode, which needs no Ray at all.

docs/onboarding.md

@@ -191,7 +191,8 @@ project or person, the teams and projects with their limits and spend, and every
 member. Only what was spent in that organization's projects is on it; what its
 people do elsewhere is not. The page is read-only — a manager is not an
 administrator of the cluster, and limits and memberships stay with those who
-are.
+are. An administrator following `/org` lands on the same organization's panel
+on the people page instead, where all of that can be changed.
 
 `llmgw keys init` still works for provisioning scripts and is idempotent, but it
 will no longer move a team or project that already exists under another parent;
@@ -259,7 +260,7 @@ Allowlists compose by intersection: a key cannot add a model its team withheld.
 A model's canonical name and any of its aliases name the same thing, so either
 form works.
 
-Budgets are given in whole units, as the quotas page shows them, and what a unit
+Budgets are given in whole units, as the Limits tab shows them, and what a unit
 means is yours to decide. Self-hosted models have no per-token price, so it can
 be GPU-seconds, a notional rate, or one unit per thousand tokens everywhere —
 a price of 1 000 000 in `auth.pricing`, which is in micro-units — and a budget
@@ -409,10 +410,19 @@ page exists.
 opening onto its members and its keys. From there an admin approves people
 (into a project or on their own), adds them to and takes them out of
 organizations, teams and projects, suspends and removes them, revokes keys, and
-adds, renames, moves and removes organizations, teams and projects. `/admin/quotas`
-sets limits on the tree, per member of any part of it, and per person. A searchable list of every account is at the
-bottom. The page works as plain forms; with scripts on, a change updates it in
-place and the projects you had open stay open.
+adds, renames, moves and removes organizations, teams and projects. A
+searchable list of every account is at the bottom. Choosing any organization,
+team, project or person opens a panel beside the tree with its limits, its
+capacity quota and its use over the last week, and its limits can be changed
+there.
+
+The same page has two more tabs, each at the address it had as a page of its
+own: **Limits** (`/admin/quotas`) is every limit in the tree at once — per
+branch, per member of any part of it, and per person — with the capacity quotas;
+**Usage** (`/admin/usage`) is the cluster's use over time by organization, team,
+project or person, and the GPU time each owner held. The page works as plain
+forms; with scripts on, a change updates it in place and the projects you had
+open stay open.
 
 Three things are deliberately not there:
 

docs/quick-install.md

@@ -35,10 +35,12 @@ install it.
 ## 2. Set up the control plane
 
 ```
-sudo llmgw server init --admin you@example.com
+sudo llmgw server init
 ```
 
-This one command sets up everything on this machine, and takes a few minutes:
+It asks for your email and a few choices, then sets up everything on this
+machine, in a few minutes. In a script, give them as flags instead:
+`sudo llmgw server init --admin you@example.com`.
 
 - the service, its certificates and its configuration;
 - your administrator account. **The password is printed once. Copy it now.**
@@ -54,18 +56,33 @@ sudo llmgw server init --admin you@example.com --host llm.example.com
 
 It is safe to run again. It keeps what exists and finishes what is missing.
 
+**The mirror on its own machine**, when it wants a larger disk than the
+control plane has: first on that machine, with llmgw installed as in step 1,
+
+```
+sudo llmgw mirror init --control-plane <control plane address>
+```
+
+then on the control plane `sudo llmgw server init --mirror http://<mirror
+address>:8082`. The mirror serves model files to every node, and takes
+orders only from the control plane. Untick "This machine as a node" in the
+form (or add `--no-node`) to keep models off the control plane; the starter
+model then starts on the first node you add.
+
 **Check:** open `http://<control-plane>:8080/chat`, sign in with the printed
 password, and ask something. The small model answers. Change the password on
 the account page.
 
 ## 3. Add the GPU nodes
 
-On the control plane, for each node:
+On the control plane:
 
 ```
 sudo llmgw node add ubuntu@10.0.0.21
 ```
 
+Several at once: `sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23`.
+
 Replace `ubuntu` with the user you log in to the node as, and `10.0.0.21` with
 its address. That user needs `sudo` without a password on the node.
 
@@ -89,6 +106,13 @@ the same fabric the same name. In a script, give it instead:
 sudo llmgw node add ubuntu@10.0.0.21 --fabric ib-a:mlx5_0,mlx5_1,mlx5_2,mlx5_3
 ```
 
+Once two nodes on a fabric are in, check they reach each other at the link's
+rate before placing anything across them:
+
+```
+llmgw ctl fabric-test gpu01 gpu02
+```
+
 **If it cannot log in.** `sudo` runs SSH as root, with root's keys. To use your
 own keys instead, log in to the control plane with `ssh -A` and keep your
 agent through sudo:

go.mod

@@ -6,7 +6,6 @@ require (
 	connectrpc.com/connect v1.21.0
 	github.com/NVIDIA/go-nvml v0.13.4-0
 	github.com/cespare/xxhash/v2 v2.3.0
-	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/jackc/pgx/v5 v5.11.0
@@ -20,6 +19,11 @@ require (
 	github.com/prometheus/common v0.71.0
 	github.com/spf13/cobra v1.10.2
 	github.com/yuin/goldmark v1.8.6
+	go.opentelemetry.io/otel v1.46.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
+	go.opentelemetry.io/otel/sdk v1.46.0
+	go.opentelemetry.io/otel/trace v1.46.0
+	go.yaml.in/yaml/v3 v3.0.5
 	golang.org/x/crypto v0.57.0
 	golang.org/x/net v0.59.0
 	golang.org/x/sync v0.23.0
@@ -28,9 +32,9 @@ require (
 )
 
 require (
-	github.com/atotto/clipboard v0.1.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/charmbracelet/colorprofile v0.4.1 // indirect
 	github.com/charmbracelet/x/ansi v0.11.6 // indirect
 	github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
@@ -41,13 +45,17 @@ require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/go-logr/logr v1.4.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/knadh/koanf/maps v0.1.2 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
 	github.com/mattn/go-runewidth v0.0.19 // indirect
@@ -63,9 +71,15 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	go.yaml.in/yaml/v3 v3.0.5 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect
+	go.opentelemetry.io/otel/metric v1.46.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
 	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect
+	google.golang.org/grpc v1.83.1 // indirect
 	modernc.org/libc v1.75.7 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.12.1 // indirect

go.sum

@@ -2,16 +2,14 @@ connectrpc.com/connect v1.21.0 h1:LhqSJt7jHf5NJBo9Jq/t/9FjcYAideif0mg+qe2jCUs=
 connectrpc.com/connect v1.21.0/go.mod h1:A2ygJrukXwWy32vkCAAHNVguZrqZ+jeZ9rGRnGR4dN4=
 github.com/NVIDIA/go-nvml v0.13.4-0 h1:o3jp9u2x1R9ShFE3v+Aesp55XOSIQFMJz/VGNUcJaNE=
 github.com/NVIDIA/go-nvml v0.13.4-0/go.mod h1:id63qwpoDWpFXwnwM6psDCSqW4BmNu6mWpr4YeQtPGo=
-github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
-github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/charmbracelet/bubbles v1.0.0 h1:12J8/ak/uCZEMQ6KU7pcfwceyjLlWsDLAxB5fXonfvc=
-github.com/charmbracelet/bubbles v1.0.0/go.mod h1:9d/Zd5GdnauMI5ivUIVisuEm3ave1XwXtD1ckyV6r3E=
 github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
 github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
 github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
@@ -38,14 +36,23 @@ github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
-github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
+github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
 github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
@@ -121,6 +128,24 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4=
 github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc=
+go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM=
+go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8=
+go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o=
+go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI=
+go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM=
+go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE=
+go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4=
+go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c=
+go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI=
+go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk=
+go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
@@ -141,10 +166,20 @@ golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
 golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
 golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI=
+google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA=
+google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y=
+google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ=
 google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
 google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/account/password.go

@@ -15,6 +15,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"runtime"
 	"strings"
 
 	"golang.org/x/crypto/argon2"
@@ -56,7 +57,7 @@ func HashPassword(password string) (string, error) {
 	if _, err := rand.Read(salt); err != nil {
 		return "", err
 	}
-	sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
+	sum := idKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen)
 
 	b64 := base64.RawStdEncoding.EncodeToString
 	return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
@@ -93,7 +94,7 @@ func VerifyPassword(password, encoded string) (bool, error) {
 		return false, ErrBadHash
 	}
 
-	got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
+	got := idKey([]byte(password), salt, time, memory, threads, uint32(len(want)))
 	return subtle.ConstantTimeCompare(got, want) == 1, nil
 }
 
@@ -110,3 +111,16 @@ func init() {
 	}
 	dummyHash = h
 }
+
+// argonSlots bounds how many hashes run at once. Each takes 64 MiB, and the
+// login and signup forms are on the client-facing port: without a bound, a
+// few hundred parallel attempts at random addresses asked for gigabytes at
+// once, and the per-address throttle does not see them — each address is new.
+// Beyond the bound, attempts wait their turn instead.
+var argonSlots = make(chan struct{}, max(2, runtime.NumCPU()/2))
+
+func idKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte {
+	argonSlots <- struct{}{}
+	defer func() { <-argonSlots }()
+	return argon2.IDKey(password, salt, time, memory, threads, keyLen)
+}

internal/account/service.go

@@ -230,6 +230,22 @@ func (s *Service) allowAttempt(email string) bool {
 	defer s.mu.Unlock()
 
 	w, ok := s.attempts[email]
+	if !ok && len(s.attempts) >= maxTrackedAddresses {
+		// Full of addresses nobody has signed in as for a while: forget the
+		// ones whose window has passed. Only expiring, never evicting early —
+		// dropping a live window is how an attacker spraying random addresses
+		// would buy unlimited guesses at the one they care about.
+		for k, v := range s.attempts {
+			if now.Sub(v.since) > s.cfg.AttemptWindow {
+				delete(s.attempts, k)
+			}
+		}
+		if len(s.attempts) >= maxTrackedAddresses {
+			// Still full, so somebody is spraying addresses faster than they
+			// expire. New ones wait; those already tracked are unaffected.
+			return false
+		}
+	}
 	if !ok || now.Sub(w.since) > s.cfg.AttemptWindow {
 		s.attempts[email] = &attemptWindow{count: 1, since: now}
 		return true
@@ -238,6 +254,10 @@ func (s *Service) allowAttempt(email string) bool {
 	return w.count <= s.cfg.MaxAttempts
 }
 
+// maxTrackedAddresses bounds the throttle's memory. Unbounded, it grew by one
+// entry per address tried and shrank only on a successful sign-in.
+const maxTrackedAddresses = 50000
+
 func (s *Service) clearAttempts(email string) {
 	s.mu.Lock()
 	delete(s.attempts, email)

internal/account/throttle_bound_test.go

@@ -0,0 +1,33 @@
+package account
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestSprayingAddressesCannotGrowTheThrottleOrEscapeIt(t *testing.T) {
+	now := time.Unix(0, 0)
+	s := &Service{cfg: DefaultConfig(), attempts: map[string]*attemptWindow{}, now: func() time.Time { return now }}
+
+	// The target is being guessed at.
+	for i := 0; i < s.cfg.MaxAttempts; i++ {
+		s.allowAttempt("victim@example.com")
+	}
+	// Then the table is flooded with made-up addresses.
+	for i := 0; i < maxTrackedAddresses+100; i++ {
+		s.allowAttempt(fmt.Sprintf("x%d@example.com", i))
+	}
+	if n := len(s.attempts); n > maxTrackedAddresses {
+		t.Errorf("the table grew to %d", n)
+	}
+	if s.allowAttempt("victim@example.com") {
+		t.Error("flooding the table reset the target's throttle")
+	}
+
+	// Once the window passes, everything expires and sign-ins work again.
+	now = now.Add(s.cfg.AttemptWindow + time.Second)
+	if !s.allowAttempt("new@example.com") {
+		t.Error("a new address was refused after the flood expired")
+	}
+}

internal/accounting/accounting.go

@@ -316,6 +316,17 @@ func (a *Accountant) Record(res Reservation, u store.Usage) {
 	}
 }
 
+// RecordRefused queues the row of a request a limit refused: nothing is
+// charged — it used nothing — but it is counted, so a team running into its
+// limits shows as one rather than as a team that went quiet.
+func (a *Accountant) RecordRefused(u store.Usage) {
+	if u.KeyID == "" || a.recorder == nil {
+		return
+	}
+	u.PromptTokens, u.CompletionTokens, u.CostMicros = 0, 0, 0
+	a.recorder.Record(u)
+}
+
 // Status returns what a key has used, for headers and `ctl`.
 func (a *Accountant) Status(p auth.Principal) (spent int64, rpm, tpm float64) {
 	if p.KeyID == "" {
@@ -383,3 +394,39 @@ func keyCounter(keyID string) string {
 // they are asking is whether the budgets they are about to display mean
 // anything, and the accountant is what would enforce them.
 func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
+
+// Headroom is the tightest of a principal's request and token limits: how
+// much is left of each, and when the minute it is counted over rolls on.
+type Headroom struct {
+	RequestLimit, RequestsLeft int
+	TokenLimit, TokensLeft     int
+	Reset                      time.Duration
+}
+
+// Headroom reports what a principal has left, after whatever Check just
+// charged. Of several levels the one with least left is the one that will
+// refuse first, so that is the one reported. A limit of 0 means none is set.
+func (a *Accountant) Headroom(p auth.Principal) Headroom {
+	var h Headroom
+	firstReq, firstTok := true, true
+	for _, l := range levels(p.Limits, p.KeyID) {
+		if l.RPM != nil && *l.RPM > 0 {
+			left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
+			if firstReq || left < h.RequestsLeft {
+				h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
+			}
+		}
+		if l.TPM != nil && *l.TPM > 0 {
+			left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
+			if firstTok || left < h.TokensLeft {
+				h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
+			}
+		}
+	}
+	// The windows slide, so room comes back continuously; the next minute
+	// boundary is when a whole window's worth has moved on, and is the figure
+	// a client pacing itself can act on.
+	now := time.Now()
+	h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
+	return h
+}

internal/accounting/accounting_test.go

@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
 		t.Error("no price table at all is not priced")
 	}
 }
+
+func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
+	// A project allowing 100 a minute under a team allowing 3: the team is
+	// what will refuse, so the team is what a client is told about.
+	a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
+	p := principal("k1", func(e *store.Effective) {
+		e.Chain = []store.Limits{
+			{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
+			{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
+		}
+	})
+	if _, err := a.Check(p, "m", 10); err != nil {
+		t.Fatal(err)
+	}
+	h := a.Headroom(p)
+	if h.RequestLimit != 3 || h.RequestsLeft != 2 {
+		t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
+	}
+	if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
+		t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
+	}
+	if h.Reset <= 0 || h.Reset > time.Minute {
+		t.Errorf("reset %v", h.Reset)
+	}
+	if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
+		t.Errorf("a key with no limits advertised some: %+v", none)
+	}
+}

internal/agent/agent_test.go

@@ -23,6 +23,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
@@ -399,11 +400,16 @@ func TestHeartbeatKeepsNodeAlive(t *testing.T) {
 	n, _ := h.inv.Get("n1")
 	first := n.LastSeen
 
-	time.Sleep(200 * time.Millisecond)
-	n, _ = h.inv.Get("n1")
-	if !n.LastSeen.After(first) {
-		t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
+	// Waited for rather than slept through: four heartbeat intervals was not
+	// always enough on a loaded runner for one to have landed.
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		if n, _ = h.inv.Get("n1"); n.LastSeen.After(first) {
+			return
+		}
+		time.Sleep(20 * time.Millisecond)
 	}
+	t.Error("heartbeats are not refreshing LastSeen; the node would be declared lost")
 }
 
 func TestStartAndStopShardEndToEnd(t *testing.T) {
@@ -881,3 +887,34 @@ func TestAFabricScanIsAskedForAndAnswered(t *testing.T) {
 		t.Errorf("asked %q, got %+v", asked, r)
 	}
 }
+
+func TestABandwidthTestIsRunOnTheAdapterAskedFor(t *testing.T) {
+	oldRun, oldHave := runWriteBW, haveWriteBW
+	t.Cleanup(func() { runWriteBW, haveWriteBW = oldRun, oldHave })
+	haveWriteBW = func() bool { return true }
+	var got discovery.WriteBW
+	runWriteBW = func(_ context.Context, w discovery.WriteBW) (float64, string, error) {
+		got = w
+		return 388.2, " #bytes  #iterations  BW peak[Gb/sec]  BW average[Gb/sec]\n", nil
+	}
+	h := start(t, func(c *Config) { c.Discovery.IBRoot = ibTreeWithActivePorts(t, "mlx5_0") })
+
+	r, err := h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{
+		Role: "client", Hca: "mlx5_0", Port: 1, Peer: "10.0.0.2", TcpPort: 18515, Seconds: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if r.GetGbps() != 388.2 || r.GetError() != "" {
+		t.Errorf("reply %+v", r)
+	}
+	if got.HCA != "mlx5_0" || got.Peer != "10.0.0.2" || got.TCPPort != 18515 || got.GIDIndex != -1 {
+		t.Errorf("ran %+v", got)
+	}
+
+	// An adapter the node does not have is said so, without running anything.
+	got = discovery.WriteBW{}
+	r, _ = h.svc.FabricTest(t.Context(), "n1", &llmgwv1.FabricTestRequest{Role: "server", Hca: "mlx5_9", Port: 1, Seconds: 1})
+	if !strings.Contains(r.GetError(), "no port mlx5_9/1") || got.HCA != "" {
+		t.Errorf("reply %+v, ran %+v", r, got)
+	}
+}

internal/agent/config.go

@@ -0,0 +1,286 @@
+package agent
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+// Config configures the agent.
+type Config struct {
+	// NodeID defaults to the hostname. It must equal the common name of the
+	// agent's certificate, so set it only where the two differ.
+	NodeID string `koanf:"node_id"`
+	// Server is the control plane, and Servers is the same thing when there is
+	// more than one. Both are accepted: every configuration written before
+	// there could be a second control plane names one address, and none of
+	// them should have to change.
+	//
+	// The order is tried in order, and a refusal moves to the next. Only the
+	// leader accepts an agent, so a standby is skipped in the time it takes to
+	// be told so — which is what makes failover a reconnect rather than an
+	// edit to every node's configuration.
+	Server  string   `koanf:"server"`
+	Servers []string `koanf:"servers"`
+	// Hostname defaults to the machine's.
+	Hostname string `koanf:"hostname"`
+	// MgmtAddr is the address engines are reached on. It defaults to this
+	// machine's address on the route to the control plane, which is the one
+	// the control plane and the router can reach.
+	MgmtAddr string `koanf:"mgmt_addr"`
+	// Labels are free-form facts about a node for a profile's node_selector.
+	// Nothing needs them: memory, GPUs, engines and disk are measured, and a
+	// label restating one of those goes stale the day the machine changes.
+	Labels map[string]string `koanf:"labels"`
+
+	// MgmtInterface is the ethernet interface NCCL bootstraps over. It
+	// defaults to the interface holding MgmtAddr, then eth0.
+	MgmtInterface string `koanf:"mgmt_interface"`
+	// Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's
+	// default subnet prefix is the same on every fabric, so nothing can be
+	// inferred from the wire (plan §5.2).
+	Fabrics []inventory.FabricAttachment `koanf:"fabrics"`
+
+	HeartbeatInterval time.Duration `koanf:"heartbeat_interval"`
+	// ReconnectBase is the first reconnect delay; it backs off from there.
+	ReconnectBase time.Duration `koanf:"reconnect_base"`
+	ReconnectMax  time.Duration `koanf:"reconnect_max"`
+
+	// AdoptReadyTimeout bounds how long an adopted engine is given to answer a
+	// health check before the agent stops waiting on it.
+	//
+	// Shorter than a startup timeout on purpose: this engine is already
+	// running, so it is not loading a model, it is either serving or it is not.
+	AdoptReadyTimeout time.Duration `koanf:"adopt_ready_timeout"`
+
+	// TLS authenticates both ends of the control-plane stream. Leaving it unset
+	// keeps the plaintext behaviour, which only belongs on a trusted network.
+	TLS tlsconf.Config `koanf:"tls"`
+
+	// ModelDirs are the directories this node keeps model weights in, beyond
+	// WeightsCache, which is always looked in. Storage differs at every installation,
+	// so llmgw observes it rather than owning it: a Lustre mount, an image
+	// exported over the fabric, local NVMe and a copy made by hand all look
+	// identical from here, and placement only needs to know whether the file
+	// is present.
+	ModelDirs []string `koanf:"model_dirs"`
+
+	// EnginesDir is where the agent installs engines, one directory per
+	// build. Defaults to /opt/llmgw/engines.
+	EnginesDir string `koanf:"engines_dir"`
+
+	// WeightsCache is where a remote source is downloaded to. Defaults to the
+	// first entry of ModelDirs, because a node that says where its models live
+	// has already answered this question, and otherwise to
+	// /var/lib/llmgw-weights.
+	//
+	// Fetching is the simple arrangement: every node keeps its own copy. The
+	// exotic ones need no code — weights on a shared filer or an image exported
+	// over the fabric are simply present, and present means nothing is fetched.
+	WeightsCache string `koanf:"weights_cache"`
+
+	Discovery discovery.Options `koanf:"discovery"`
+	Engines   engines.Config    `koanf:"engines"`
+	Superv    supervisor.Config `koanf:"supervisor"`
+
+	Version string
+}
+
+// DefaultConfig returns sensible defaults.
+func DefaultConfig() Config {
+	// No MgmtInterface: normalize finds the one holding the management
+	// address, and falls back to eth0 only when it cannot. Defaulted here, it
+	// was never found — every node said eth0 unless told otherwise, and a node
+	// on ens3 or bond0 had NCCL bootstrap over an interface it does not have.
+	return Config{
+		HeartbeatInterval: 5 * time.Second,
+		ReconnectBase:     time.Second,
+		ReconnectMax:      30 * time.Second,
+		AdoptReadyTimeout: 60 * time.Second,
+		Superv:            supervisor.DefaultConfig(),
+		// NVML gives NVLink topology, so prefer it when this build has it.
+		Discovery: discovery.Options{PreferNVML: true},
+	}
+}
+
+func (c *Config) normalize() error {
+	d := DefaultConfig()
+	if err := c.TLS.Validate(); err != nil {
+		return err
+	}
+	if err := c.normalizeServerURL(); err != nil {
+		return err
+	}
+	// Everything below is found rather than written. A file that has to say
+	// which machine it is on, and at what address, is a file that differs on
+	// every node — and a copy taken from one node to the next registers the
+	// second as the first.
+	host, _ := os.Hostname()
+	if c.NodeID == "" {
+		c.NodeID = host
+	}
+	if c.NodeID == "" {
+		return errors.New("node_id is not set and the hostname could not be read")
+	}
+	if c.Hostname == "" {
+		c.Hostname = host
+	}
+	if c.MgmtAddr == "" {
+		c.MgmtAddr = addrToward(c.Server)
+	}
+	if c.MgmtInterface == "" {
+		c.MgmtInterface = interfaceOf(c.MgmtAddr)
+	}
+	if c.MgmtInterface == "" {
+		c.MgmtInterface = "eth0"
+	}
+	// The directory downloads land in is always one that is looked in. When
+	// they were two settings, a node could fill its disk with fetched weights
+	// that placement never saw — which is what every node here was doing.
+	if c.WeightsCache == "" && len(c.ModelDirs) > 0 {
+		c.WeightsCache = c.ModelDirs[0]
+	}
+	if c.WeightsCache == "" {
+		c.WeightsCache = DefaultWeightsCache
+	}
+	if !slices.Contains(c.ModelDirs, c.WeightsCache) {
+		c.ModelDirs = append(c.ModelDirs, c.WeightsCache)
+	}
+	if c.HeartbeatInterval <= 0 {
+		c.HeartbeatInterval = d.HeartbeatInterval
+	}
+	if c.ReconnectBase <= 0 {
+		c.ReconnectBase = d.ReconnectBase
+	}
+	if c.AdoptReadyTimeout <= 0 {
+		c.AdoptReadyTimeout = d.AdoptReadyTimeout
+	}
+	if c.ReconnectMax < c.ReconnectBase {
+		c.ReconnectMax = max(c.ReconnectBase, d.ReconnectMax)
+	}
+	return nil
+}
+
+// normalizeServerURL makes the configured address a URL connect can use, and
+// refuses the combination that would quietly undo the certificates: a scheme
+// that says plaintext while TLS is configured. Silently connecting anyway would
+// mean a cluster believing it has mutual authentication when it has none.
+func (c *Config) normalizeServerURL() error {
+	// One list, however it was written. Server first when both are given,
+	// because an operator adding a standby to an existing file expects the
+	// one already there to stay preferred.
+	all := make([]string, 0, 1+len(c.Servers))
+	if c.Server != "" {
+		all = append(all, c.Server)
+	}
+	all = append(all, c.Servers...)
+
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(all))
+	for _, s := range all {
+		fixed, err := normalizeOne(s, c.TLS.Enabled())
+		if err != nil {
+			return err
+		}
+		// A duplicate would be tried twice in a row on failover, which reads
+		// as the rotation not working.
+		if _, dup := seen[fixed]; dup {
+			continue
+		}
+		seen[fixed] = struct{}{}
+		out = append(out, fixed)
+	}
+	if len(out) == 0 {
+		return errors.New("server is required")
+	}
+	c.Servers = out
+	c.Server = out[0]
+	return nil
+}
+
+// DefaultWeightsCache is where fetched weights go when nothing says otherwise.
+const DefaultWeightsCache = "/var/lib/llmgw-weights"
+
+// addrToward is this machine's address on the route to the control plane.
+//
+// The address engines are reached on is whichever one the control plane can
+// reach, and the kernel already knows which that is: a UDP "connection" picks
+// the source address for the route without sending a packet. Asking the
+// interfaces instead would have to guess among docker bridges, VPNs and
+// link-local addresses.
+func addrToward(server string) string {
+	u, err := url.Parse(server)
+	if err != nil || u.Host == "" {
+		return ""
+	}
+	host := u.Hostname()
+	port := u.Port()
+	if port == "" {
+		port = "443"
+	}
+	conn, err := net.Dial("udp", net.JoinHostPort(host, port))
+	if err != nil {
+		return ""
+	}
+	defer conn.Close()
+	if a, ok := conn.LocalAddr().(*net.UDPAddr); ok {
+		return a.IP.String()
+	}
+	return ""
+}
+
+// interfaceOf names the interface that holds an address, or "" when none does.
+func interfaceOf(addr string) string {
+	ip := net.ParseIP(addr)
+	if ip == nil {
+		return ""
+	}
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return ""
+	}
+	for _, ifc := range ifaces {
+		addrs, err := ifc.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, a := range addrs {
+			if n, ok := a.(*net.IPNet); ok && n.IP.Equal(ip) {
+				return ifc.Name
+			}
+		}
+	}
+	return ""
+}
+
+// normalizeOne applies the scheme rules to a single address.
+func normalizeOne(addr string, secure bool) (string, error) {
+	switch {
+	case strings.HasPrefix(addr, "https://"):
+		if !secure {
+			return "", fmt.Errorf("%s is an https:// address but no tls certificates are configured", addr)
+		}
+	case strings.HasPrefix(addr, "http://"):
+		if secure {
+			return "", fmt.Errorf("tls is configured but %s is an http:// address; use https://", addr)
+		}
+	case strings.Contains(addr, "://"):
+		return "", fmt.Errorf("server %q must be an http:// or https:// address", addr)
+	case secure:
+		return "https://" + addr, nil
+	default:
+		return "http://" + addr, nil
+	}
+	return addr, nil
+}

internal/agent/controlplanes.go

@@ -0,0 +1,77 @@
+package agent
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"slices"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// controlPlanesFile keeps the control planes the leader last named, beside
+// the agent's configuration rather than in it: the configuration is the
+// operator's, and a list rewritten in it would be a file nobody wrote.
+var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json"
+
+// servers is every address to dial, in the order to try them: the
+// configuration's, then any the leader named that it lacks.
+func (a *Agent) servers() []string {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	if len(a.serverList) == 0 {
+		return a.cfg.Servers
+	}
+	return a.serverList
+}
+
+// mergeServers puts the leader's list after the configured one, without
+// repeats, and says whether that changed anything.
+func mergeServers(configured, told []string, secure bool) []string {
+	out := slices.Clone(configured)
+	for _, s := range told {
+		fixed, err := normalizeOne(s, secure)
+		if err != nil {
+			continue
+		}
+		if !slices.Contains(out, fixed) {
+			out = append(out, fixed)
+		}
+	}
+	return out
+}
+
+// setControlPlanes takes the leader's list: used from the next reconnect,
+// and kept for the next start.
+func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) {
+	told := m.GetServers()
+	merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled())
+	a.mu.Lock()
+	changed := !slices.Equal(merged, a.serverList)
+	a.serverList = merged
+	a.mu.Unlock()
+	if !changed {
+		return
+	}
+	a.log.Info("the control planes to dial", "servers", merged)
+	b, _ := json.Marshal(told)
+	if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil {
+		tmp := controlPlanesFile + ".new"
+		if err := os.WriteFile(tmp, b, 0o644); err == nil {
+			_ = os.Rename(tmp, controlPlanesFile)
+		}
+	}
+}
+
+// loadControlPlanes is the list kept from the last time the leader named it.
+func loadControlPlanes(configured []string, secure bool) []string {
+	b, err := os.ReadFile(controlPlanesFile)
+	if err != nil {
+		return configured
+	}
+	var told []string
+	if json.Unmarshal(b, &told) != nil {
+		return configured
+	}
+	return mergeServers(configured, told, secure)
+}

internal/agent/controlplanes_test.go

@@ -0,0 +1,41 @@
+package agent
+
+import (
+	"log/slog"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) {
+	old := controlPlanesFile
+	controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json")
+	t.Cleanup(func() { controlPlanesFile = old })
+
+	configured := []string{"https://10.0.0.140:7443"}
+	// TLS on, as every onboarded node has it: an agent without it refuses
+	// https:// addresses, and the leader's are https://.
+	a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)}
+	if got := a.servers(); !slices.Equal(got, configured) {
+		t.Fatalf("before anything is named: %v", got)
+	}
+	// The configured one stays first; the new one is added after it, once.
+	a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}})
+	want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}
+	if got := a.servers(); !slices.Equal(got, want) {
+		t.Errorf("after the leader named two: %v", got)
+	}
+	// And after a restart, before the leader says anything.
+	if got := loadControlPlanes(configured, true); !slices.Equal(got, want) {
+		t.Errorf("after a restart: %v", got)
+	}
+	// A list that drops the configured one does not drop it: the
+	// configuration is the operator's.
+	a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}})
+	if got := a.servers(); got[0] != "https://10.0.0.140:7443" {
+		t.Errorf("the configured server was dropped: %v", got)
+	}
+}

internal/agent/disabled_test.go

@@ -0,0 +1,32 @@
+package agent
+
+import (
+	"testing"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+func TestAnEngineDisabledOnTheNodeIsNeitherInstalledNorAdvertised(t *testing.T) {
+	a := &Agent{}
+	a.cfg.Engines.Disabled = []string{"vllm"}
+	a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{
+		{Engine: "llamacpp", Version: "b1"}, {Engine: "vllm", Version: "0.30.0"},
+	}})
+	if len(a.provWants) != 1 || a.provWants[0].Engine != "llamacpp" {
+		t.Errorf("wants %+v: vLLM is disabled here and must not be installed", a.provWants)
+	}
+	a.configuredVers = map[string]string{"vllm": "0.25.0"}
+	a.installedVers = map[string]string{"llamacpp": "b1"}
+	if got := a.engineReportLocked(); got["vllm"] != "" || got["llamacpp"] != "b1" {
+		t.Errorf("advertised %v: a disabled engine found on disk is still not offered", got)
+	}
+}
+
+func TestAnEngineNamedByHandIsNotInstalledOverIt(t *testing.T) {
+	a := &Agent{reg: engines.NewRegistry(engines.Config{LlamaCpp: engines.LlamaCppConfig{Bin: "/opt/own/llama-server"}})}
+	a.setProvision(&llmgwv1.Provision{Engines: []*llmgwv1.EngineWant{{Engine: "llamacpp", Version: "b1"}}})
+	if len(a.provWants) != 0 {
+		t.Errorf("wants %+v: an installed build would be used over the operator's own", a.provWants)
+	}
+}

internal/agent/discovery/gpu_smi.go

@@ -106,9 +106,27 @@ func parseSMI(out string) ([]GPU, error) {
 	return gpus, nil
 }
 
-// residentMemoryToleranceMB is how much allocated memory is written off as
+// ForeignMemoryToleranceMB is how much allocated memory is written off as
 // driver overhead rather than a foreign process.
-const residentMemoryToleranceMB = 512
+const ForeignMemoryToleranceMB = 512
+
+// foreignMemorySuffix ends the problem a card with another tenant on it is
+// kept out of placement for. One phrase, so that the control plane can find
+// it again and drop it when the tenant leaves.
+const foreignMemorySuffix = " MiB already allocated by another process"
+
+// ForeignMemoryProblem is the problem for mb of memory held by something
+// that is not one of this agent's engines, or "" when that is too little to
+// count.
+func ForeignMemoryProblem(mb int) string {
+	if mb <= ForeignMemoryToleranceMB {
+		return ""
+	}
+	return fmt.Sprintf("%d%s", mb, foreignMemorySuffix)
+}
+
+// IsForeignMemoryProblem reports whether a problem is ForeignMemoryProblem's.
+func IsForeignMemoryProblem(p string) bool { return strings.HasSuffix(p, foreignMemorySuffix) }
 
 // assessHealth flags conditions that should keep a GPU out of placement.
 func assessHealth(g *GPU) {
@@ -119,8 +137,12 @@ func assessHealth(g *GPU) {
 	// Memory already in use means something else is on the card. Starting an
 	// engine there would either fail to allocate or quietly evict the other
 	// process's throughput.
-	if g.MemoryUsedMB > residentMemoryToleranceMB {
-		g.Health.Problems = append(g.Health.Problems,
-			fmt.Sprintf("%d MiB already allocated by another process", g.MemoryUsedMB))
+	//
+	// Only a first reading: this runs before the agent knows which processes
+	// are its own, so an engine still running from before a restart reads as
+	// foreign here. Every heartbeat measures it again, without the agent's
+	// own engines, and the control plane replaces this with that.
+	if p := ForeignMemoryProblem(g.MemoryUsedMB); p != "" {
+		g.Health.Problems = append(g.Health.Problems, p)
 	}
 }

internal/agent/discovery/gpuhealth.go

@@ -0,0 +1,280 @@
+package discovery
+
+import (
+	"context"
+	"encoding/xml"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// GPUWear is what wears a card out or takes it down, beyond temperature:
+// read every minute, from nvidia-smi and the kernel's log. The early signs of
+// a card failing are here — a growing count of rows its memory had to remap,
+// PCIe link retries, XID errors — long before a replica on it fails.
+//
+// A count of -1 is not reported: a card without row remapping (before
+// Ampere) has no such count, which is not the same as none.
+type GPUWear struct {
+	Index int
+	// PCIeReplays are link-level retries on the card's PCIe link: a few are
+	// noise, a count that keeps climbing is a riser, slot or cable.
+	PCIeReplays int64
+	// RetiredSBE and RetiredDBE are memory pages retired for single and
+	// double bit errors (cards before Ampere); RetirePending is a page
+	// waiting for a reset to be retired.
+	RetiredSBE, RetiredDBE int64
+	RetirePending          bool
+	// RemappedCorrectable and RemappedUncorrectable are memory rows remapped
+	// (Ampere and later). RemapPending waits for a reset; RemapFailure is a
+	// card out of spare rows — it needs replacing.
+	RemappedCorrectable, RemappedUncorrectable int64
+	RemapPending, RemapFailure                 bool
+	// NVLink errors summed over the card's links, by kind.
+	NVLinkCRC, NVLinkReplay, NVLinkRecovery int64
+	// XIDs are the driver's error events for this card since the agent
+	// started, and the last of them.
+	XIDs        int64
+	LastXID     int
+	LastXIDText string
+	LastXIDAt   time.Time
+}
+
+// healthEvery is how often health is read: it changes over hours, and the
+// kernel log and nvidia-smi's full report are the heavier reads.
+const healthEvery = time.Minute
+
+// HealthSampler reads GPUWear, keeping the XID counts between reads.
+type HealthSampler struct {
+	run func(ctx context.Context, name string, args ...string) ([]byte, error)
+	now func() time.Time
+
+	mu      sync.Mutex
+	at      time.Time
+	last    []GPUWear
+	since   time.Time
+	xids    map[int]*GPUWear
+	started bool
+}
+
+// NewHealthSampler reads the machine's own nvidia-smi and kernel log.
+func NewHealthSampler() *HealthSampler {
+	return &HealthSampler{now: time.Now, xids: map[int]*GPUWear{},
+		run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
+			return exec.CommandContext(ctx, name, args...).Output() //nolint:gosec // fixed tools
+		}}
+}
+
+// Sample is the cards' health, read again once healthEvery has passed. Nil
+// on a machine without nvidia-smi.
+func (h *HealthSampler) Sample(ctx context.Context) []GPUWear {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	now := h.now()
+	if !h.at.IsZero() && now.Sub(h.at) < healthEvery {
+		return h.last
+	}
+	h.at = now
+	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+	defer cancel()
+	report, err := h.run(ctx, "nvidia-smi", "-q", "-x")
+	if err != nil {
+		h.last = nil
+		return nil
+	}
+	cards, bus := parseHealthReport(report)
+	if out, err := h.run(ctx, "nvidia-smi", "nvlink", "-e"); err == nil {
+		for i, e := range parseNVLinkErrors(string(out)) {
+			if i < len(cards) {
+				cards[i].NVLinkCRC, cards[i].NVLinkReplay, cards[i].NVLinkRecovery = e[0], e[1], e[2]
+			}
+		}
+	}
+	h.readXIDs(ctx, now, bus)
+	for i := range cards {
+		if x := h.xids[i]; x != nil {
+			cards[i].XIDs, cards[i].LastXID, cards[i].LastXIDText, cards[i].LastXIDAt = x.XIDs, x.LastXID, x.LastXIDText, x.LastXIDAt
+		}
+	}
+	h.last = cards
+	return cards
+}
+
+// readXIDs adds the XID events logged since the last read. The first read
+// starts from now: what happened before the agent started was before, and
+// counting the whole log would raise every alert on every restart.
+func (h *HealthSampler) readXIDs(ctx context.Context, now time.Time, bus map[string]int) {
+	if !h.started {
+		h.started, h.since = true, now
+		return
+	}
+	since := h.since
+	h.since = now
+	out, err := h.run(ctx, "journalctl", "-k", "--no-pager", "-o", "short-unix",
+		"--since", "@"+strconv.FormatInt(since.Unix(), 10), "-g", "NVRM: Xid")
+	if err != nil {
+		// No journal, or nothing matched (journalctl exits 1 for that).
+		return
+	}
+	for _, e := range parseXIDs(string(out)) {
+		i, ok := bus[e.bus]
+		if !ok {
+			continue
+		}
+		x := h.xids[i]
+		if x == nil {
+			x = &GPUWear{Index: i}
+			h.xids[i] = x
+		}
+		x.XIDs++
+		x.LastXID, x.LastXIDText = e.code, e.text
+		x.LastXIDAt = now
+		if e.at > 0 {
+			x.LastXIDAt = time.Unix(int64(e.at), 0)
+		}
+	}
+}
+
+// healthLog is the part of nvidia-smi's XML report health is read from.
+type healthLog struct {
+	GPUs []struct {
+		ID  string `xml:"id,attr"`
+		PCI struct {
+			Replay string `xml:"replay_counter"`
+		} `xml:"pci"`
+		Retired struct {
+			SBE     string `xml:"multiple_single_bit_retirement>retired_count"`
+			DBE     string `xml:"double_bit_retirement>retired_count"`
+			Pending string `xml:"pending_retirement"`
+			// Older drivers name it pending_blacklist.
+			Blacklist string `xml:"pending_blacklist"`
+		} `xml:"retired_pages"`
+		Remapped struct {
+			Corr    string `xml:"remapped_row_corr"`
+			Unc     string `xml:"remapped_row_unc"`
+			Pending string `xml:"remapped_row_pending"`
+			Failure string `xml:"remapped_row_failure"`
+		} `xml:"remapped_rows"`
+	} `xml:"gpu"`
+}
+
+// parseHealthReport reads each card's health, in index order, and maps each
+// card's PCI bus ("3b:00", as the kernel names it) to its index.
+func parseHealthReport(report []byte) ([]GPUWear, map[string]int) {
+	var log healthLog
+	if err := xml.Unmarshal(report, &log); err != nil {
+		return nil, nil
+	}
+	bus := map[string]int{}
+	out := make([]GPUWear, 0, len(log.GPUs))
+	for i, g := range log.GPUs {
+		bus[busOf(g.ID)] = i
+		out = append(out, GPUWear{
+			Index:                 i,
+			PCIeReplays:           count(g.PCI.Replay),
+			RetiredSBE:            count(g.Retired.SBE),
+			RetiredDBE:            count(g.Retired.DBE),
+			RetirePending:         yes(g.Retired.Pending) || yes(g.Retired.Blacklist),
+			RemappedCorrectable:   count(g.Remapped.Corr),
+			RemappedUncorrectable: count(g.Remapped.Unc),
+			RemapPending:          yes(g.Remapped.Pending),
+			RemapFailure:          yes(g.Remapped.Failure),
+		})
+	}
+	return out, bus
+}
+
+// busOf is a PCI address as the kernel's XID lines carry it: nvidia-smi
+// says "00000000:3B:00.0", the kernel "0000:3b:00".
+func busOf(id string) string {
+	id = strings.ToLower(strings.TrimSpace(id))
+	if i := strings.LastIndex(id, "."); i >= 0 {
+		id = id[:i]
+	}
+	parts := strings.Split(id, ":")
+	if len(parts) >= 2 {
+		return parts[len(parts)-2] + ":" + parts[len(parts)-1]
+	}
+	return id
+}
+
+func count(s string) int64 {
+	n, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64)
+	if err != nil {
+		return -1
+	}
+	return n
+}
+
+func yes(s string) bool { return strings.EqualFold(strings.TrimSpace(s), "yes") }
+
+// nvlinkError matches a line of `nvidia-smi nvlink -e`, whose wording varies
+// by driver: "Link 0: Replay Errors: 0", "Link 0: CRC Errors: 0",
+// "Link 3: Data CRC Errors: 2".
+var (
+	nvlinkGPU   = regexp.MustCompile(`^GPU (\d+):`)
+	nvlinkError = regexp.MustCompile(`Link \d+:\s*(.*?)\s*Errors?\s*:\s*(\d+)`)
+)
+
+// parseNVLinkErrors sums each card's link errors by kind: CRC, replay,
+// recovery.
+func parseNVLinkErrors(out string) map[int][3]int64 {
+	res := map[int][3]int64{}
+	gpu := -1
+	for _, line := range strings.Split(out, "\n") {
+		if m := nvlinkGPU.FindStringSubmatch(strings.TrimSpace(line)); m != nil {
+			gpu, _ = strconv.Atoi(m[1])
+			continue
+		}
+		m := nvlinkError.FindStringSubmatch(line)
+		if m == nil || gpu < 0 {
+			continue
+		}
+		n, _ := strconv.ParseInt(m[2], 10, 64)
+		e := res[gpu]
+		kind := strings.ToLower(m[1])
+		switch {
+		case strings.Contains(kind, "crc"):
+			e[0] += n
+		case strings.Contains(kind, "replay"):
+			e[1] += n
+		case strings.Contains(kind, "recovery"):
+			e[2] += n
+		}
+		res[gpu] = e
+	}
+	return res
+}
+
+type xidEvent struct {
+	bus  string
+	code int
+	text string
+	at   float64
+}
+
+// xidLine matches the driver's report of an XID:
+// "NVRM: Xid (PCI:0000:3b:00): 79, pid=..., GPU has fallen off the bus."
+var xidLine = regexp.MustCompile(`NVRM: Xid \(PCI:([0-9a-fA-F:.]+)\): (\d+),\s*(.*)$`)
+
+// parseXIDs reads journalctl's short-unix lines: a timestamp, then the
+// message.
+func parseXIDs(out string) []xidEvent {
+	var evs []xidEvent
+	for _, line := range strings.Split(out, "\n") {
+		m := xidLine.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		code, _ := strconv.Atoi(m[2])
+		e := xidEvent{bus: busOf(m[1]), code: code, text: strings.TrimSpace(m[3])}
+		if f := strings.Fields(line); len(f) > 0 {
+			e.at, _ = strconv.ParseFloat(f[0], 64)
+		}
+		evs = append(evs, e)
+	}
+	return evs
+}

internal/agent/discovery/gpuhealth_test.go

@@ -0,0 +1,101 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+)
+
+const healthXML = `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <gpu id="00000000:3B:00.0">
+    <pci><replay_counter>0</replay_counter></pci>
+    <retired_pages>
+      <multiple_single_bit_retirement><retired_count>N/A</retired_count></multiple_single_bit_retirement>
+      <double_bit_retirement><retired_count>N/A</retired_count></double_bit_retirement>
+      <pending_retirement>N/A</pending_retirement>
+    </retired_pages>
+    <remapped_rows>
+      <remapped_row_corr>2</remapped_row_corr>
+      <remapped_row_unc>1</remapped_row_unc>
+      <remapped_row_pending>Yes</remapped_row_pending>
+      <remapped_row_failure>No</remapped_row_failure>
+    </remapped_rows>
+  </gpu>
+  <gpu id="00000000:5E:00.0">
+    <pci><replay_counter>4021</replay_counter></pci>
+    <retired_pages>
+      <multiple_single_bit_retirement><retired_count>3</retired_count></multiple_single_bit_retirement>
+      <double_bit_retirement><retired_count>1</retired_count></double_bit_retirement>
+      <pending_blacklist>Yes</pending_blacklist>
+    </retired_pages>
+    <remapped_rows>N/A</remapped_rows>
+  </gpu>
+</nvidia_smi_log>`
+
+const nvlinkOut = `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-aaa)
+	 Link 0: Replay Errors: 1
+	 Link 0: Recovery Errors: 0
+	 Link 0: CRC Errors: 5
+	 Link 1: Data CRC Errors: 2
+GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-bbb)
+	 Link 0: Replay Errors: 0
+`
+
+func TestAGPUsWearAndFailuresAreReadAndCountedFromWhenTheAgentStarted(t *testing.T) {
+	now := time.Unix(1_800_000_000, 0)
+	journal := ""
+	h := &HealthSampler{now: func() time.Time { return now }, xids: map[int]*GPUWear{},
+		run: func(_ context.Context, name string, args ...string) ([]byte, error) {
+			switch {
+			case name == "nvidia-smi" && args[0] == "-q":
+				return []byte(healthXML), nil
+			case name == "nvidia-smi" && args[0] == "nvlink":
+				return []byte(nvlinkOut), nil
+			case name == "journalctl":
+				if journal == "" {
+					return nil, errors.New("exit status 1")
+				}
+				return []byte(journal), nil
+			}
+			return nil, errors.New("unexpected " + name)
+		}}
+	got := h.Sample(t.Context())
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	a, b := got[0], got[1]
+	if a.RemappedCorrectable != 2 || a.RemappedUncorrectable != 1 || !a.RemapPending || a.RemapFailure ||
+		a.RetiredSBE != -1 || a.PCIeReplays != 0 || a.NVLinkCRC != 7 || a.NVLinkReplay != 1 {
+		t.Errorf("card 0 %+v", a)
+	}
+	if b.PCIeReplays != 4021 || b.RetiredSBE != 3 || b.RetiredDBE != 1 || !b.RetirePending || b.RemappedCorrectable != -1 {
+		t.Errorf("card 1 %+v", b)
+	}
+
+	// An XID on card 1, logged after the agent started; read on the next
+	// sample a minute on.
+	journal = "1800000030.123456 gpu07 kernel: NVRM: Xid (PCI:0000:5e:00): 79, pid=1234, name=llama-server, GPU has fallen off the bus.\n"
+	now = now.Add(61 * time.Second)
+	got = h.Sample(t.Context())
+	if got[1].XIDs != 1 || got[1].LastXID != 79 || !strings.Contains(got[1].LastXIDText, "fallen off the bus") ||
+		got[1].LastXIDAt.Unix() != 1_800_000_030 || got[0].XIDs != 0 {
+		t.Errorf("after an XID: %+v / %+v", got[0], got[1])
+	}
+	// Within the minute, the last reading.
+	journal = ""
+	now = now.Add(10 * time.Second)
+	if again := h.Sample(t.Context()); again[1].XIDs != 1 {
+		t.Errorf("%+v", again[1])
+	}
+}
+
+func TestBusIDsAreReadAsTheKernelWritesThem(t *testing.T) {
+	for in, want := range map[string]string{"00000000:3B:00.0": "3b:00", "0000:3b:00": "3b:00", "0000:5e:00.0": "5e:00"} {
+		if got := busOf(in); got != want {
+			t.Errorf("%s: %s, want %s", in, got, want)
+		}
+	}
+}

internal/agent/discovery/gpuprocs.go

@@ -0,0 +1,120 @@
+package discovery
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+// GPUProcesses reports which processes hold memory on each GPU, by the GPU's
+// nvidia-smi index: what a process actually computes on, as opposed to what it
+// was told to use. A llama.cpp started with --n-gpu-layers on a GPU whose
+// backend did not load runs on the CPU without a word, and this is where that
+// shows — the process is on no GPU's list.
+//
+// From the full query rather than --query-compute-apps, which lists compute
+// processes only: llama.cpp's Vulkan build is listed as a graphics one, and
+// would have read as not on its GPU at all.
+//
+// A node with no nvidia-smi returns nil and no error, as discovery does.
+func GPUProcesses(ctx context.Context, binary string) (map[int][]int, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseGPUProcesses(out)
+}
+
+// GPUMemory is one card's memory as nvidia-smi reports it at a moment: what
+// is allocated on it, and which processes hold how much of that.
+type GPUMemory struct {
+	UsedMB int
+	Procs  []GPUProcess
+}
+
+// GPUProcess is one process holding memory on a card.
+type GPUProcess struct {
+	PID    int
+	UsedMB int
+}
+
+// GPUMemoryByProcess reports each GPU's memory, by index, with the processes
+// holding it — from the same report as GPUProcesses, so the two never
+// disagree about who is on a card.
+//
+// A node with no nvidia-smi returns nil and no error, as discovery does.
+func GPUMemoryByProcess(ctx context.Context, binary string) ([]GPUMemory, error) {
+	if binary == "" {
+		if _, err := exec.LookPath("nvidia-smi"); err != nil {
+			return nil, nil
+		}
+		binary = "nvidia-smi"
+	}
+	out, err := exec.CommandContext(ctx, binary, "-q", "-x").Output()
+	if err != nil {
+		return nil, fmt.Errorf("running %s: %w", binary, err)
+	}
+	return parseGPUMemory(out)
+}
+
+// smiLog is the part of nvidia-smi's XML report this reads: each GPU, in
+// index order, its memory and the processes on it.
+type smiLog struct {
+	GPUs []struct {
+		Used      string `xml:"fb_memory_usage>used"`
+		Processes []struct {
+			PID    string `xml:"pid"`
+			Type   string `xml:"type"`
+			UsedMB string `xml:"used_memory"`
+		} `xml:"processes>process_info"`
+	} `xml:"gpu"`
+}
+
+// parseGPUProcesses reads the report. GPUs are listed in index order, which
+// is nvidia-smi's own numbering — the one placement assigns by.
+func parseGPUProcesses(report []byte) (map[int][]int, error) {
+	mem, err := parseGPUMemory(report)
+	if err != nil {
+		return nil, err
+	}
+	out := map[int][]int{}
+	for i, g := range mem {
+		for _, p := range g.Procs {
+			out[i] = append(out[i], p.PID)
+		}
+	}
+	return out, nil
+}
+
+// parseGPUMemory reads the report into each GPU's memory and its holders.
+func parseGPUMemory(report []byte) ([]GPUMemory, error) {
+	var log smiLog
+	if err := xml.Unmarshal(report, &log); err != nil {
+		return nil, fmt.Errorf("reading nvidia-smi's report: %w", err)
+	}
+	out := make([]GPUMemory, len(log.GPUs))
+	for i, g := range log.GPUs {
+		out[i].UsedMB = mib(g.Used)
+		for _, p := range g.Processes {
+			if pid, err := strconv.Atoi(strings.TrimSpace(p.PID)); err == nil {
+				out[i].Procs = append(out[i].Procs, GPUProcess{PID: pid, UsedMB: mib(p.UsedMB)})
+			}
+		}
+	}
+	return out, nil
+}
+
+// mib reads "4932 MiB"; anything else, "N/A" included, is zero.
+func mib(s string) int {
+	n, _ := strconv.Atoi(strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(s), "MiB")))
+	return n
+}

internal/agent/discovery/gpuprocs_test.go

@@ -0,0 +1,78 @@
+package discovery
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGPUProcessesOfEveryKindAreListedByIndex(t *testing.T) {
+	report := `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <driver_version>580.65.06</driver_version>
+  <gpu id="00000000:17:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
+    <processes></processes>
+  </gpu>
+  <gpu id="00000000:65:00.0"><product_name>Tesla V100-SXM2-32GB</product_name>
+    <processes>
+      <process_info><pid>4100</pid><type>C</type><process_name>llama-server</process_name></process_info>
+      <process_info><pid>4300</pid><type>G</type><process_name>llama-server</process_name></process_info>
+    </processes>
+  </gpu>
+  <gpu id="00000000:B3:00.0"><processes>
+      <process_info><pid>4200</pid><type>C+G</type></process_info>
+  </processes></gpu>
+</nvidia_smi_log>`
+	got, err := parseGPUProcesses([]byte(report))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// A Vulkan engine is a graphics process ("G") and is on its GPU all the
+	// same; a compute-only query would have missed it.
+	want := map[int][]int{1: {4100, 4300}, 2: {4200}}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %v, want %v", got, want)
+	}
+	if _, err := parseGPUProcesses([]byte("not xml")); err == nil {
+		t.Error("a report that does not parse was read as no processes")
+	}
+}
+
+func TestGPUMemoryIsReadPerProcess(t *testing.T) {
+	report := `<?xml version="1.0" ?>
+<nvidia_smi_log>
+  <gpu id="00000000:01:00.0"><product_name>Quadro RTX 6000</product_name>
+    <fb_memory_usage><total>24576 MiB</total><used>14514 MiB</used><free>10062 MiB</free></fb_memory_usage>
+    <processes>
+      <process_info><pid>71315</pid><type>C</type><used_memory>4932 MiB</used_memory></process_info>
+      <process_info><pid>4100</pid><type>C</type><used_memory>9500 MiB</used_memory></process_info>
+    </processes>
+  </gpu>
+  <gpu id="00000000:02:00.0">
+    <fb_memory_usage><used>N/A</used></fb_memory_usage>
+    <processes></processes>
+  </gpu>
+</nvidia_smi_log>`
+	got, err := parseGPUMemory([]byte(report))
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := []GPUMemory{
+		{UsedMB: 14514, Procs: []GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 4100, UsedMB: 9500}}},
+		{},
+	}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got %+v, want %+v", got, want)
+	}
+}
+
+func TestForeignMemoryBelowTheToleranceIsNoProblem(t *testing.T) {
+	// A CUDA context that has let go of its model still holds a few hundred
+	// MiB: ComfyUI after /free held 324.
+	if p := ForeignMemoryProblem(330); p != "" {
+		t.Errorf("330 MiB is a problem: %q", p)
+	}
+	p := ForeignMemoryProblem(5014)
+	if !IsForeignMemoryProblem(p) || IsForeignMemoryProblem("2 uncorrected ECC errors") {
+		t.Errorf("%q is not recognised as the foreign memory problem, or ECC is", p)
+	}
+}

internal/agent/discovery/perftest.go

@@ -0,0 +1,133 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// WriteBW is one side of an ib_write_bw run, from perftest: an RDMA write
+// bandwidth test between two adapters, which is what a collective spanning
+// nodes rides on.
+type WriteBW struct {
+	HCA  string
+	Port int
+	// GIDIndex is the RoCE v2 GID to use; negative on InfiniBand, where the
+	// port's LID is used.
+	GIDIndex int
+	// Peer is the server's address, for the client; empty makes the server.
+	Peer    string
+	TCPPort int
+	Seconds int
+}
+
+// Args are the command line: a duration rather than a count, so the test
+// takes as long on a slow link as a fast one, and results in Gb/s.
+func (w WriteBW) Args() []string {
+	args := []string{"-d", w.HCA, "-i", strconv.Itoa(max(w.Port, 1)),
+		"-p", strconv.Itoa(w.TCPPort), "-D", strconv.Itoa(max(w.Seconds, 1)),
+		"-F", "--report_gbits"}
+	if w.GIDIndex >= 0 {
+		args = append(args, "-x", strconv.Itoa(w.GIDIndex))
+	}
+	if w.Peer != "" {
+		args = append(args, w.Peer)
+	}
+	return args
+}
+
+// RunWriteBW runs one side and returns the average bandwidth the output
+// reports, in Gb/s, and the output. The client retries its connection for a
+// while: the server is started first, but a few seconds may pass before it
+// listens.
+func RunWriteBW(ctx context.Context, w WriteBW) (float64, string, error) {
+	bin, err := exec.LookPath("ib_write_bw")
+	if err != nil {
+		return 0, "", errors.New("ib_write_bw is not installed (the perftest package)")
+	}
+	limit := time.Duration(w.Seconds+60) * time.Second
+	var out []byte
+	for attempt := 0; ; attempt++ {
+		cctx, cancel := context.WithTimeout(ctx, limit)
+		out, err = exec.CommandContext(cctx, bin, w.Args()...).CombinedOutput() //nolint:gosec // fixed tool
+		cancel()
+		if err == nil || w.Peer == "" || !refused(string(out)) || attempt >= 10 {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return 0, string(out), ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	text := string(out)
+	if err != nil {
+		return 0, text, fmt.Errorf("ib_write_bw: %v: %s", err, lastNonEmpty(text))
+	}
+	gbps, perr := ParseWriteBW(text)
+	if perr != nil && w.Peer != "" {
+		// Only the client is sure to print the result; a server that does not
+		// is not a failure.
+		return 0, text, perr
+	}
+	return gbps, text, nil
+}
+
+func refused(out string) bool {
+	for _, s := range []string{"Couldn't connect", "Unable to init the socket connection", "Connection refused"} {
+		if strings.Contains(out, s) {
+			return true
+		}
+	}
+	return false
+}
+
+var columnSplit = regexp.MustCompile(`\s{2,}`)
+
+// ParseWriteBW finds the average bandwidth in perftest's result table:
+//
+//	#bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
+//	65536      887036           0.00               96.93               0.184877
+//
+// by the heading's position rather than a fixed column, since versions add
+// columns and the peak reads 0.00 when the test runs for a duration.
+func ParseWriteBW(out string) (float64, error) {
+	lines := strings.Split(out, "\n")
+	for i, l := range lines {
+		if !strings.Contains(l, "BW average") {
+			continue
+		}
+		heads := columnSplit.Split(strings.TrimSpace(l), -1)
+		col := -1
+		for j, h := range heads {
+			if strings.HasPrefix(h, "BW average") {
+				col = j
+			}
+		}
+		for _, d := range lines[i+1:] {
+			f := strings.Fields(d)
+			if len(f) == 0 || strings.HasPrefix(f[0], "-") {
+				continue
+			}
+			if len(f) != len(heads) || col < 0 {
+				return 0, fmt.Errorf("perftest's result table has a shape this does not know: %q / %q", l, d)
+			}
+			v, err := strconv.ParseFloat(f[col], 64)
+			if err != nil {
+				return 0, fmt.Errorf("reading the average bandwidth %q: %w", f[col], err)
+			}
+			return v, nil
+		}
+	}
+	return 0, errors.New("perftest printed no result")
+}
+
+func lastNonEmpty(s string) string {
+	lines := strings.Split(strings.TrimSpace(s), "\n")
+	return strings.TrimSpace(lines[len(lines)-1])
+}

internal/agent/discovery/perftest_test.go

@@ -0,0 +1,49 @@
+package discovery
+
+import (
+	"slices"
+	"testing"
+)
+
+// The client's output of perftest 24.x, with a duration.
+const writeBWClient = `---------------------------------------------------------------------------------------
+                    RDMA_Write BW Test
+ Dual-port       : OFF		Device         : mlx5_0
+ Number of qps   : 1		Transport type : IB
+ Connection type : RC		Using SRQ      : OFF
+ TX depth        : 128
+ CQ Moderation   : 100
+ Mtu             : 4096[B]
+ Link type       : IB
+ Max inline data : 0[B]
+ rdma_cm QPs	 : OFF
+ Data ex. method : Ethernet
+---------------------------------------------------------------------------------------
+ local address: LID 0x05 QPN 0x0112 PSN 0x7b1f02 RKey 0x1fffbd VAddr 0x007f1d2a5f0000
+ remote address: LID 0x09 QPN 0x0113 PSN 0x2c1e44 RKey 0x1ffeaf VAddr 0x007fa4e1a60000
+---------------------------------------------------------------------------------------
+ #bytes     #iterations    BW peak[Gb/sec]    BW average[Gb/sec]   MsgRate[Mpps]
+ 65536      1851214          0.00               388.21		   0.740422
+---------------------------------------------------------------------------------------
+`
+
+func TestTheAverageBandwidthIsReadFromPerftest(t *testing.T) {
+	got, err := ParseWriteBW(writeBWClient)
+	if err != nil || got != 388.21 {
+		t.Errorf("got %v, %v", got, err)
+	}
+	if _, err := ParseWriteBW(" Couldn't connect to 10.0.0.2:18515\n"); err == nil {
+		t.Error("output with no result table gave a number")
+	}
+}
+
+func TestWriteBWArguments(t *testing.T) {
+	ib := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: -1, TCPPort: 18515, Seconds: 3}.Args()
+	if slices.Contains(ib, "-x") {
+		t.Errorf("InfiniBand was given a GID index: %v", ib)
+	}
+	roce := WriteBW{HCA: "mlx5_0", Port: 1, GIDIndex: 3, Peer: "10.0.0.2", TCPPort: 18515, Seconds: 3}.Args()
+	if !slices.Contains(roce, "-x") || roce[len(roce)-1] != "10.0.0.2" {
+		t.Errorf("RoCE client: %v", roce)
+	}
+}

internal/agent/discovery/sensors.go

@@ -0,0 +1,287 @@
+package discovery
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Sensors is what a node's own hardware says about heat and power: the CPUs,
+// the board where it has an inlet sensor, and each GPU. A zero is not
+// measured, never a reading of zero — a VM has none of these, and must not
+// look like a machine running at 0 °C on no power.
+type Sensors struct {
+	// CPUTempC is the hottest CPU sensor: the package, or the hottest core
+	// where no package reading exists.
+	CPUTempC float64
+	// CPUPowerW is the CPU packages' draw, from RAPL: the change in their
+	// energy counters over the time between two samples.
+	CPUPowerW float64
+	// InletTempC is the air coming in, where the board has a sensor it calls
+	// that. Most do not, and a CPU's temperature is not a substitute: it
+	// follows the load, not the room.
+	InletTempC float64
+	GPUs       []GPUSensor
+}
+
+// GPUSensor is one card's heat, power and whether it is slowing itself down.
+type GPUSensor struct {
+	Index       int
+	TempC       float64
+	MemoryTempC float64
+	PowerW      float64
+	PowerLimitW float64
+	// Throttle names what is holding the card's clocks down now:
+	// sw_power_cap, hw_slowdown, sw_thermal, hw_thermal, power_brake. Empty
+	// is running free. Idle and application clock settings are left out;
+	// they are not the card being held back.
+	Throttle []string
+	// How busy it is, read with the rest every sample. Discovery reads the
+	// same once, at start, which made a busy card look as idle as it was
+	// when the agent came up. -1 is not reported by the driver; zero
+	// MemoryTotalMB is a sample without these at all.
+	UtilPercent, MemUtilPercent float64
+	MemoryUsedMB, MemoryTotalMB float64
+	SMClockMHz                  float64
+	ECCUncorrected              float64
+}
+
+// sensorEvery bounds how often nvidia-smi is run. Heartbeats come more often
+// than temperatures change, and each run costs a few hundred milliseconds.
+const sensorEvery = 10 * time.Second
+
+// SensorSampler reads Sensors. The first sample has no CPU power, which needs
+// two readings of the energy counters.
+type SensorSampler struct {
+	sys string // /sys, or a test's copy of it
+	smi func(ctx context.Context, fields string) ([]byte, error)
+	now func() time.Time
+
+	mu         sync.Mutex
+	last       Sensors
+	lastAt     time.Time
+	prevEnergy map[string]uint64
+	prevAt     time.Time
+}
+
+// NewSensorSampler reads this machine.
+func NewSensorSampler() *SensorSampler {
+	return &SensorSampler{sys: "/sys", smi: runSMI, now: time.Now}
+}
+
+func runSMI(ctx context.Context, fields string) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	//nolint:gosec // fixed binary, fields from this file
+	return exec.CommandContext(ctx, "nvidia-smi", "--query-gpu="+fields,
+		"--format=csv,noheader,nounits").Output()
+}
+
+// Sample reads the sensors, or returns the last reading when that is recent.
+func (s *SensorSampler) Sample(ctx context.Context) Sensors {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	now := s.now()
+	if !s.lastAt.IsZero() && now.Sub(s.lastAt) < sensorEvery {
+		return s.last
+	}
+	out := Sensors{}
+	out.CPUTempC, out.InletTempC = s.hwmon()
+	out.CPUPowerW = s.rapl(now)
+	out.GPUs = s.gpus(ctx)
+	s.last, s.lastAt = out, now
+	return out
+}
+
+// hwmon reads the hottest CPU sensor and an inlet sensor, from every hwmon
+// device the kernel exposes — what lm_sensors reads, without it installed.
+func (s *SensorSampler) hwmon() (cpu, inlet float64) {
+	devs, _ := filepath.Glob(filepath.Join(s.sys, "class/hwmon/hwmon*"))
+	var pkg, core float64
+	for _, d := range devs {
+		name := readTrim(filepath.Join(d, "name"))
+		inputs, _ := filepath.Glob(filepath.Join(d, "temp*_input"))
+		for _, in := range inputs {
+			milli, err := strconv.ParseFloat(readTrim(in), 64)
+			if err != nil || milli <= 0 {
+				continue
+			}
+			c := milli / 1000
+			label := strings.ToLower(readTrim(strings.TrimSuffix(in, "_input") + "_label"))
+			switch {
+			case isCPUSensor(name) && (strings.HasPrefix(label, "package") || label == "tctl" || label == "tdie"):
+				pkg = max(pkg, c)
+			case isCPUSensor(name):
+				core = max(core, c)
+			case label == "cpu":
+				// A board's own reading of the CPU, as Dell's SMM gives.
+				core = max(core, c)
+			case strings.Contains(label, "inlet") || strings.Contains(label, "ambient") ||
+				strings.Contains(label, "intake") || label == "systin":
+				inlet = max(inlet, c)
+			}
+		}
+	}
+	if pkg > 0 {
+		return pkg, inlet
+	}
+	return core, inlet
+}
+
+func isCPUSensor(name string) bool {
+	switch name {
+	case "coretemp", "k10temp", "zenpower", "cpu_thermal":
+		return true
+	}
+	return false
+}
+
+// rapl is the CPU packages' power from their energy counters. Package
+// domains only: the dram and core subdomains are parts of a package, and
+// adding them would count the same energy twice.
+func (s *SensorSampler) rapl(now time.Time) float64 {
+	doms, _ := filepath.Glob(filepath.Join(s.sys, "class/powercap/intel-rapl:*"))
+	energy := map[string]uint64{}
+	wrap := map[string]uint64{}
+	for _, d := range doms {
+		if strings.Count(filepath.Base(d), ":") != 1 {
+			continue // intel-rapl:0:0 is a subdomain of intel-rapl:0
+		}
+		e, err := strconv.ParseUint(readTrim(filepath.Join(d, "energy_uj")), 10, 64)
+		if err != nil {
+			continue // unreadable without root on newer kernels
+		}
+		energy[d] = e
+		wrap[d], _ = strconv.ParseUint(readTrim(filepath.Join(d, "max_energy_range_uj")), 10, 64)
+	}
+	prev, prevAt := s.prevEnergy, s.prevAt
+	s.prevEnergy, s.prevAt = energy, now
+	dt := now.Sub(prevAt).Seconds()
+	if prev == nil || dt <= 0 || len(energy) == 0 {
+		return 0
+	}
+	var joules float64
+	for d, e := range energy {
+		p, ok := prev[d]
+		if !ok {
+			return 0 // a domain appeared: no fair difference this time
+		}
+		if e < p { // the counter wrapped
+			e += wrap[d]
+		}
+		joules += float64(e-p) / 1e6
+	}
+	return joules / dt
+}
+
+// The throttle reasons nvidia-smi reports, as bits. Only those that mean the
+// card is being held back are named.
+var throttleBits = []struct {
+	bit  uint64
+	name string
+}{
+	{0x4, "sw_power_cap"},
+	{0x8, "hw_slowdown"},
+	{0x20, "sw_thermal"},
+	{0x40, "hw_thermal"},
+	{0x80, "power_brake"},
+}
+
+// gpus asks nvidia-smi for each card's heat and power. The memory temperature
+// is not known to every driver, and a field nvidia-smi does not know fails the
+// whole query, so it is asked for first and dropped on failure.
+// gpuFields are what each sample asks nvidia-smi for, in order. The busy
+// figures follow the throttle reasons; the memory temperature is last, being
+// the one field a driver may reject outright.
+const (
+	gpuBase  = "index,temperature.gpu,power.draw,enforced.power.limit,clocks_throttle_reasons.active"
+	gpuBusy  = ",utilization.gpu,utilization.memory,memory.used,memory.total,clocks.sm,ecc.errors.uncorrected.volatile.total"
+	gpuHBMTC = ",temperature.memory"
+)
+
+func (s *SensorSampler) gpus(ctx context.Context) []GPUSensor {
+	// Most fields first, fewer on a driver that refuses one: a query naming
+	// a field the driver does not know fails as a whole.
+	var out []byte
+	var busy bool
+	for _, q := range []struct {
+		fields string
+		busy   bool
+	}{{gpuBase + gpuBusy + gpuHBMTC, true}, {gpuBase + gpuBusy, true}, {gpuBase + gpuHBMTC, false}, {gpuBase, false}} {
+		var err error
+		if out, err = s.smi(ctx, q.fields); err == nil {
+			busy = q.busy
+			break
+		}
+		out = nil
+	}
+	if out == nil {
+		return nil
+	}
+	var gs []GPUSensor
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		f := strings.Split(line, ",")
+		if len(f) < 5 {
+			continue
+		}
+		for i := range f {
+			f[i] = strings.TrimSpace(f[i])
+		}
+		idx, err := strconv.Atoi(f[0])
+		if err != nil {
+			continue
+		}
+		g := GPUSensor{Index: idx, TempC: num(f[1]), PowerW: num(f[2]), PowerLimitW: num(f[3])}
+		if bits, err := strconv.ParseUint(strings.TrimPrefix(f[4], "0x"), 16, 64); err == nil {
+			for _, b := range throttleBits {
+				if bits&b.bit != 0 {
+					g.Throttle = append(g.Throttle, b.name)
+				}
+			}
+		}
+		rest := f[5:]
+		if busy && len(rest) >= 6 {
+			g.UtilPercent, g.MemUtilPercent = numOrUnknown(rest[0]), numOrUnknown(rest[1])
+			g.MemoryUsedMB, g.MemoryTotalMB = num(rest[2]), num(rest[3])
+			g.SMClockMHz, g.ECCUncorrected = num(rest[4]), numOrUnknown(rest[5])
+			rest = rest[6:]
+		}
+		if len(rest) > 0 {
+			g.MemoryTempC = num(rest[0])
+		}
+		gs = append(gs, g)
+	}
+	return gs
+}
+
+// numOrUnknown is a figure, or -1 where nvidia-smi says it has none: a card
+// without ECC has no error count, which is not the same as none.
+func numOrUnknown(s string) float64 {
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil || v < 0 {
+		return -1
+	}
+	return v
+}
+
+// num reads a figure nvidia-smi may give as "[N/A]" or "N/A".
+func num(s string) float64 {
+	v, err := strconv.ParseFloat(s, 64)
+	if err != nil || v < 0 {
+		return 0
+	}
+	return v
+}
+
+func readTrim(path string) string {
+	b, err := os.ReadFile(path) //nolint:gosec // sysfs
+	if err != nil {
+		return ""
+	}
+	return strings.TrimSpace(string(b))
+}

internal/agent/discovery/sensors_test.go

@@ -0,0 +1,210 @@
+package discovery
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func put(t *testing.T, root, rel, body string) {
+	t.Helper()
+	p := filepath.Join(root, rel)
+	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p, []byte(body+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// dellHost is the layout of the Proxmox host these tests were written
+// against: Dell's SMM with a CPU and SODIMM sensors, coretemp with a package
+// and cores, and RAPL with a package and its dram subdomain.
+func dellHost(t *testing.T) string {
+	root := t.TempDir()
+	put(t, root, "class/hwmon/hwmon0/name", "dell_smm")
+	put(t, root, "class/hwmon/hwmon0/temp1_input", "55000")
+	put(t, root, "class/hwmon/hwmon0/temp1_label", "CPU")
+	put(t, root, "class/hwmon/hwmon0/temp2_input", "47000")
+	put(t, root, "class/hwmon/hwmon0/temp2_label", "SODIMM")
+	put(t, root, "class/hwmon/hwmon1/name", "coretemp")
+	put(t, root, "class/hwmon/hwmon1/temp1_input", "58000")
+	put(t, root, "class/hwmon/hwmon1/temp1_label", "Package id 0")
+	put(t, root, "class/hwmon/hwmon1/temp10_input", "61000")
+	put(t, root, "class/hwmon/hwmon1/temp10_label", "Core 8")
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1000000000")
+	put(t, root, "class/powercap/intel-rapl:0/max_energy_range_uj", "262143328850")
+	put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "5000000")
+	return root
+}
+
+func sampler(sys string, smi func(context.Context, string) ([]byte, error), clock *time.Time) *SensorSampler {
+	return &SensorSampler{sys: sys, smi: smi, now: func() time.Time { return *clock }}
+}
+
+func noSMI(context.Context, string) ([]byte, error) { return nil, errors.New("not found") }
+
+func TestAHostsCPUHeatAndPowerAreReadAsLmSensorsWould(t *testing.T) {
+	root := dellHost(t)
+	now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	s := sampler(root, noSMI, &now)
+
+	first := s.Sample(t.Context())
+	// The package reading, not the hotter core: the package is what the
+	// chip itself reports as its temperature.
+	if first.CPUTempC != 58 {
+		t.Errorf("CPU %.1f °C, expected the package's 58", first.CPUTempC)
+	}
+	if first.CPUPowerW != 0 {
+		t.Error("power from one reading of a counter is invented")
+	}
+	if first.InletTempC != 0 || first.GPUs != nil {
+		t.Errorf("sensors this host does not have were reported: %+v", first)
+	}
+
+	// 10 s later the package has used 650 J, and its dram subdomain some of
+	// that — counted once.
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "1650000000")
+	put(t, root, "class/powercap/intel-rapl:0:0/energy_uj", "99000000")
+	now = now.Add(10 * time.Second)
+	if got := s.Sample(t.Context()).CPUPowerW; got < 64.9 || got > 65.1 {
+		t.Errorf("CPU power %.2f W, expected 65", got)
+	}
+}
+
+func TestACounterThatWrapsIsNotANegativeDraw(t *testing.T) {
+	root := dellHost(t)
+	now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "262143000000")
+	s := sampler(root, noSMI, &now)
+	s.Sample(t.Context())
+	// 328,850 µJ to the wrap, then 499,671,150 past it: 500 J in 10 s.
+	put(t, root, "class/powercap/intel-rapl:0/energy_uj", "499671150")
+	now = now.Add(10 * time.Second)
+	if got := s.Sample(t.Context()).CPUPowerW; got < 49.9 || got > 50.1 {
+		t.Errorf("across a wrap: %.2f W, expected 50", got)
+	}
+}
+
+func TestAVMHasNoSensorsAndSaysNothing(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, "class/thermal/cooling_device0/type", "Processor")
+	now := time.Now()
+	got := sampler(root, noSMI, &now).Sample(t.Context())
+	if got.CPUTempC != 0 || got.CPUPowerW != 0 || got.InletTempC != 0 || got.GPUs != nil {
+		t.Errorf("a machine with no sensors reported %+v", got)
+	}
+}
+
+func TestABoardsInletSensorIsTheRoomNotTheCPU(t *testing.T) {
+	root := t.TempDir()
+	put(t, root, "class/hwmon/hwmon2/name", "nct6798")
+	put(t, root, "class/hwmon/hwmon2/temp1_input", "24500")
+	put(t, root, "class/hwmon/hwmon2/temp1_label", "SYSTIN")
+	put(t, root, "class/hwmon/hwmon3/name", "k10temp")
+	put(t, root, "class/hwmon/hwmon3/temp1_input", "71250")
+	put(t, root, "class/hwmon/hwmon3/temp1_label", "Tctl")
+	now := time.Now()
+	got := sampler(root, noSMI, &now).Sample(t.Context())
+	if got.InletTempC != 24.5 || got.CPUTempC != 71.25 {
+		t.Errorf("inlet %.2f, CPU %.2f", got.InletTempC, got.CPUTempC)
+	}
+}
+
+func TestEachGPUSaysHowHotHowHungryAndWhatHoldsItBack(t *testing.T) {
+	var asked []string
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		asked = append(asked, fields)
+		return []byte(
+			"0, 71, 612.45, 700.00, 0x0000000000000000, 84\n" +
+				"1, 88, 698.10, 700.00, 0x0000000000000024, 95\n" + // sw_power_cap + sw_thermal
+				"2, 45, [N/A], 700.00, 0x0000000000000001, [N/A]\n"), nil // idle only
+	}
+	now := time.Now()
+	ss := sampler(t.TempDir(), smi, &now)
+	got := ss.Sample(t.Context())
+	if len(got.GPUs) != 3 {
+		t.Fatalf("%+v", got.GPUs)
+	}
+	g0, g1, g2 := got.GPUs[0], got.GPUs[1], got.GPUs[2]
+	if g0.TempC != 71 || g0.PowerW != 612.45 || g0.PowerLimitW != 700 || g0.MemoryTempC != 84 || g0.Throttle != nil {
+		t.Errorf("card 0 %+v", g0)
+	}
+	if strings.Join(g1.Throttle, ",") != "sw_power_cap,sw_thermal" {
+		t.Errorf("card 1 throttled by %v", g1.Throttle)
+	}
+	if g2.PowerW != 0 || g2.MemoryTempC != 0 || g2.Throttle != nil {
+		t.Errorf("card 2: N/A is not a reading, and idle is not being held back: %+v", g2)
+	}
+	// Within ten seconds the last reading is reused rather than running
+	// nvidia-smi on every heartbeat; after, it is asked again.
+	now = now.Add(5 * time.Second)
+	ss.Sample(t.Context())
+	if len(asked) != 1 {
+		t.Errorf("nvidia-smi was asked %d times within ten seconds", len(asked))
+	}
+	now = now.Add(6 * time.Second)
+	ss.Sample(t.Context())
+	if len(asked) != 2 {
+		t.Errorf("nvidia-smi was asked %d times after eleven seconds, expected 2", len(asked))
+	}
+}
+
+func TestADriverThatDoesNotKnowMemoryTemperatureStillReportsTheRest(t *testing.T) {
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if strings.Contains(fields, "temperature.memory") {
+			return nil, errors.New(`Field "temperature.memory" is not a valid field to query.`)
+		}
+		return []byte("0, 66, 250.0, 300.0, 0x0000000000000040\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context())
+	if len(got.GPUs) != 1 || got.GPUs[0].TempC != 66 || strings.Join(got.GPUs[0].Throttle, ",") != "hw_thermal" {
+		t.Errorf("%+v", got.GPUs)
+	}
+}
+
+func TestEachSampleSaysHowBusyEachGPUIs(t *testing.T) {
+	// What the card is doing now, every sample: discovery read it once, at
+	// start, and a card busy since looked as idle as it was then.
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if !strings.Contains(fields, "utilization.gpu") {
+			return nil, errors.New("asked without how busy the cards are")
+		}
+		return []byte(
+			"0, 71, 612.45, 700.00, 0x0000000000000000, 97, 64, 71234, 81559, 1980, 0, 84\n" +
+				"1, 30, 70.10, 700.00, 0x0000000000000001, 0, 0, 512, 81559, 345, [N/A], [N/A]\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	g0, g1 := got[0], got[1]
+	if g0.UtilPercent != 97 || g0.MemUtilPercent != 64 || g0.MemoryUsedMB != 71234 || g0.MemoryTotalMB != 81559 ||
+		g0.SMClockMHz != 1980 || g0.ECCUncorrected != 0 || g0.MemoryTempC != 84 {
+		t.Errorf("card 0 %+v", g0)
+	}
+	// Idle is zero; a count the card does not keep is unknown, not zero.
+	if g1.UtilPercent != 0 || g1.ECCUncorrected != -1 || g1.MemoryTempC != 0 {
+		t.Errorf("card 1 %+v", g1)
+	}
+}
+
+func TestADriverThatDoesNotKnowTheBusyFieldsStillReportsHeatAndPower(t *testing.T) {
+	smi := func(_ context.Context, fields string) ([]byte, error) {
+		if strings.Contains(fields, "utilization.gpu") {
+			return nil, errors.New(`Field "ecc.errors.uncorrected.volatile.total" is not a valid field to query.`)
+		}
+		return []byte("0, 66, 250.0, 300.0, 0x0000000000000000, 70\n"), nil
+	}
+	now := time.Now()
+	got := sampler(t.TempDir(), smi, &now).Sample(t.Context()).GPUs
+	if len(got) != 1 || got[0].TempC != 66 || got[0].MemoryTempC != 70 || got[0].MemoryTotalMB != 0 {
+		t.Errorf("%+v", got)
+	}
+}

internal/agent/engines/engine.go

@@ -9,6 +9,7 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"slices"
@@ -40,6 +41,9 @@ type Profile struct {
 	ModelPath string
 	// ServedModelName is the name clients use, which need not match the path.
 	ServedModelName string
+	// Adapters are LoRA adapters to load beside the model, in order: an
+	// adapter's index is the id requests apply it by.
+	Adapters []Adapter
 }
 
 // ShardSpec is one shard of one replica on this node.
@@ -70,6 +74,17 @@ type ShardSpec struct {
 	HTTPPort int
 	Profile  Profile
 
+	// APIKey is what the engine demands of every request. Engines listen on
+	// all of a node's addresses; without a key anybody who can reach the port
+	// talks to the model directly, past the gateway's authentication, quotas
+	// and accounting. Given to the engine in its environment rather than on
+	// its command line, which every user on the node can read.
+	APIKey string `json:"api_key,omitempty"`
+	// Threads is the CPU cores placement reserved for a shard with no GPU,
+	// and so what its engine computes on. Zero leaves it to the engine's own
+	// configuration.
+	Threads int `json:"threads,omitempty"`
+
 	// Pool and Fabric are control plane concepts the agent cannot derive. They
 	// are carried so the agent can report them back after a restart; Fabric
 	// also says, with an "nvl:" prefix, that the replica is inside a
@@ -132,6 +147,15 @@ type Engine interface {
 	MetricsPath() string
 }
 
+// CPUFallback is an engine that can come up on the CPU when it was placed on
+// GPUs, and keep serving there without an error. The agent checks such a shard
+// is on its GPUs before calling it ready.
+type CPUFallback interface {
+	FallsBackToCPU() bool
+	// Devices lists the GPUs the engine's build can see, for saying why.
+	Devices(ctx context.Context) ([]string, error)
+}
+
 // GPUInfo is the subset of GPU facts validation needs.
 type GPUInfo struct {
 	Index             int
@@ -189,6 +213,10 @@ func (r *Registry) Names() []string {
 
 // Config configures the engine adapters.
 type Config struct {
+	// Disabled are engines this node never installs or advertises, whatever
+	// the control plane asks: vLLM on a node of cards no build of it runs on,
+	// that is meant for llama.cpp alone.
+	Disabled []string       `koanf:"disabled"`
 	VLLM     VLLMConfig     `koanf:"vllm"`
 	SGLang   SGLangConfig   `koanf:"sglang"`
 	LlamaCpp LlamaCppConfig `koanf:"llamacpp"`
@@ -218,6 +246,11 @@ func NCCLEnv(s ShardSpec) map[string]string { return baseEnv(s) }
 func baseEnv(s ShardSpec) map[string]string {
 	env := map[string]string{
 		"CUDA_VISIBLE_DEVICES": joinInts(s.GPUIndices),
+		// The indices above are nvidia-smi's, which counts by PCI bus. CUDA
+		// counts fastest first unless told otherwise, and on a node with two
+		// kinds of GPU the shard would get a different card than placement
+		// gave it — one another shard may hold.
+		"CUDA_DEVICE_ORDER": "PCI_BUS_ID",
 	}
 
 	if len(s.HCAs) > 0 {
@@ -249,6 +282,11 @@ func baseEnv(s ShardSpec) map[string]string {
 	// Profile overrides win: they are the escape hatch for the per-cluster
 	// tuning no adapter can guess.
 	for k, v := range s.Profile.Env {
+		// Checked again here, not only where the model was defined: this is
+		// the process about to run as root, whatever sent the profile.
+		if bad, _ := UnsafeEnv(k); bad {
+			continue
+		}
 		env[k] = v
 	}
 	return env
@@ -357,3 +395,10 @@ func quantizationOf(args []string) string {
 	}
 	return ""
 }
+
+// Adapter is a LoRA adapter an engine loads beside its model. Path is a
+// source until the agent has fetched it, a local file after.
+type Adapter struct {
+	Name string `json:"name"`
+	Path string `json:"path"`
+}

internal/agent/engines/llamacpp.go

@@ -1,9 +1,12 @@
 package engines
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -21,8 +24,21 @@ type LlamaCppConfig struct {
 	// does not say. llama.cpp defaults to one, which would make every second
 	// request wait behind the first no matter what the router decided.
 	Parallel int `koanf:"parallel"`
+	// Threads is how many CPU threads a shard with no GPU computes on. Zero
+	// takes half this machine's CPUs.
+	//
+	// Not all of them, which is llama.cpp's own default: two busy servers
+	// that each spin on every CPU fight for them, and on an 8-vCPU node
+	// that made 7B generation fall from 10 tokens a second to under 1 each.
+	// At half, one server alone is about a third slower and two together
+	// keep 5.5 each. A node that only ever runs one CPU replica does better
+	// with all of them; say so here.
+	Threads int `koanf:"threads"`
 }
 
+// numCPU is the CPUs this process may run on; a variable for tests.
+var numCPU = runtime.NumCPU
+
 // LlamaCpp builds llama.cpp launch commands.
 //
 // It is the engine that makes a cluster without GPUs useful: llama.cpp runs a
@@ -51,25 +67,25 @@ func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true }
 
 // Validate checks the profile against the GPUs it would run on.
 //
-// The two refusals here are about not pretending: llama.cpp has no equivalent
-// of a tensor-parallel group spanning machines, and its multi-GPU support is a
-// different mechanism (--split-mode, --tensor-split) that this adapter does not
-// drive. Accepting either would produce a replica that starts and then serves
-// from one machine while the catalog believes it has several.
+// llama.cpp spreads a model over a node's GPUs by layer (--split-mode layer),
+// so gpus_per_node is how many it gets. It has no tensor parallelism and no
+// group spanning machines, and accepting tp or nodes would produce a replica
+// that serves from fewer cards, or one machine, while the catalog believes
+// otherwise.
 func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error {
-	if err := validateCommon(p, gpus); err != nil {
-		return err
-	}
 	if p.Nodes > 1 {
 		return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica",
 			p.Name, p.Nodes)
 	}
-	if p.TP > 1 {
-		return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+
-			"it splits a model across local GPUs with --tensor-split, which this adapter does not drive",
-			p.Name)
+	if p.TP > 1 || p.PP > 1 {
+		return fmt.Errorf("profile %q: llamacpp splits a model across a node's GPUs by layer, not by tp or pp; "+
+			"write gpus_per_node: %d instead", p.Name, max(p.TP, 1)*max(p.PP, 1))
 	}
-	return nil
+	// The common checks count a replica's GPUs as tp; for llama.cpp the
+	// count is gpus_per_node.
+	q := p
+	q.TP = max(p.GPUsPerNode, 1)
+	return validateCommon(q, gpus)
 }
 
 // Commands returns the processes for one shard, which is always one process.
@@ -118,10 +134,50 @@ func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) {
 	if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") {
 		args = append(args, "--n-gpu-layers", "999")
 	}
+	// Several GPUs: the layers spread across all of them, which is the only
+	// split llama.cpp does without a matched interconnect. In proportion to
+	// each card's free memory, which for the whole cards placement gives is
+	// an even split.
+	if len(s.GPUIndices) > 1 && !hasFlag(p.Args, "--split-mode", "-sm") {
+		args = append(args, "--split-mode", "layer")
+	}
+	if len(s.GPUIndices) == 0 && !hasFlag(p.Args, "--threads", "-t") {
+		// What placement reserved first: it is what the node's other
+		// replicas were placed around.
+		n := s.Threads
+		if n <= 0 {
+			n = l.cfg.Threads
+		}
+		if n <= 0 {
+			n = max(numCPU()/2, 1)
+		}
+		args = append(args, "--threads", strconv.Itoa(n))
+	}
+	// Loaded at start, applied per request: a request that names an adapter
+	// carries "lora": [{"id": i, "scale": s}], and one that names none is the
+	// base model. Each is loaded at scale zero, not with --lora and
+	// --lora-init-without-apply: that leaves an adapter's scale at 1, and
+	// llama-server applies it at that scale to every request with no "lora"
+	// of its own — every request for the model itself.
+	for _, a := range p.Adapters {
+		args = append(args, "--lora-scaled", a.Path+":0")
+	}
 	args = append(args, p.Args...)
 
+	env := baseEnv(s)
+	if s.APIKey != "" {
+		env["LLAMA_API_KEY"] = s.APIKey
+	}
+	// The Vulkan build — what a GPU older than the CUDA builds gets — does
+	// not read CUDA_VISIBLE_DEVICES; this is its own. Vulkan numbers devices
+	// as the driver lists them, which on NVIDIA follows the same bus order
+	// as nvidia-smi; were it not to, the shard would not hold memory on its
+	// own GPUs, and starting it checks exactly that.
+	if len(s.GPUIndices) > 0 {
+		env["GGML_VK_VISIBLE_DEVICES"] = joinInts(s.GPUIndices)
+	}
 	return []Command{{
-		Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)),
+		Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(env),
 	}}, nil
 }
 
@@ -195,3 +251,38 @@ func (l *LlamaCpp) bin() string {
 	}
 	return "llama-server"
 }
+
+// FallsBackToCPU says that llama.cpp keeps running when its GPU backend does
+// not load: --n-gpu-layers asks for layers on a GPU it then has none of, and
+// the server comes up and answers on the CPU, at a tenth of the speed, with no
+// error anywhere. Its readiness on a GPU placement needs checking separately.
+func (*LlamaCpp) FallsBackToCPU() bool { return true }
+
+// Devices lists the GPUs this build can compute on, as llama-server's
+// --list-devices names them ("CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)").
+// Empty, on a machine with GPUs, is a build whose GPU backend did not load:
+// the CPU build, or a CUDA build missing a library its backend links.
+func (l *LlamaCpp) Devices(ctx context.Context) ([]string, error) {
+	out, err := exec.CommandContext(ctx, l.bin(), "--list-devices").CombinedOutput() //nolint:gosec // the engine's own binary
+	if err != nil {
+		return nil, fmt.Errorf("%s --list-devices: %v: %s", l.bin(), err, strings.TrimSpace(string(out)))
+	}
+	return parseDevices(string(out)), nil
+}
+
+// parseDevices reads --list-devices: the lines after "Available devices:",
+// which says "(none)" when there are none. Log lines before it are skipped.
+func parseDevices(out string) []string {
+	var devs []string
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			devs = append(devs, line)
+		}
+	}
+	return devs
+}

internal/agent/engines/llamacpp_test.go

@@ -245,3 +245,119 @@ func TestAMissingPathIsLeftToTheEngine(t *testing.T) {
 		t.Errorf("expected it passed through, got %q %v", got, err)
 	}
 }
+
+func TestLlamaCppIsGivenItsKeyOutOfSightOfTheProcessList(t *testing.T) {
+	// The command line is readable by every user on the node; the
+	// environment of a root process is not.
+	cmds, err := (&LlamaCpp{}).Commands(ShardSpec{HTTPPort: 8080, APIKey: "k1",
+		Profile: Profile{ModelPath: "/m.gguf"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if slices.Contains(cmds[0].Args, "k1") || slices.Contains(cmds[0].Args, "--api-key") {
+		t.Errorf("the key is on the command line: %v", cmds[0].Args)
+	}
+	if !slices.Contains(cmds[0].Env, "LLAMA_API_KEY=k1") {
+		t.Error("llama-server was not given its key")
+	}
+}
+
+func TestACPUShardLeavesHalfTheMachineForAnother(t *testing.T) {
+	prev := numCPU
+	numCPU = func() int { return 8 }
+	t.Cleanup(func() { numCPU = prev })
+
+	threads := func(l *LlamaCpp, s ShardSpec) (string, bool) {
+		cmds, err := l.Commands(s)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return flagValue(cmds[0].Args, "--threads")
+	}
+	if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "4" {
+		t.Errorf("--threads %q on 8 CPUs, expected 4", v)
+	}
+	if v, _ := threads(&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}, llamaSpec(nil)); v != "8" {
+		t.Errorf("--threads %q with 8 configured", v)
+	}
+	if _, ok := threads(&LlamaCpp{}, llamaSpec([]int{0})); ok {
+		t.Error("a GPU shard was given a thread count")
+	}
+	s := llamaSpec(nil)
+	s.Profile.Args = []string{"-t", "2"}
+	cmds, _ := (&LlamaCpp{}).Commands(s)
+	if slices.Contains(cmds[0].Args, "--threads") {
+		t.Error("the profile's own -t was overridden")
+	}
+	numCPU = func() int { return 1 }
+	if v, _ := threads(&LlamaCpp{}, llamaSpec(nil)); v != "1" {
+		t.Errorf("--threads %q on one CPU", v)
+	}
+}
+
+func TestTheCoresPlacementReservedAreTheThreads(t *testing.T) {
+	s := llamaSpec(nil)
+	s.Threads = 3
+	cmds, err := (&LlamaCpp{cfg: LlamaCppConfig{Threads: 8}}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--threads"); v != "3" {
+		t.Errorf("--threads %q; what placement reserved should win over the node's setting", v)
+	}
+}
+
+func TestAdaptersAreLoadedAtStartAndAppliedToNoneByDefault(t *testing.T) {
+	s := llamaSpec(nil)
+	s.Profile.Adapters = []Adapter{{Name: "sql", Path: "/w/sql.gguf"}, {Name: "docs", Path: "/w/docs.gguf"}}
+	cmds, err := (&LlamaCpp{}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := strings.Join(cmds[0].Args, " ")
+	if !strings.Contains(args, "--lora-scaled /w/sql.gguf:0 --lora-scaled /w/docs.gguf:0") {
+		t.Errorf("args %s: both adapters, in order, loaded at scale zero", args)
+	}
+	plain, _ := (&LlamaCpp{}).Commands(llamaSpec(nil))
+	if strings.Contains(strings.Join(plain[0].Args, " "), "--lora") {
+		t.Error("a model with no adapters was given lora flags")
+	}
+}
+
+func TestDevicesAreReadFromListDevices(t *testing.T) {
+	cuda := "ggml_cuda_init: found 2 CUDA devices:\nAvailable devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n  CUDA1: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := parseDevices(cuda); len(got) != 2 || !strings.HasPrefix(got[0], "CUDA0: NVIDIA L4") {
+		t.Errorf("CUDA build: %q", got)
+	}
+	// A CUDA build whose backend did not load says what the CPU build says.
+	if got := parseDevices("Available devices:\n  (none)\n"); len(got) != 0 {
+		t.Errorf("no devices: %q", got)
+	}
+}
+
+func TestLlamaCppSpreadsAReplicaOverItsNodesGPUsByLayer(t *testing.T) {
+	four := Profile{Name: "t4x4", Engine: "llamacpp", GPUClass: "t4", TP: 1, GPUsPerNode: 4, ModelPath: "/m"}
+	if err := (&LlamaCpp{}).Validate(four, h100(4)); err != nil {
+		t.Fatalf("four GPUs on a four-GPU node: %v", err)
+	}
+	if err := (&LlamaCpp{}).Validate(four, h100(2)); err == nil {
+		t.Error("four GPUs were accepted on a node with two")
+	}
+
+	s := llamaSpec([]int{0, 1, 2, 3})
+	s.Profile.GPUsPerNode = 4
+	cmds, err := (&LlamaCpp{}).Commands(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v, _ := flagValue(cmds[0].Args, "--split-mode"); v != "layer" {
+		t.Errorf("--split-mode %q over four GPUs", v)
+	}
+	if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=0,1,2,3") {
+		t.Errorf("env %v: the replica must see all four of its GPUs", cmds[0].Env)
+	}
+	one, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2}))
+	if hasFlag(one[0].Args, "--split-mode") {
+		t.Error("a one-GPU replica was given a split mode")
+	}
+}

internal/agent/engines/sglang.go

@@ -96,6 +96,11 @@ func (g *SGLang) serverCommand(s ShardSpec) (Command, error) {
 		)
 	}
 	args = append(args, p.Args...)
+	// SGLang reads its key only from the command line, so on a node with
+	// other users it is visible to them in the process list.
+	if s.APIKey != "" && s.Leader() {
+		args = append(args, "--api-key", s.APIKey)
+	}
 
 	return Command{Name: "sglang", Path: python, Args: args, Env: materializeEnv(baseEnv(s))}, nil
 }

internal/agent/engines/unsafeenv.go

@@ -0,0 +1,30 @@
+package engines
+
+import "strings"
+
+// UnsafeEnv reports whether a profile may not set an environment variable,
+// and why.
+//
+// A profile's env is the escape hatch for tuning no adapter can guess —
+// CUDA_*, NCCL_*, VLLM_*, HF_* — and it reaches an engine the agent starts as
+// root. Some variables are not tuning but a way to run other code in that
+// process: the dynamic loader's LD_PRELOAD, Python's PYTHONSTARTUP, a shell's
+// BASH_ENV. Set from a model definition, which an admin API or a config file
+// supplies, they would turn "add a model" into "run this as root on every
+// node". They are refused where a model is defined and again where an engine
+// starts.
+func UnsafeEnv(key string) (bool, string) {
+	k := strings.ToUpper(key)
+	switch {
+	case strings.HasPrefix(k, "LD_"), strings.HasPrefix(k, "DYLD_"):
+		return true, "it changes what the dynamic loader loads"
+	case k == "PATH":
+		return true, "it changes which binaries run"
+	case strings.HasPrefix(k, "PYTHON") && k != "PYTHONUNBUFFERED" && k != "PYTHONHASHSEED":
+		return true, "it changes what Python imports or runs at start"
+	case k == "BASH_ENV" || k == "ENV" || k == "PERL5OPT" || k == "PERL5LIB" ||
+		k == "NODE_OPTIONS" || k == "RUBYOPT" || k == "GCONV_PATH" || k == "OPENSSL_CONF":
+		return true, "it makes an interpreter or library load code"
+	}
+	return false, ""
+}

internal/agent/engines/vllm.go

@@ -150,6 +150,9 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
 	// Profile arguments last, so an operator can override anything above.
 	args = append(args, p.Args...)
 
+	if s.APIKey != "" {
+		env["VLLM_API_KEY"] = s.APIKey
+	}
 	return Command{Name: "vllm", Path: bin, Args: args, Env: materializeEnv(env)}, nil
 }
 
@@ -166,7 +169,10 @@ func (v *VLLM) rayCommand(s ShardSpec) (Command, bool) {
 
 	args := []string{"start", "--block"}
 	if s.Leader() {
-		args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort))
+		// Ray's head has no authentication: whoever reaches its port can run
+		// code on the node. Its dashboard, and the job API behind it, is the
+		// easiest way in, and nothing here uses it.
+		args = append(args, "--head", "--port", strconv.Itoa(s.LeaderPort), "--include-dashboard=false")
 	} else {
 		args = append(args, "--address", fmt.Sprintf("%s:%d", s.LeaderAddr, s.LeaderPort))
 	}

internal/agent/fabrics.go

@@ -0,0 +1,345 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+)
+
+// usableFabrics reports the fabrics this node can actually carry traffic on.
+//
+// Loud about what it drops. A fabric silently missing from a registration looks
+// identical to one never configured, and the operator who typed the adapter
+// name is the only person who can fix it — so the reason is logged here, on the
+// machine where both halves are visible, rather than inferred later from a
+// replica that will not converge.
+func (a *Agent) usableFabrics(inv discovery.Inventory) []inventory.FabricAttachment {
+	out := a.checkFabrics(inv.HCAs, true)
+	a.mu.Lock()
+	a.fabricsSent = fabricsFingerprint(out)
+	a.mu.Unlock()
+	return out
+}
+
+// checkFabrics is the declared fabrics this node can carry traffic on, from
+// the adapters as they are now. Loud logs every problem; the heartbeat checks
+// quietly and speaks only when the answer changes.
+func (a *Agent) checkFabrics(hcas []discovery.HCA, loud bool) []inventory.FabricAttachment {
+	declared := map[string][]string{}
+	for _, f := range a.cfg.Fabrics {
+		declared[f.Fabric] = f.HCAs
+	}
+
+	var out []inventory.FabricAttachment
+	for _, c := range discovery.CheckFabrics(declared, hcas) {
+		if loud {
+			for _, p := range c.Problems {
+				a.log.Error("a declared fabric adapter cannot carry traffic",
+					"fabric", c.Fabric, "problem", p)
+			}
+		}
+		if !c.OK() {
+			if loud {
+				a.log.Error("fabric not advertised; no declared adapter on it can carry traffic",
+					"fabric", c.Fabric,
+					"consequence", "no multi-node replica will be placed here")
+			}
+			continue
+		}
+		out = append(out, inventory.FabricAttachment{
+			Fabric: c.Fabric, HCAs: c.Usable, RateGbps: c.RateGbps,
+		})
+	}
+	return out
+}
+
+// fabricsFingerprint is enough to notice a fabric, an adapter or a rate
+// changing.
+func fabricsFingerprint(fs []inventory.FabricAttachment) string {
+	var b strings.Builder
+	for _, f := range fs {
+		fmt.Fprintf(&b, "%s:%s:%d;", f.Fabric, strings.Join(f.HCAs, ","), f.RateGbps)
+	}
+	return b.String()
+}
+
+// sampleIB re-reads the adapters for the heartbeat: every port's state and
+// counters, and the fabrics again when they no longer match what the control
+// plane was told.
+//
+// Every heartbeat, because it is a few small files per port, and because a
+// port that went down used to be invisible until the agent restarted — the
+// node went on being offered multi-node work that would hang on its first
+// collective.
+func (a *Agent) sampleIB(hb *llmgwv1.Heartbeat) {
+	a.mu.Lock()
+	had := len(a.inv.HCAs) > 0
+	a.mu.Unlock()
+	if !had && len(a.cfg.Fabrics) == 0 {
+		return
+	}
+	hcas, err := discovery.DiscoverIB(a.cfg.Discovery.IBRoot)
+	if err != nil {
+		return
+	}
+	a.attachSM(hcas)
+	for _, h := range hcas {
+		for _, p := range h.Ports {
+			ip := &llmgwv1.IBPort{
+				Hca: h.Name, Port: int32(p.Num), State: p.State, PhysState: p.PhysState,
+				RateGbps: int32(p.RateGbps), LinkLayer: string(p.LinkLayer), Counters: p.Counters,
+				NodeGuid: h.NodeGUID, PortGuid: p.PortGUID, Network: p.Network, GidPrefix: p.GIDPrefix,
+			}
+			a.mu.Lock()
+			if p.SM != nil {
+				ip.SmGuid, ip.SmLid, ip.SmPriority, ip.SmState = p.SM.GUID, int32(p.SM.LID), int32(p.SM.Priority), p.SM.State
+			} else {
+				ip.SmError = a.smErr[smKey(h.Name, p.Num)]
+			}
+			for _, r := range a.sms[smKey(h.Name, p.Num)] {
+				ip.SubnetManagers = append(ip.SubnetManagers, &llmgwv1.SubnetManager{
+					Guid: r.GUID, Lid: int32(r.LID), Priority: int32(r.Priority), State: r.State})
+			}
+			ip.SubnetManagersError = a.smsErr[smKey(h.Name, p.Num)]
+			a.mu.Unlock()
+			hb.IbPorts = append(hb.IbPorts, ip)
+		}
+	}
+	a.mu.Lock()
+	a.inv.HCAs = hcas // a reconnect registers what is true now
+	a.mu.Unlock()
+	if len(a.cfg.Fabrics) == 0 {
+		return
+	}
+	now := a.checkFabrics(hcas, false)
+	fp := fabricsFingerprint(now)
+	a.mu.Lock()
+	changed := fp != a.fabricsSent
+	a.fabricsSent = fp
+	a.mu.Unlock()
+	if !changed {
+		return
+	}
+	// Said once, with the reasons, the moment it happens.
+	a.checkFabrics(hcas, true)
+	a.log.Warn("the fabrics this node can carry traffic on changed", "now", fp)
+	hb.FabricsChanged = true
+	for _, f := range now {
+		hb.Fabrics = append(hb.Fabrics, &llmgwv1.FabricAttachment{
+			Fabric: f.Fabric, Hcas: f.HCAs, RateGbps: int32(f.RateGbps),
+		})
+	}
+}
+
+func smKey(hca string, port int) string { return fmt.Sprintf("%s/%d", hca, port) }
+
+// scanFabric is discovery.ScanFabric, replaced in tests.
+var scanFabric = discovery.ScanFabric
+
+// querySMs is discovery.QuerySMs, replaced in tests.
+var querySMs = func(hca string, port int) ([]discovery.SMRecord, error) {
+	return discovery.QuerySMs(context.Background(), hca, port)
+}
+
+func splitSMKey(k string) (string, int) {
+	i := strings.LastIndexByte(k, '/')
+	port, _ := strconv.Atoi(k[i+1:])
+	return k[:i], port
+}
+
+// querySM is discovery.QuerySM, replaced in tests.
+var querySM = func(hca string, port int) (*discovery.SMInfo, error) {
+	return discovery.QuerySM(context.Background(), hca, port)
+}
+
+// attachSM puts on each active InfiniBand port the subnet manager it sees,
+// asking again once a minute and otherwise reusing the last answer.
+func (a *Agent) attachSM(hcas []discovery.HCA) {
+	a.mu.Lock()
+	stale := time.Since(a.smAsked) > time.Minute || a.sm == nil
+	a.mu.Unlock()
+	if stale {
+		sm, errs := map[string]*discovery.SMInfo{}, map[string]string{}
+		_ = discovery.AttachSM(hcas, func(hca string, port int) (*discovery.SMInfo, error) {
+			got, err := querySM(hca, port)
+			if err != nil {
+				errs[smKey(hca, port)] = err.Error()
+				return nil, err
+			}
+			sm[smKey(hca, port)] = got
+			return got, nil
+		})
+		// Every subnet manager on each fabric, through one port per master:
+		// the same list through every port of one fabric is one query too
+		// many for each.
+		sms, smsErr := map[string][]discovery.SMRecord{}, map[string]string{}
+		asked := map[string]bool{}
+		for k, v := range sm {
+			if asked[v.GUID] {
+				continue
+			}
+			asked[v.GUID] = true
+			hca, port := splitSMKey(k)
+			if recs, err := querySMs(hca, port); err != nil {
+				smsErr[k] = err.Error()
+			} else {
+				sms[k] = recs
+			}
+		}
+		a.mu.Lock()
+		for k, v := range sm {
+			if old := a.sm[k]; old != nil && old.GUID != v.GUID {
+				a.log.Warn("the master subnet manager changed", "port", k, "from", old.GUID, "to", v.GUID)
+			}
+		}
+		a.sm, a.smErr, a.smAsked = sm, errs, time.Now()
+		a.sms, a.smsErr = sms, smsErr
+		a.mu.Unlock()
+		return
+	}
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	for i := range hcas {
+		for j := range hcas[i].Ports {
+			if sm := a.sm[smKey(hcas[i].Name, hcas[i].Ports[j].Num)]; sm != nil && hcas[i].Ports[j].Active() {
+				hcas[i].Ports[j].SM = sm
+			}
+		}
+	}
+}
+
+// ensureDiags installs infiniband-diags on a node with adapters and without
+// it. sminfo is what tells which physical fabric a port is on; without it,
+// two separate fabrics given one name cannot be caught. Installed the way
+// engine dependencies are, from the machine's own package sources.
+func (a *Agent) ensureDiags(ctx context.Context) {
+	a.mu.Lock()
+	hasIB := len(a.inv.HCAs) > 0
+	a.mu.Unlock()
+	if !hasIB || a.prov == nil {
+		return
+	}
+	if _, err := exec.LookPath("sminfo"); err == nil {
+		return
+	}
+	a.log.Info("installing infiniband-diags, to tell which fabric each port is on")
+	pkgs := map[string][]string{"deb": {"infiniband-diags"}, "rpm": {"infiniband-diags"}}
+	if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
+		a.log.Warn("could not install infiniband-diags; fabrics cannot be told apart by their subnet manager",
+			"err", err)
+	}
+}
+
+// FeatureFabricTest is the agent answering FabricTestRequest.
+const FeatureFabricTest = "fabric-test"
+
+// runWriteBW is discovery.RunWriteBW, and haveWriteBW whether it is
+// installed; both replaced in tests.
+var (
+	runWriteBW  = discovery.RunWriteBW
+	haveWriteBW = func() bool { _, err := exec.LookPath("ib_write_bw"); return err == nil }
+)
+
+// fabricTest runs this node's side of a bandwidth test with another.
+func (a *Agent) fabricTest(ctx context.Context, req *llmgwv1.FabricTestRequest) *llmgwv1.FabricTestReply {
+	start := time.Now()
+	reply := &llmgwv1.FabricTestReply{RequestId: req.GetRequestId()}
+	defer func() { reply.TookMs = time.Since(start).Milliseconds() }()
+
+	w := discovery.WriteBW{HCA: req.GetHca(), Port: int(req.GetPort()), GIDIndex: -1,
+		TCPPort: int(req.GetTcpPort()), Seconds: int(req.GetSeconds())}
+	if req.GetRole() == "client" {
+		w.Peer = req.GetPeer()
+	}
+	a.mu.Lock()
+	found := false
+	for _, h := range a.inv.HCAs {
+		for _, p := range h.Ports {
+			if h.Name == w.HCA && p.Num == w.Port {
+				found = true
+				// RoCE needs its v2 GID named, as NCCL does; InfiniBand uses the LID.
+				if p.LinkLayer == discovery.LinkEthernet && p.RoCEv2GID >= 0 {
+					w.GIDIndex = p.RoCEv2GID
+				}
+			}
+		}
+	}
+	a.mu.Unlock()
+	if !found {
+		reply.Error = fmt.Sprintf("this node has no port %s/%d", w.HCA, w.Port)
+		return reply
+	}
+	if !haveWriteBW() && a.prov != nil {
+		a.log.Info("installing perftest, for a bandwidth test with another node")
+		pkgs := map[string][]string{"deb": {"perftest"}, "rpm": {"perftest"}}
+		if err := a.prov.InstallPackages(ctx, pkgs); err != nil {
+			reply.Error = "installing perftest: " + err.Error()
+			return reply
+		}
+	}
+	gbps, out, err := runWriteBW(ctx, w)
+	reply.Gbps = gbps
+	reply.Output = tailLines(out, 12)
+	if err != nil {
+		reply.Error = err.Error()
+	}
+	return reply
+}
+
+func tailLines(s string, n int) string {
+	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
+	if len(lines) > n {
+		lines = lines[len(lines)-n:]
+	}
+	return strings.Join(lines, "\n")
+}
+
+// discoverGPUFabric and discoverIMEX are replaced in tests.
+var (
+	discoverGPUFabric = discovery.DiscoverGPUFabric
+	discoverIMEX      = discovery.DiscoverIMEX
+)
+
+// sampleNVLink reads, once a minute, which NVLink domain each GPU is joined
+// to and whether IMEX is up. nvidia-smi -q takes a second or two, too long
+// for every heartbeat, and a fabric that was joined an hour ago rarely
+// changes in five seconds — but when a fabric manager restarts it does, for
+// every card at once.
+func (a *Agent) sampleNVLink(hb *llmgwv1.Heartbeat) {
+	a.mu.Lock()
+	gpus := slices.Clone(a.inv.GPUs)
+	due := time.Since(a.nvlAsked) > time.Minute
+	if due {
+		a.nvlAsked = time.Now()
+	}
+	a.mu.Unlock()
+	if len(gpus) == 0 || !due {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+	fab, err := discoverGPUFabric(ctx, a.cfg.Discovery.NvidiaSMI)
+	if err != nil || len(fab) == 0 {
+		return
+	}
+	discovery.ApplyFabric(gpus, fab)
+	for _, g := range gpus {
+		if g.Fabric == nil {
+			continue
+		}
+		hb.GpuFabric = append(hb.GpuFabric, &llmgwv1.GPUFabricState{
+			Index: int32(g.Index), State: g.Fabric.State, Status: g.Fabric.Status,
+			ClusterUuid: g.Fabric.ClusterUUID, CliqueId: g.Fabric.CliqueID, Bandwidth: g.Fabric.Bandwidth,
+		})
+	}
+	m := discoverIMEX(ctx)
+	hb.Imex = &llmgwv1.IMEX{State: m.State, Channels: int32(m.Channels)}
+}

internal/agent/gpucheck.go

@@ -0,0 +1,155 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// gpuProcesses and processGroup are variables so tests can stand in for
+// nvidia-smi and /proc.
+var (
+	gpuProcesses = discovery.GPUProcesses
+	gpuMemory    = discovery.GPUMemoryByProcess
+	processGroup = pgidOf
+	// gpuCheckWait is how long a shard has, once it answers, to show up on
+	// its GPUs: the weights are on them by then, but nvidia-smi can lag.
+	gpuCheckWait = 15 * time.Second
+)
+
+// onItsGPUs checks that a shard placed on GPUs computes on them.
+//
+// For an engine that falls back to the CPU when its GPU backend does not load,
+// "it answers" is not "it is ready": llama.cpp built for CUDA but missing
+// cuBLAS comes up on the CPU, answers at a tenth of the speed, and the GPU it
+// was given sits idle and reserved. What it computes on is what nvidia-smi
+// lists it on, so a shard none of whose processes hold memory on one of its
+// GPUs is refused, with what the engine itself can see.
+func (a *Agent) onItsGPUs(ctx context.Context, eng engines.Engine, spec engines.ShardSpec, pid int) error {
+	fb, ok := eng.(engines.CPUFallback)
+	if !ok || !fb.FallsBackToCPU() || len(spec.GPUIndices) == 0 || pid <= 0 {
+		return nil
+	}
+	group := processGroup(pid)
+	deadline := time.Now().Add(gpuCheckWait)
+	var missing []int
+	for {
+		procs, err := gpuProcesses(ctx, a.cfg.Discovery.NvidiaSMI)
+		if err != nil || procs == nil {
+			// No nvidia-smi to ask is no evidence either way, and a node that
+			// was given GPUs has one; say so and let the shard serve.
+			a.log.Warn("could not check that a GPU shard runs on its GPUs", "shard", spec.ShardID, "err", err)
+			return nil
+		}
+		missing = missing[:0]
+		for _, i := range spec.GPUIndices {
+			if !holds(procs[i], pid, group) {
+				missing = append(missing, i)
+			}
+		}
+		if len(missing) == 0 {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+		}
+	}
+	seen := "it lists no GPU at all"
+	if devs, err := fb.Devices(ctx); err != nil {
+		seen = "listing its devices failed: " + err.Error()
+	} else if len(devs) > 0 {
+		seen = "it lists " + strings.Join(devs, "; ")
+	}
+	return fmt.Errorf("placed on GPU %s but computing on the CPU: the engine holds no memory there. "+
+		"Its GPU backend did not load (%s). Check the build with `ldd` on its libggml-cuda.so, "+
+		"or run `llmgw doctor` on this node", joinIndices(missing), seen)
+}
+
+// foreignGPUMemory is what each GPU holds for anything other than this
+// agent's shards, by index, and each GPU's memory in use. Nil when nvidia-smi
+// cannot be asked, which is no reading rather than an empty card.
+//
+// Everything allocated less what the shards' processes hold, rather than the
+// sum of the other processes: a process in another container's PID namespace
+// holds memory nvidia-smi lists under no PID this node can see, and it is
+// still not memory placement can hand out.
+func (a *Agent) foreignGPUMemory(ctx context.Context, shards []int) (foreign, used []int32) {
+	mem, err := gpuMemory(ctx, a.cfg.Discovery.NvidiaSMI)
+	if err != nil || mem == nil {
+		return nil, nil
+	}
+	groups := make([]int, len(shards))
+	for i, pid := range shards {
+		groups[i] = processGroup(pid)
+	}
+	ours := func(p int) bool {
+		for i, pid := range shards {
+			if holds([]int{p}, pid, groups[i]) {
+				return true
+			}
+		}
+		return false
+	}
+	for _, g := range mem {
+		f := g.UsedMB
+		for _, p := range g.Procs {
+			if ours(p.PID) {
+				f -= p.UsedMB
+			}
+		}
+		foreign = append(foreign, int32(max(f, 0)))
+		used = append(used, int32(g.UsedMB))
+	}
+	return foreign, used
+}
+
+// holds reports whether any of pids is the shard's process or in its group:
+// an engine that computes in a child process is still the shard.
+func holds(pids []int, pid, group int) bool {
+	for _, p := range pids {
+		if p == pid || (group > 0 && processGroup(p) == group) {
+			return true
+		}
+	}
+	return false
+}
+
+// pgidOf reads a process's group from /proc; zero when it cannot.
+func pgidOf(pid int) int {
+	b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/stat")
+	if err != nil {
+		return 0
+	}
+	// The command name, in parentheses, may hold spaces; the fields after its
+	// closing parenthesis are state, ppid, pgrp.
+	s := string(b)
+	i := strings.LastIndexByte(s, ')')
+	if i < 0 {
+		return 0
+	}
+	f := strings.Fields(s[i+1:])
+	if len(f) < 3 {
+		return 0
+	}
+	n, _ := strconv.Atoi(f[2])
+	return n
+}
+
+func joinIndices(is []int) string {
+	s := make([]string, len(is))
+	for k, i := range is {
+		s[k] = strconv.Itoa(i)
+	}
+	return strings.Join(s, ", ")
+}

internal/agent/gpucheck_test.go

@@ -0,0 +1,124 @@
+package agent
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+)
+
+// fallbackEngine is an engine that comes up on the CPU when its GPU backend
+// does not load, and lists the devices it can see.
+type fallbackEngine struct {
+	engines.Engine
+	devices []string
+}
+
+func (fallbackEngine) FallsBackToCPU() bool { return true }
+func (f fallbackEngine) Devices(context.Context) ([]string, error) {
+	return f.devices, nil
+}
+
+func TestAGPUShardComputingOnTheCPUIsNotReady(t *testing.T) {
+	procs := map[int][]int{}
+	oldProcs, oldGroup, oldWait := gpuProcesses, processGroup, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) { return procs, nil }
+	processGroup = func(pid int) int {
+		if pid >= 500 && pid < 600 {
+			return 500 // the shard's group: 500 and its children
+		}
+		return pid
+	}
+	gpuCheckWait = 0
+	t.Cleanup(func() { gpuProcesses, processGroup, gpuCheckWait = oldProcs, oldGroup, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	spec := engines.ShardSpec{ShardID: "s", GPUIndices: []int{0, 1}}
+	eng := fallbackEngine{}
+
+	// On no GPU at all: refused, saying what the engine sees.
+	err := a.onItsGPUs(context.Background(), eng, spec, 500)
+	if err == nil || !strings.Contains(err.Error(), "computing on the CPU") || !strings.Contains(err.Error(), "lists no GPU") {
+		t.Fatalf("a shard on no GPU was let through: %v", err)
+	}
+	// On one of its two: refused, naming the one it is missing.
+	procs[0] = []int{500}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil || !strings.Contains(err.Error(), "GPU 1 ") {
+		t.Fatalf("a shard on half its GPUs: %v", err)
+	}
+	// Another process on GPU 1 is not the shard.
+	procs[1] = []int{900}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err == nil {
+		t.Fatal("another process's memory was taken for the shard's")
+	}
+	// A child of the shard holding GPU 1 is the shard.
+	procs[1] = []int{900, 512}
+	if err := a.onItsGPUs(context.Background(), eng, spec, 500); err != nil {
+		t.Fatalf("a shard on both its GPUs: %v", err)
+	}
+
+	// An engine that fails rather than falling back is not checked, nor is a
+	// shard with no GPUs.
+	procs = map[int][]int{}
+	if err := a.onItsGPUs(context.Background(), struct{ engines.Engine }{}, spec, 500); err != nil {
+		t.Errorf("checked an engine that does not fall back: %v", err)
+	}
+	if err := a.onItsGPUs(context.Background(), eng, engines.ShardSpec{}, 500); err != nil {
+		t.Errorf("checked a CPU shard: %v", err)
+	}
+}
+
+func TestTheCheckWaitsForTheShardToShowUpOnItsGPU(t *testing.T) {
+	calls := 0
+	oldProcs, oldWait := gpuProcesses, gpuCheckWait
+	gpuProcesses = func(context.Context, string) (map[int][]int, error) {
+		calls++
+		if calls < 2 {
+			return map[int][]int{}, nil
+		}
+		return map[int][]int{0: {77}}, nil
+	}
+	gpuCheckWait = 5 * time.Second
+	t.Cleanup(func() { gpuProcesses, gpuCheckWait = oldProcs, oldWait })
+
+	a := &Agent{log: slog.Default()}
+	if err := a.onItsGPUs(context.Background(), fallbackEngine{}, engines.ShardSpec{GPUIndices: []int{0}}, 77); err != nil {
+		t.Fatalf("nvidia-smi lagging by a second failed the shard: %v", err)
+	}
+}
+
+func TestForeignGPUMemoryLeavesOutTheAgentsOwnShards(t *testing.T) {
+	// cuda01: ComfyUI and one of our engines on the same card. Only ComfyUI's
+	// share — and whatever no listed process accounts for — is foreign.
+	oldMem, oldGroup := gpuMemory, processGroup
+	gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) {
+		return []discovery.GPUMemory{
+			{UsedMB: 15000, Procs: []discovery.GPUProcess{{PID: 71315, UsedMB: 4932}, {PID: 512, UsedMB: 9500}}},
+			{UsedMB: 300},
+		}, nil
+	}
+	processGroup = func(pid int) int {
+		if pid >= 500 && pid < 600 {
+			return 500
+		}
+		return pid
+	}
+	t.Cleanup(func() { gpuMemory, processGroup = oldMem, oldGroup })
+
+	a := &Agent{log: slog.Default()}
+	foreign, used := a.foreignGPUMemory(context.Background(), []int{500})
+	if !slices.Equal(foreign, []int32{5500, 300}) || !slices.Equal(used, []int32{15000, 300}) {
+		t.Errorf("foreign %v, used %v; want [5500 300] and [15000 300]", foreign, used)
+	}
+
+	// No nvidia-smi to ask is no reading, not an empty card.
+	gpuMemory = func(context.Context, string) ([]discovery.GPUMemory, error) { return nil, nil }
+	if foreign, _ := a.foreignGPUMemory(context.Background(), nil); foreign != nil {
+		t.Errorf("no reading came back as %v", foreign)
+	}
+}

internal/agent/logs.go

@@ -0,0 +1,34 @@
+package agent
+
+import (
+	"slices"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// logsReply answers a request for one of this node's logs.
+func (a *Agent) logsReply(req *llmgwv1.LogsRequest) *llmgwv1.LogsReply {
+	n := int(req.GetLines())
+	if n <= 0 {
+		n = 200
+	}
+	n = min(n, agentLogLines)
+	shards := a.sup.IDs()
+	slices.Sort(shards)
+	out := &llmgwv1.LogsReply{
+		RequestId: req.GetRequestId(),
+		Sources:   append([]string{"agent"}, shards...),
+	}
+	switch src := req.GetSource(); src {
+	case "", "agent":
+		out.Lines = a.logRing.tail(n)
+	default:
+		sh, err := a.sup.Get(src)
+		if err != nil {
+			out.Error = "no engine " + src + " runs on this node"
+			return out
+		}
+		out.Lines = sh.Logs(n)
+	}
+	return out
+}

internal/agent/provision.go

@@ -0,0 +1,227 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"maps"
+	"slices"
+	"strings"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
+)
+
+// engines is what this node can run: what the agent installed and saw run,
+// and what the configuration names and answers. Nothing merely found on the
+// machine — a llama-server on PATH is somebody's, not necessarily one that
+// works, and placing on it is how a replica fails on a node nobody watches.
+func (a *Agent) engines(ctx context.Context) map[string]string {
+	a.engineOnce.Do(func() {
+		vers := a.reg.ProbeConfigured(ctx)
+		a.mu.Lock()
+		a.configuredVers, a.lastEngineProbe = vers, time.Now()
+		a.mu.Unlock()
+		if len(vers) > 0 {
+			a.log.Info("engines named in the configuration",
+				"engines", engines.ProbedNames(vers))
+		}
+	})
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	vers := a.engineReportLocked()
+	// Whatever registration sends is what the control plane now knows, so a
+	// heartbeat repeats it only once it differs.
+	a.enginesSent = enginesFingerprint(vers)
+	return vers
+}
+
+// engineReportLocked merges the two sources. Installed wins: it is the one
+// that was seen to run. Called with mu held.
+func (a *Agent) engineReportLocked() map[string]string {
+	out := maps.Clone(a.configuredVers)
+	if out == nil {
+		out = map[string]string{}
+	}
+	maps.Copy(out, a.installedVers)
+	maps.DeleteFunc(out, func(e, _ string) bool { return a.disabled(e) })
+	return out
+}
+
+// byHand reports whether agent.yaml names this engine's binary, which opts
+// it out of being installed.
+func (a *Agent) byHand(engine string) bool {
+	if a.reg == nil {
+		return false
+	}
+	e, err := a.reg.Get(engine)
+	if err != nil {
+		return false
+	}
+	c, ok := e.(engines.Configured)
+	return ok && c.Configured()
+}
+
+// disabled reports whether agent.yaml says this node never runs an engine:
+// not installed when asked, and not advertised when found.
+func (a *Agent) disabled(engine string) bool {
+	return slices.Contains(a.cfg.Engines.Disabled, engine)
+}
+
+// provisionEvery is how often installed engines are checked again. A library
+// removed by an unrelated upgrade is found here rather than by the next
+// replica to start.
+const provisionEvery = 10 * time.Minute
+
+// setProvision takes the control plane's list of wanted engines.
+func (a *Agent) setProvision(p *llmgwv1.Provision) {
+	wants := []provision.Want{}
+	for _, e := range p.GetEngines() {
+		if a.disabled(e.GetEngine()) {
+			continue // a control plane older than the node's say in it
+		}
+		if a.byHand(e.GetEngine()) {
+			// The operator named a binary for it: a local build, a CUDA
+			// runtime of their own. Installing one beside it would win —
+			// the installed build is used first — and quietly replace theirs.
+			continue
+		}
+		wants = append(wants, provision.Want{Engine: e.GetEngine(), Version: e.GetVersion(), Variant: e.GetVariant()})
+	}
+	a.mu.Lock()
+	a.provWants = wants
+	a.mu.Unlock()
+	select {
+	case a.provKick <- struct{}{}:
+	default:
+	}
+}
+
+// provisionLoop installs and re-checks the wanted engines, one pass at a time.
+func (a *Agent) provisionLoop(ctx context.Context) {
+	// After a failure the next pass comes sooner, and backs off: a lock held
+	// by a new machine's own updates clears in a minute or two, while ten
+	// minutes' wait left a node idle for no reason.
+	next := provisionEvery
+	retry := time.Minute
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-a.provKick:
+		case <-time.After(next):
+		}
+		a.mu.Lock()
+		wants := slices.Clone(a.provWants)
+		managed := a.provWants != nil
+		a.mu.Unlock()
+		if !managed {
+			continue
+		}
+
+		statuses := a.prov.Converge(ctx, wants)
+		next = provisionEvery
+		for _, st := range statuses {
+			if st.State == provision.Failed {
+				next, retry = retry, min(retry*2, provisionEvery)
+				break
+			}
+		}
+		if next == provisionEvery {
+			retry = time.Minute
+		}
+		installed := map[string]string{}
+		keep := map[string]bool{}
+		for _, st := range statuses {
+			keep[st.Engine] = true
+			if st.State == provision.Ready {
+				a.reg.UseInstalled(st.Engine, st.Binary)
+				installed[st.Engine] = st.Version
+			}
+		}
+		a.mu.Lock()
+		if enginesFingerprint(installed) != enginesFingerprint(a.installedVers) {
+			a.log.Info("engines installed and running", "engines", engines.ProbedNames(installed))
+		}
+		a.installedVers = installed
+		// An engine no longer asked for is no longer reported on.
+		for e := range a.provStatus {
+			if !keep[e] {
+				delete(a.provStatus, e)
+			}
+		}
+		a.mu.Unlock()
+	}
+}
+
+// provisionStatus records a change in one engine's installation.
+func (a *Agent) provisionStatus(st provision.Status) {
+	a.mu.Lock()
+	prev, had := a.provStatus[st.Engine]
+	a.provStatus[st.Engine] = st
+	a.mu.Unlock()
+	if !had || prev.State != st.State || prev.Detail != st.Detail {
+		lvl := slog.LevelInfo
+		if st.State == provision.Failed || st.State == provision.Unsupported {
+			lvl = slog.LevelWarn
+		}
+		a.log.Log(context.Background(), lvl, "engine installation",
+			"engine", st.Engine, "version", st.Version, "variant", st.Variant,
+			"state", st.State, "detail", st.Detail)
+	}
+}
+
+func provisionFingerprint(m map[string]provision.Status) string {
+	var b strings.Builder
+	for _, k := range slices.Sorted(maps.Keys(m)) {
+		st := m[k]
+		fmt.Fprintf(&b, "%s|%s|%s|%s|%s\n", st.Engine, st.Version, st.Variant, st.State, st.Detail)
+	}
+	return b.String()
+}
+
+// engineReprobe is how often the installed engines are looked for again.
+//
+// Minutes, not seconds: an engine is installed by somebody at a keyboard, so a
+// minute's delay in noticing is invisible, while SGLang's probe starts a
+// Python interpreter that imports it.
+const engineReprobe = time.Minute
+
+// reprobeEngines looks for installed engines again when it is time, off the
+// heartbeat goroutine so a slow probe never delays a heartbeat.
+func (a *Agent) reprobeEngines() {
+	a.mu.Lock()
+	due := !a.probingEngines && !a.lastEngineProbe.IsZero() &&
+		time.Since(a.lastEngineProbe) >= engineReprobe
+	if due {
+		a.probingEngines = true
+	}
+	a.mu.Unlock()
+	if !due {
+		return
+	}
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		vers := a.reg.ProbeConfigured(ctx)
+
+		a.mu.Lock()
+		changed := enginesFingerprint(vers) != enginesFingerprint(a.configuredVers)
+		a.configuredVers, a.lastEngineProbe, a.probingEngines = vers, time.Now(), false
+		a.mu.Unlock()
+		if changed {
+			a.log.Info("engines named in the configuration changed",
+				"engines", engines.ProbedNames(vers))
+		}
+	}()
+}
+
+func enginesFingerprint(vers map[string]string) string {
+	var b strings.Builder
+	for _, k := range slices.Sorted(maps.Keys(vers)) {
+		fmt.Fprintf(&b, "%s=%s\n", k, vers[k])
+	}
+	return b.String()
+}

internal/agent/provision/install.go

@@ -16,6 +16,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -26,6 +27,9 @@ import (
 type Want struct {
 	Engine  string
 	Version string
+	// Variant is a build chosen for this node ("cu129"); empty leaves it to
+	// the node's hardware.
+	Variant string
 }
 
 // The states an engine's installation can be in.
@@ -77,6 +81,10 @@ type Installer struct {
 	PyPIBase     string
 }
 
+// validVersion is what a version may look like: "b10991", "0.29.0",
+// "0.11.0+cu128". No separators, and not starting with a dot.
+var validVersion = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$`)
+
 // DefaultRoot is where engines are installed when nothing says otherwise.
 const DefaultRoot = "/opt/llmgw/engines"
 
@@ -114,6 +122,15 @@ func (in *Installer) Converge(ctx context.Context, wants []Want) []Status {
 
 func (in *Installer) one(ctx context.Context, w Want) Status {
 	st := Status{Engine: w.Engine, Version: w.Version}
+	// Checked here, on the node, whatever the control plane checked: the
+	// version names a directory under the install root and is passed to a
+	// package manager, and "../x" would put an install — and the RemoveAll
+	// that replaces one — outside it, as root.
+	if !validVersion.MatchString(w.Version) {
+		st.State = Unsupported
+		st.Detail = fmt.Sprintf("%q is not a version this agent will install", w.Version)
+		return st
+	}
 	recipe, ok := Recipes[w.Engine]
 	if !ok {
 		st.State = Unsupported
@@ -121,7 +138,12 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
 			"; install it by hand and name it in the agent's configuration"
 		return st
 	}
-	build, err := recipe.Choose(w.Version, in.Host)
+	if w.Variant != "" && !validVersion.MatchString(w.Variant) {
+		st.State = Unsupported
+		st.Detail = fmt.Sprintf("%q is not a build this agent will install", w.Variant)
+		return st
+	}
+	builds, err := recipe.Candidates(w.Version, w.Variant, in.Host)
 	if err != nil {
 		st.State, st.Detail = Unsupported, err.Error()
 		var nh NotHere
@@ -130,14 +152,18 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
 		}
 		return st
 	}
-	st.Variant = build.Variant
-	dir := filepath.Join(in.root(), w.Engine, w.Version+"-"+build.Variant)
 
-	// Installed before: check it still runs, and trust nothing else about it.
-	if _, err := os.Stat(filepath.Join(dir, marker)); err == nil {
+	// Installed before: whichever candidate is, best first, checked that it
+	// still runs and trusted for nothing else.
+	for _, build := range builds {
+		dir := in.dirFor(w, build)
+		if _, err := os.Stat(filepath.Join(dir, marker)); err != nil {
+			continue
+		}
+		st := st.as(w, build)
 		// The packages are checked too. Removing one is exactly the kind of
-		// change an unrelated upgrade makes, and ldd below would catch it only
-		// as a missing library with no hint of which package held it.
+		// change an unrelated upgrade makes, and ldd below would catch it
+		// only as a missing library with no hint of which package held it.
 		if err := in.packages(ctx, build.Packages, &st); err != nil {
 			st.State, st.Detail = Failed, err.Error()
 			return st
@@ -147,24 +173,64 @@ func (in *Installer) one(ctx context.Context, w Want) Status {
 			return in.ready(st, build, bin, version)
 		}
 		in.log().Warn("an installed engine no longer runs; installing it again",
-			"engine", w.Engine, "version", w.Version, "err", err)
+			"engine", w.Engine, "version", st.Version, "variant", build.Variant, "err", err)
+		break
 	}
 
-	if err := in.install(ctx, recipe, w, build, dir, &st); err != nil {
-		st.State, st.Detail, st.Binary = Failed, err.Error(), ""
-		return st
+	// Each candidate in turn until one installs and is seen to work: a build
+	// the table was wrong about — a driver too old for it after all, a GPU
+	// it has no kernels for — costs its download, and the next is tried.
+	var tried []string
+	for _, build := range builds {
+		st := st.as(w, build)
+		if err := in.install(ctx, recipe, w, build, in.dirFor(w, build), &st); err != nil {
+			tried = append(tried, build.Variant+": "+err.Error())
+			in.log().Warn("an engine build did not work here; trying the next",
+				"engine", w.Engine, "variant", build.Variant, "err", err)
+			continue
+		}
+		bin, version, err := in.verify(ctx, in.dirFor(w, build), build)
+		if err != nil {
+			tried = append(tried, build.Variant+": "+err.Error())
+			continue
+		}
+		return in.ready(st, build, bin, version)
 	}
-	bin, version, err := in.verify(ctx, dir, build)
-	if err != nil {
-		st.State, st.Detail = Failed, err.Error()
-		return st
+	st.State, st.Binary = Failed, ""
+	st.Detail = strings.Join(tried, "; then ")
+	if len(builds) == 1 {
+		_, st.Detail, _ = strings.Cut(st.Detail, ": ")
 	}
-	return in.ready(st, build, bin, version)
+	return st
+}
+
+// dirFor is where a build of an engine is installed: by the version it is
+// and its variant, so two builds of one version sit side by side.
+func (in *Installer) dirFor(w Want, b Build) string {
+	return filepath.Join(in.root(), w.Engine, b.versionOr(w.Version)+"-"+b.Variant)
+}
+
+// versionOr is the version a build is: its own, when it is not the one asked
+// for.
+func (b Build) versionOr(asked string) string {
+	if b.Version != "" {
+		return b.Version
+	}
+	return asked
+}
+
+// as is a status for one build of what was asked for.
+func (st Status) as(w Want, b Build) Status {
+	st.Variant, st.Version = b.Variant, b.versionOr(w.Version)
+	return st
 }
 
 func (in *Installer) ready(st Status, b Build, bin, version string) Status {
 	st.State, st.Binary = Ready, bin
 	st.Detail = strings.TrimSpace(filepath.Base(bin) + " " + version)
+	if b.Note != "" {
+		st.Detail += "; " + b.Note
+	}
 	var missing []string
 	for _, f := range b.CPUFlags {
 		if !in.Host.HasCPUFlag(f) {
@@ -285,9 +351,55 @@ func (in *Installer) verify(ctx context.Context, dir string, b Build) (bin, vers
 	if err != nil {
 		return "", "", fmt.Errorf("%s --version failed: %v: %s", b.Binary, err, lastLine(out))
 	}
+	if b.GPUBackend != "" {
+		if err := in.verifyGPUBackend(tctx, dir, bin, b); err != nil {
+			return "", "", err
+		}
+	}
 	return bin, firstVersionLine(out), nil
 }
 
+// verifyGPUBackend checks a GPU build can compute on this node's GPUs.
+//
+// The binary's own ldd says nothing about it: the GPU backend is a library
+// the server opens at run time, and when it cannot — cuBLAS missing, a driver
+// too old — the server carries on with its CPU backend. So the build is asked
+// which devices it sees, and when it sees none, ldd on the backend says why.
+func (in *Installer) verifyGPUBackend(ctx context.Context, dir, bin string, b Build) error {
+	out, err := in.run(ctx, nil, bin, "--list-devices")
+	if err != nil {
+		return fmt.Errorf("%s --list-devices failed: %v: %s", b.Binary, err, lastLine(out))
+	}
+	if listsDevice(string(out)) {
+		return nil
+	}
+	why := "its GPU backend did not load"
+	if lib, err := findFile(dir, b.GPUBackend); err != nil {
+		why = "the build has no " + b.GPUBackend
+	} else if out, err := in.run(ctx, nil, "ldd", lib); err == nil {
+		if missing := lddMissing(string(out)); len(missing) > 0 {
+			why = b.GPUBackend + " cannot load: " + strings.Join(missing, "; ")
+		}
+	}
+	return fmt.Errorf("the %s build of %s sees no GPU, and would run on the CPU: %s", b.Variant, b.Binary, why)
+}
+
+// listsDevice reads llama-server --list-devices: a device line after
+// "Available devices:", which holds "(none)" when there is none.
+func listsDevice(out string) bool {
+	listing := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "Available devices"):
+			listing = true
+		case listing && line != "" && line != "(none)":
+			return true
+		}
+	}
+	return false
+}
+
 // packages installs the operating system packages a build needs.
 //
 // Only those missing, and from the node's own package sources: a package the
@@ -509,6 +621,11 @@ func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want s
 // untarGz unpacks an archive under dir, refusing anything that would land
 // outside it. A release archive is somebody else's file, and a path of
 // "../../etc/cron.d/x" in it would otherwise be written as root.
+//
+// Every write goes through an os.Root, which refuses to leave dir however the
+// path gets there. Checking names alone was not enough: "a -> ." followed by
+// "a/a/b -> ../.." reads as inside and resolves two levels above it, and a
+// file written through that lands wherever the links point.
 func untarGz(archive, dir string) error {
 	f, err := os.Open(archive) //nolint:gosec // inside the install root
 	if err != nil {
@@ -519,6 +636,11 @@ func untarGz(archive, dir string) error {
 	if err != nil {
 		return err
 	}
+	root, err := os.OpenRoot(dir)
+	if err != nil {
+		return err
+	}
+	defer root.Close()
 	tr := tar.NewReader(gz)
 	for {
 		h, err := tr.Next()
@@ -528,22 +650,22 @@ func untarGz(archive, dir string) error {
 		if err != nil {
 			return err
 		}
-		target, err := within(dir, h.Name)
-		if err != nil {
+		name := filepath.Clean(h.Name)
+		if _, err := within(dir, name); err != nil {
 			return err
 		}
 		switch h.Typeflag {
 		case tar.TypeDir:
-			if err := os.MkdirAll(target, 0o755); err != nil {
+			if err := root.MkdirAll(name, 0o755); err != nil {
 				return err
 			}
 		case tar.TypeReg:
-			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 				return err
 			}
-			out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above
+			out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755)
 			if err != nil {
-				return err
+				return fmt.Errorf("%s: %w", h.Name, err)
 			}
 			if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum
 				out.Close()
@@ -553,19 +675,19 @@ func untarGz(archive, dir string) error {
 				return err
 			}
 		case tar.TypeSymlink:
-			// Libraries are versioned through symlinks, and those are
-			// relative. One pointing anywhere else is refused.
-			if filepath.IsAbs(h.Linkname) {
-				return fmt.Errorf("%s links outside the archive", h.Name)
+			// Libraries are versioned through symlinks to a file beside them —
+			// libllama.so -> libllama.so.0 — and that is all a release needs.
+			// A target with a separator in it could point anywhere once other
+			// links are followed, and a binary found through one would be run
+			// as root, so it is refused rather than reasoned about.
+			if h.Linkname == "" || strings.ContainsAny(h.Linkname, `/\`) || h.Linkname == ".." || h.Linkname == "." {
+				return fmt.Errorf("%s links to %q; only a file beside it is allowed", h.Name, h.Linkname)
 			}
-			if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil {
-				return fmt.Errorf("%s links outside the archive", h.Name)
-			}
-			if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+			if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 				return err
 			}
-			_ = os.Remove(target)
-			if err := os.Symlink(h.Linkname, target); err != nil {
+			_ = root.Remove(name)
+			if err := root.Symlink(h.Linkname, name); err != nil {
 				return err
 			}
 		default:

internal/agent/provision/install_test.go

@@ -89,7 +89,11 @@ type fakeSystem struct {
 	installed map[string]bool
 	tools     map[string]bool
 	lddOut    string
-	calls     []string
+	// devices is what llama-server --list-devices prints, and backendLdd
+	// what ldd says of its GPU backend.
+	devices    string
+	backendLdd string
+	calls      []string
 	// locked is how many more package commands find the lock held.
 	locked int
 }
@@ -127,8 +131,12 @@ func (f *fakeSystem) run(_ context.Context, _ []string, name string, args ...str
 			}
 		}
 		return nil, nil
+	case name == "ldd" && strings.HasSuffix(args[0], "libggml-cuda.so"):
+		return []byte(f.backendLdd), nil
 	case name == "ldd":
 		return []byte(f.lddOut), nil
+	case strings.HasSuffix(name, "llama-server") && args[0] == "--list-devices":
+		return []byte(f.devices), nil
 	case strings.HasSuffix(name, "llama-server"):
 		return []byte("version: 10991 (abc)\n"), nil
 	}
@@ -247,13 +255,38 @@ func TestAnArchiveCannotWriteOutsideItsDirectory(t *testing.T) {
 func TestTheBuildFollowsTheHardware(t *testing.T) {
 	gpu := ubuntu
 	gpu.NVIDIADriver = "570.86.15"
-	b, err := chooseLlamaCpp("b1", gpu)
-	if err != nil || !strings.HasPrefix(b.Variant, "cuda") || len(b.Assets) != 2 {
-		t.Errorf("an NVIDIA node got %+v, %v; expected the CUDA build and its runtime", b, err)
+	got, err := llamaCppCandidates("b1", "", gpu)
+	if err != nil || got[0].Variant != "cuda-12.8" || len(got[0].Assets) != 2 || got[len(got)-1].Variant != "vulkan" {
+		t.Errorf("an NVIDIA node on driver 570 got %+v, %v; expected CUDA 12.8 with its runtime, then Vulkan", got, err)
 	}
-	if _, err := chooseLlamaCpp("b1", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil {
+	if _, err := llamaCppCandidates("b1", "", discovery.Host{OSID: "ubuntu", Arch: "riscv64"}); err == nil {
 		t.Error("a build was chosen for an architecture llama.cpp does not publish for")
 	}
+	for _, c := range []struct {
+		name string
+		host discovery.Host
+		want []string
+	}{
+		// Driver 580 and Turing: CUDA 13.3 first.
+		{"current", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 8.0}, []string{"cuda-13.3", "cuda-12.8", "vulkan"}},
+		// A V100 on driver 550: CUDA 12.8 ships 7.0 as PTX, which needs 570.
+		{"V100, 550", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}, []string{"vulkan"}},
+		// An RTX 3090 on 550: 8.6 is machine code in the CUDA 12.8 build.
+		{"3090, 550", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 8.6}, []string{"cuda-12.8", "vulkan"}},
+		// A Pascal card: CUDA 12.8, not 13.3.
+		{"P40", discovery.Host{Arch: "amd64", NVIDIADriver: "575.51.03", MinComputeCapability: 6.1}, []string{"cuda-12.8", "vulkan"}},
+		// arm64 with a GPU: never the CPU build.
+		{"GH200", discovery.Host{Arch: "arm64", NVIDIADriver: "580.65.06", MinComputeCapability: 9.0}, []string{"cuda-13.3", "vulkan"}},
+	} {
+		got, err := llamaCppCandidates("b1", "", c.host)
+		var names []string
+		for _, b := range got {
+			names = append(names, b.Variant)
+		}
+		if err != nil || strings.Join(names, " ") != strings.Join(c.want, " ") {
+			t.Errorf("%s: %v, %v; want %v", c.name, names, err, c.want)
+		}
+	}
 
 	slow := ubuntu
 	slow.CPUFlags = nil
@@ -330,3 +363,90 @@ func TestAPackageManagerBusyWithItsOwnUpdatesIsWaitedFor(t *testing.T) {
 		t.Error("the wait was not reported")
 	}
 }
+
+func TestAVersionThatIsAPathIsRefused(t *testing.T) {
+	// The version names a directory under the install root; "../" would put
+	// an install, and the RemoveAll that replaces one, outside it, as root.
+	in := &Installer{Root: t.TempDir(), Host: ubuntu}
+	for _, v := range []string{"../../etc", "..", ".hidden", "b1/../../x", "", "b1 && rm -rf /"} {
+		got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: v}})
+		if got[0].State != Unsupported {
+			t.Errorf("version %q: %+v", v, got[0])
+		}
+	}
+}
+
+func TestChainedLinksCannotCarryAFileOutOfTheInstall(t *testing.T) {
+	// Each link reads as inside the directory; together they point above it.
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	_ = tw.WriteHeader(&tar.Header{Name: "a", Linkname: ".", Typeflag: tar.TypeSymlink})
+	_ = tw.WriteHeader(&tar.Header{Name: "a/a/b", Linkname: "../..", Typeflag: tar.TypeSymlink})
+	_ = tw.WriteHeader(&tar.Header{Name: "a/a/b/escaped", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg})
+	_, _ = tw.Write([]byte("x"))
+	_ = tw.Close()
+	_ = gz.Close()
+
+	parent := t.TempDir()
+	dir := filepath.Join(parent, "install")
+	_ = os.MkdirAll(dir, 0o755)
+	archive := filepath.Join(parent, "evil.tar.gz")
+	_ = os.WriteFile(archive, buf.Bytes(), 0o644)
+
+	if err := untarGz(archive, dir); err == nil {
+		t.Error("an archive of chained links was unpacked")
+	}
+	if _, err := os.Stat(filepath.Join(parent, "escaped")); err == nil {
+		t.Fatal("a file was written outside the install directory")
+	}
+}
+
+func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) {
+	var buf bytes.Buffer
+	gz := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gz)
+	_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so.0", Mode: 0o755, Size: 1, Typeflag: tar.TypeReg})
+	_, _ = tw.Write([]byte("x"))
+	_ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so", Linkname: "libllama.so.0", Typeflag: tar.TypeSymlink})
+	_ = tw.Close()
+	_ = gz.Close()
+	dir := t.TempDir()
+	archive := filepath.Join(t.TempDir(), "ok.tar.gz")
+	_ = os.WriteFile(archive, buf.Bytes(), 0o644)
+	if err := untarGz(archive, dir); err != nil {
+		t.Fatal(err)
+	}
+	if target, err := os.Readlink(filepath.Join(dir, "lib", "libllama.so")); err != nil || target != "libllama.so.0" {
+		t.Errorf("link %q %v", target, err)
+	}
+}
+
+func TestACUDABuildThatSeesNoGPUIsNotInstalled(t *testing.T) {
+	// Found in the field: libggml-cuda.so in the build, cuBLAS nowhere the
+	// loader looks, and llama-server serving a GPU placement on the CPU.
+	cuda := "bin-ubuntu-cuda-12.8-x64.tar.gz"
+	rel := newRelease(map[string][]byte{
+		"llama-b1-" + cuda: tarGz(t, map[string]string{
+			"llama-b1/llama-server": "#!/bin/sh\n", "llama-b1/libggml-cuda.so": "x"}, nil),
+		"cudart-llama-b1-" + cuda: tarGz(t, map[string]string{"libcudart.so.12": "x"}, nil),
+	})
+	gpu := ubuntu
+	gpu.NVIDIADriver = "570.86.15"
+	sys := &fakeSystem{installed: map[string]bool{"libgomp1": true},
+		devices:    "Available devices:\n  (none)\n",
+		backendLdd: "\tlibcublas.so.12 => not found\n\tlibcudart.so.12 => /x/libcudart.so.12\n"}
+	in, _ := installer(t, rel.serve(t), sys, gpu)
+
+	got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "sees no GPU") ||
+		!strings.Contains(got[0].Detail, "libcublas.so.12 is not installed") {
+		t.Fatalf("expected a failure naming the missing library: %+v", got[0])
+	}
+
+	// The same build seeing its GPU is installed.
+	sys.devices = "Available devices:\n  CUDA0: NVIDIA L4 (22994 MiB, 22713 MiB free)\n"
+	if got := in.Converge(t.Context(), []Want{{Engine: "llamacpp", Version: "b1"}}); got[0].State != Ready {
+		t.Fatalf("a CUDA build that sees its GPU: %+v", got[0])
+	}
+}

internal/agent/provision/python.go

@@ -36,6 +36,10 @@ const lockName = "llmgw-requirements.txt"
 func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
 	ctx, cancel := context.WithTimeout(ctx, pythonInstallTimeout)
 	defer cancel()
+	// The version this build is, which for a GPU older than every build of
+	// the one asked for is an older one: its publication date is the one
+	// the dependencies are resolved as of.
+	w.Version = b.versionOr(w.Version)
 
 	if err := in.packages(ctx, b.Packages, st); err != nil {
 		return err
@@ -86,28 +90,67 @@ func (in *Installer) installPython(ctx context.Context, r Recipe, w Want, b Buil
 	}
 	python := filepath.Join(tmp, "bin", "python")
 
-	in.progress(st, "resolving %s against PyPI as of %s", strings.Join(b.Requirements, " "), asOf[:10])
+	reqs := b.Requirements
+	if b.Wheel != nil {
+		// The engine from its GitHub release, pinned to the digest GitHub
+		// publishes for the file: the same trust as an archive, and uv
+		// refuses a download that does not match it.
+		in.progress(st, "asking GitHub for %s %s's %s…%s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Prefix, b.Wheel.Suffix)
+		sums, err := in.checksums(ctx, b.Wheel.Repo, b.Wheel.Tag)
+		if err != nil {
+			return err
+		}
+		var asset, sum string
+		for name, s := range sums {
+			if strings.HasPrefix(name, b.Wheel.Prefix) && strings.HasSuffix(name, b.Wheel.Suffix) {
+				asset, sum = name, s
+			}
+		}
+		if asset == "" {
+			return fmt.Errorf("%s %s publishes no %s…%s", b.Wheel.Repo, b.Wheel.Tag, b.Wheel.Prefix, b.Wheel.Suffix)
+		}
+		url := strings.TrimSuffix(in.downloadBase(), "/") + "/" + b.Wheel.Repo + "/releases/download/" +
+			b.Wheel.Tag + "/" + asset
+		reqs = append([]string{b.Wheel.Project + " @ " + url + "#sha256=" + sum}, reqs...)
+	}
+	// A second index, for the torch the engine was built on. Every package
+	// is taken from whichever of the two has the version resolved — both are
+	// the publishers' own — and each file is still pinned by its hash.
+	var index []string
+	if b.ExtraIndex != "" {
+		index = []string{"--extra-index-url", b.ExtraIndex, "--index-strategy", "unsafe-best-match"}
+	}
+
+	in.progress(st, "resolving %s against PyPI as of %s", strings.Join(reqs, " "), asOf[:10])
 	lock := filepath.Join(tmp, lockName)
-	args := []string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
-		"--generate-hashes", "--quiet", "--output-file", lock, "-"}
-	if out, err := in.runStdin(ctx, env, strings.Join(b.Requirements, "\n")+"\n", uv.Binary, args...); err != nil {
-		return fmt.Errorf("resolving %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
+	args := append([]string{"pip", "compile", "--python", python, "--exclude-newer", asOf,
+		"--generate-hashes", "--quiet", "--output-file", lock}, index...)
+	args = append(args, "-")
+	if out, err := in.runStdin(ctx, env, strings.Join(reqs, "\n")+"\n", uv.Binary, args...); err != nil {
+		return fmt.Errorf("resolving %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
 	}
 
 	in.progress(st, "downloading and installing %s and its dependencies: several gigabytes, which takes minutes",
-		strings.Join(b.Requirements, " "))
-	if out, err := in.run(ctx, env, uv.Binary, "pip", "install", "--python", python,
-		"--require-hashes", "--no-deps", "-r", lock); err != nil {
-		return fmt.Errorf("installing %s: %v: %s", strings.Join(b.Requirements, " "), err, tail(out))
+		strings.Join(reqs, " "))
+	install := append([]string{"pip", "install", "--python", python,
+		"--require-hashes", "--no-deps", "-r", lock}, index...)
+	if out, err := in.run(ctx, env, uv.Binary, install...); err != nil {
+		return fmt.Errorf("installing %s: %v: %s", strings.Join(reqs, " "), err, tail(out))
 	}
 
 	in.progress(st, "checking that %s runs", w.Engine)
 	if _, _, err := in.verify(ctx, tmp, b); err != nil {
 		return err
 	}
+	if b.GPUCheck && in.Host.NVIDIADriver != "" {
+		in.progress(st, "checking that %s (%s) computes on this node's GPU", w.Engine, b.Variant)
+		if err := in.gpuCheck(ctx, python, w, b); err != nil {
+			return err
+		}
+	}
 	m, _ := json.MarshalIndent(markerFile{
 		Engine: w.Engine, Version: w.Version, Variant: b.Variant,
-		Binary: filepath.Join("bin", b.Binary), Requirements: b.Requirements,
+		Binary: filepath.Join("bin", b.Binary), Requirements: reqs,
 		ExcludeNewer: asOf, UV: UVVersion, Installed: time.Now().UTC(),
 	}, "", "  ")
 	if err := os.WriteFile(filepath.Join(tmp, marker), m, 0o644); err != nil { //nolint:gosec // not secret
@@ -203,3 +246,26 @@ func tail(b []byte) string {
 	}
 	return strings.Join(lines, " / ")
 }
+
+// gpuProbe is what the environment is asked to run on the GPU: one small
+// operation, which needs the torch to have kernels for the card.
+const gpuProbe = `import torch
+assert torch.cuda.is_available(), "torch sees no CUDA device"
+x = torch.ones(64, device="cuda")
+print(float((x * 2).sum()), torch.cuda.get_device_name(0), torch.cuda.get_device_capability(0))`
+
+// gpuCheck runs gpuProbe in a freshly installed environment.
+func (in *Installer) gpuCheck(ctx context.Context, python string, w Want, b Build) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
+	defer cancel()
+	out, err := in.run(ctx, nil, python, "-c", gpuProbe)
+	if err != nil {
+		hint := ""
+		if strings.Contains(string(out), "no kernel image") {
+			hint = "; this build has no kernels for this GPU, so try the other build for this node"
+		}
+		return fmt.Errorf("%s %s (%s) installed but cannot compute on this node's GPU: %s%s",
+			w.Engine, w.Version, b.Variant, tail(out), hint)
+	}
+	return nil
+}

internal/agent/provision/python_test.go

@@ -2,6 +2,7 @@ package provision
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -23,6 +24,10 @@ type fakePython struct {
 	pmu      sync.Mutex
 	commands []string
 	failPip  bool
+	// noKernel makes the GPU check fail as a torch without kernels for the
+	// card does: in every build, or in noKernelIn only ("-cu13.").
+	noKernel   bool
+	noKernelIn string
 }
 
 func (f *fakePython) run(ctx context.Context, env []string, name string, args ...string) ([]byte, error) {
@@ -43,6 +48,11 @@ func (f *fakePython) run(ctx context.Context, env []string, name string, args ..
 		return nil, os.WriteFile(filepath.Join(venv, "bin", "vllm"), []byte("#!/bin/sh\n"), 0o755)
 	case strings.HasSuffix(name, "/bin/vllm") && args[0] == "--version":
 		return []byte("0.30.0\n"), nil
+	case strings.HasSuffix(name, "/bin/python") && args[0] == "-c":
+		if f.noKernel || (f.noKernelIn != "" && strings.Contains(name, f.noKernelIn)) {
+			return []byte("RuntimeError: CUDA error: no kernel image is available for execution on the device\n"), errExit
+		}
+		return []byte("128.0 Tesla V100-SXM2-32GB (7, 0)\n"), nil
 	}
 	return f.fakeSystem.run(ctx, env, name, args...)
 }
@@ -59,6 +69,8 @@ func pythonInstaller(t *testing.T, host discovery.Host, sys *fakePython) (*Insta
 	uv := newRelease(map[string][]byte{
 		"uv-x86_64-unknown-linux-gnu.tar.gz": tarGz(t,
 			map[string]string{"uv-x86_64-unknown-linux-gnu/uv": "#!/bin/sh\n"}, nil),
+		// The fake GitHub serves every repository's release from this one.
+		"vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl": []byte("the cu129 wheel"),
 	})
 	gh := uv.serve(t)
 	mux := http.NewServeMux()
@@ -66,6 +78,9 @@ func pythonInstaller(t *testing.T, host discovery.Host, sys *fakePython) (*Insta
 		_, _ = w.Write([]byte(`{"urls":[{"upload_time_iso_8601":"2026-09-22T05:22:21.647189Z"},
 			{"upload_time_iso_8601":"2026-09-22T05:23:10.039243Z"}]}`))
 	})
+	mux.HandleFunc("/pypi/vllm/0.19.1/json", func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(`{"urls":[{"upload_time_iso_8601":"2026-04-02T10:00:00Z"}]}`))
+	})
 	pypi := httptest.NewServer(mux)
 	t.Cleanup(pypi.Close)
 	return &Installer{
@@ -130,8 +145,10 @@ func TestVLLMIsNotForEveryNode(t *testing.T) {
 		says  string
 	}{
 		{"no GPU", ubuntu, Skipped, "has none"},
-		{"old driver", discovery.Host{Arch: "amd64", NVIDIADriver: "550.54.15"}, Unsupported, "driver 580"},
-		{"V100", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 7.0}, Unsupported, "7.5"},
+		// Too old for CUDA 13 and for its CUDA 12.9 build both.
+		{"a driver too old for either", discovery.Host{Arch: "amd64", NVIDIADriver: "470.256.02"}, Unsupported, "driver 525"},
+		// No vLLM was ever built for Pascal.
+		{"P100", discovery.Host{Arch: "amd64", NVIDIADriver: "580.65.06", MinComputeCapability: 6.0}, Unsupported, "no vLLM is built for GPUs of compute capability 6.0"},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -182,3 +199,114 @@ func TestAVersionPyPIDoesNotHaveIsSaidSo(t *testing.T) {
 		t.Errorf("got %s %q", got[0].State, got[0].Detail)
 	}
 }
+
+func TestAnOlderDriverGetsTheSameVLLMBuiltForCUDA12(t *testing.T) {
+	// A Turing card on a driver too old for CUDA 13: the same vLLM's CUDA
+	// 12.9 build, from the project's GitHub release, with the torch PyTorch
+	// publishes for 12.9.
+	volta := discovery.Host{OSID: "ubuntu", Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.5}
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, gh := pythonInstaller(t, volta, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Ready || got[0].Variant != "cu129" {
+		t.Fatalf("a Turing node on driver 550: %+v", got[0])
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu129", marker)); err != nil {
+		t.Errorf("not installed under its own build's name: %v", err)
+	}
+	all := strings.Join(sys.commands, "\n")
+	wheel := "vllm-0.30.0+cu129-cp38-abi3-manylinux_2_28_x86_64.whl"
+	for _, want := range []string{
+		// Pinned to the digest GitHub publishes for the wheel.
+		"vllm @ " + in.DownloadBase + "/vllm-project/vllm/releases/download/v0.30.0/" + wheel + "#sha256=" +
+			strings.TrimPrefix(gh.digests[wheel], "sha256:"),
+		"--extra-index-url https://download.pytorch.org/whl/cu129 --index-strategy unsafe-best-match -",
+		"--require-hashes --no-deps -r",
+		// Computing on the card, not only importing.
+		"/bin/python -c import torch",
+	} {
+		if !strings.Contains(all, want) {
+			t.Errorf("no %q in:\n%s", want, all)
+		}
+	}
+	for _, c := range sys.commands {
+		if strings.Contains(c, "pip install") && !strings.Contains(c, "download.pytorch.org/whl/cu129") {
+			t.Errorf("installed without the index its torch comes from: %s", c)
+		}
+	}
+}
+
+func TestABuildChosenForANodeIsTheOneInstalled(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	if got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0", Variant: "cu129"}}); got[0].Variant != "cu129" || got[0].State != Ready {
+		t.Errorf("cu129 chosen for a node that runs cu13 too: %+v", got[0])
+	}
+	if _, err := vllmCandidates("0.30.0", "cu200", gpuNode); err == nil {
+		t.Error("a build that does not exist was accepted")
+	}
+}
+
+func TestABuildThatCannotComputeOnTheCardIsNotInstalled(t *testing.T) {
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}, noKernel: true}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})
+	if got[0].State != Failed || !strings.Contains(got[0].Detail, "no kernels for this GPU") {
+		t.Fatalf("a build without kernels for the card: %+v", got[0])
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.30.0-cu13", marker)); err == nil {
+		t.Error("marked installed")
+	}
+}
+
+func TestOnlyACUDARequirementFallsBackToTheCUDA12Build(t *testing.T) {
+	// A machine vLLM is not published for is refused, not given the other
+	// build to fail with.
+	if _, err := vllmCandidates("0.30.0", "", discovery.Host{NVIDIADriver: "580.65.06"}); err == nil ||
+		!strings.Contains(err.Error(), "no Linux build") {
+		t.Errorf("an unknown architecture: %v", err)
+	}
+	if _, err := vllmCandidates("0.30.0", "cu129", ubuntu); !errors.As(err, new(NotHere)) {
+		t.Errorf("a node with no GPU chosen cu129: %v", err)
+	}
+	// Driver 580 and Turing: CUDA 13 first, the CUDA 12.9 build behind it.
+	got, err := vllmCandidates("0.30.0", "", gpuNode)
+	if err != nil || len(got) != 2 || got[0].Variant != "cu13" || got[1].Variant != "cu129" {
+		t.Errorf("candidates on a current node: %+v, %v", got, err)
+	}
+}
+
+func TestAV100GetsTheNewestVLLMBuiltForItByItself(t *testing.T) {
+	// 0.30.0 asked for; no build of it has Volta kernels. The node installs
+	// 0.19.1, the newest that has, and says so.
+	v100 := discovery.Host{OSID: "ubuntu", Arch: "amd64", NVIDIADriver: "550.54.15", MinComputeCapability: 7.0}
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}}
+	in, _ := pythonInstaller(t, v100, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})[0]
+	if got.State != Ready || got.Version != "0.19.1" || got.Variant != "cu12" ||
+		!strings.Contains(got.Detail, "the newest built for compute capability 7.0, instead of 0.30.0") {
+		t.Fatalf("a V100: %+v", got)
+	}
+	if _, err := os.Stat(filepath.Join(in.Root, "vllm", "0.19.1-cu12", marker)); err != nil {
+		t.Errorf("not installed as what it is: %v", err)
+	}
+	all := strings.Join(sys.commands, "\n")
+	if !strings.Contains(all, "<<< vllm==0.19.1") || strings.Contains(all, "download.pytorch.org") {
+		t.Errorf("0.19.1 is PyPI's CUDA 12 build, with PyPI's torch:\n%s", all)
+	}
+	// Chosen cu13, which 0.19.1 has none of: refused, saying why.
+	if _, err := vllmCandidates("0.30.0", "cu13", v100); err == nil {
+		t.Error("a CUDA 13 build was offered to a V100")
+	}
+}
+
+func TestABuildThatFailsOnTheCardGivesWayToTheNext(t *testing.T) {
+	// The table says CUDA 13 runs here; the card says otherwise. The next
+	// build is installed rather than the node going without.
+	sys := &fakePython{fakeSystem: fakeSystem{installed: map[string]bool{}}, noKernelIn: "-cu13."}
+	in, _ := pythonInstaller(t, gpuNode, sys)
+	got := in.Converge(t.Context(), []Want{{Engine: "vllm", Version: "0.30.0"}})[0]
+	if got.State != Ready || got.Variant != "cu129" {
+		t.Fatalf("after the CUDA 13 build failed on the card: %+v", got)
+	}
+}

internal/agent/provision/recipe.go

@@ -43,12 +43,41 @@ type Build struct {
 	// VersionArgs is what Binary is run with to prove the build works, when
 	// --version is not it.
 	VersionArgs []string
+	// GPUBackend, when set, is the shared library the build computes on GPUs
+	// with, loaded at run time rather than linked: "libggml-cuda.so". The
+	// build must list a device with it, or it is not installed — a llama.cpp
+	// whose CUDA backend cannot load runs on the CPU without complaint.
+	GPUBackend string
+	// Wheel, when set, is the engine itself as a wheel from a GitHub
+	// release rather than from PyPI, checked against the digest GitHub
+	// publishes for it. ExtraIndex is a package index besides PyPI that the
+	// rest resolves from too: the CUDA 12.9 torch the wheel was built on.
+	Wheel      *Wheel
+	ExtraIndex string
+	// GPUCheck, on a node with an NVIDIA GPU, has the installed environment
+	// compute something on one before the build counts as installed. A build
+	// with no kernels for the card — a GPU generation its torch dropped —
+	// imports and prints its version, and fails on its first model.
+	GPUCheck bool
+	// Version is the engine version this build is, when it is not the one
+	// asked for: the newest built for this machine's GPUs, where the one
+	// asked for has no kernels for them. Empty is the version asked for.
+	Version string
+	// Note says why this build, when it is not the plain one: shown beside
+	// it wherever the installation is.
+	Note string
 	// FreeGB is the disk space the installation needs before it starts: a
 	// Python engine is several gigabytes, and one that fills the disk halfway
 	// leaves the node worse off than one that never began.
 	FreeGB int
 }
 
+// Wheel is a package published as a GitHub release asset: the asset whose
+// name starts with Prefix and ends with Suffix.
+type Wheel struct {
+	Project, Repo, Tag, Prefix, Suffix string
+}
+
 // Recipe knows how to install one engine.
 type Recipe struct {
 	// Repo is the GitHub repository whose releases carry the builds; empty
@@ -56,8 +85,12 @@ type Recipe struct {
 	Repo string
 	// PyPI is the project an engine installed from PyPI is published as.
 	PyPI string
-	// Choose picks the build for a machine, or says why none fits.
-	Choose func(version string, h discovery.Host) (Build, error)
+	// Candidates are the builds that may run on a machine, best first, or
+	// why none can. The installer takes the first that installs and is seen
+	// to work — computing on the GPU, where there is one — so a build this
+	// table is wrong about costs a download, not a node. variant is a build
+	// chosen for the node; empty leaves the choice to the machine.
+	Candidates func(version, variant string, h discovery.Host) ([]Build, error)
 }
 
 // Recipes are the engines this agent can install.
@@ -66,10 +99,10 @@ type Recipe struct {
 // from PyPI by uv, which is itself installed like llama.cpp: a GitHub release
 // checked against its published checksum.
 var Recipes = map[string]Recipe{
-	"llamacpp": {Repo: "ggml-org/llama.cpp", Choose: chooseLlamaCpp},
-	"vllm":     {PyPI: "vllm", Choose: chooseVLLM},
-	"sglang":   {PyPI: "sglang", Choose: chooseSGLang},
-	"uv":       {Repo: "astral-sh/uv", Choose: chooseUV},
+	"llamacpp": {Repo: "ggml-org/llama.cpp", Candidates: llamaCppCandidates},
+	"vllm":     {PyPI: "vllm", Candidates: vllmCandidates},
+	"sglang":   {PyPI: "sglang", Candidates: sglangCandidates},
+	"uv":       {Repo: "astral-sh/uv", Candidates: uvCandidates},
 }
 
 // Installable are the engines this agent can install, as it tells the
@@ -99,7 +132,15 @@ type NotHere struct{ Reason string }
 
 func (e NotHere) Error() string { return e.Reason }
 
-func chooseUV(version string, h discovery.Host) (Build, error) {
+func uvCandidates(version, _ string, h discovery.Host) ([]Build, error) {
+	b, err := chooseUV(version, h)
+	if err != nil {
+		return nil, err
+	}
+	return []Build{b}, nil
+}
+
+func chooseUV(_ string, h discovery.Host) (Build, error) {
 	var arch string
 	switch h.Arch {
 	case "amd64":
@@ -116,33 +157,227 @@ func chooseUV(version string, h discovery.Host) (Build, error) {
 	}, nil
 }
 
-// minDriverForCUDA13 is the oldest NVIDIA driver CUDA 13 runs on. Every vLLM
-// since 0.26 and SGLang 0.5 is built on torch for CUDA 13.
-const minDriverForCUDA13 = 580
+// What builds need of the NVIDIA side of a machine, read from what each
+// project publishes and how it was built:
+//
+//   - CUDA 13 needs driver 580; CUDA 12 builds run on driver 525 and newer,
+//     by CUDA's minor version compatibility — except code shipped only as
+//     PTX, which the driver compiles and so needs a driver as new as the
+//     CUDA it was compiled with.
+//   - Which GPUs a build has kernels for is the build's own list, not what
+//     its CUDA supports.
+const (
+	minDriverForCUDA13 = 580
+	minDriverForCUDA12 = 525
+)
 
-// minComputeForCUDA13 is the oldest GPU CUDA 13 supports: Turing. It dropped
-// Volta, so a V100 cannot run these builds at all.
-const minComputeForCUDA13 = 7.5
+// gpuNeeds is what one build needs of a machine's GPUs and driver.
+type gpuNeeds struct {
+	driver int
+	cc     float64
+}
 
-// cudaGate says whether a node can run an engine built on CUDA 13.
-func cudaGate(engine string, h discovery.Host) error {
+// fits says why a machine cannot run a build, or nil.
+func (n gpuNeeds) fits(what string, h discovery.Host) error {
+	if major := leadingInt(h.NVIDIADriver); n.driver > 0 && major > 0 && major < n.driver {
+		return fmt.Errorf("%s needs NVIDIA driver %d or newer; this node has %s", what, n.driver, h.NVIDIADriver)
+	}
+	if cc := h.MinComputeCapability; n.cc > 0 && cc > 0 && cc < n.cc {
+		return fmt.Errorf("%s has kernels for compute capability %.1f and newer; this node has %.1f", what, n.cc, cc)
+	}
+	return nil
+}
+
+// candidate is a build and what it needs.
+type candidate struct {
+	b     Build
+	needs gpuNeeds
+	what  string
+}
+
+// pick keeps the candidates a machine can run, in order, and those a
+// variant chosen for the node names; with none left, it says why each went.
+func pick(engine, variant string, cands []candidate, h discovery.Host) ([]Build, error) {
+	var out []Build
+	var why []string
+	for _, c := range cands {
+		if variant != "" && c.b.Variant != variant {
+			continue
+		}
+		if err := c.needs.fits(c.what, h); err != nil {
+			why = append(why, err.Error())
+			continue
+		}
+		out = append(out, c.b)
+	}
+	switch {
+	case len(out) > 0:
+		return out, nil
+	case len(why) == 0 && variant != "":
+		var names []string
+		for _, c := range cands {
+			names = append(names, c.b.Variant)
+		}
+		return nil, fmt.Errorf("%s has no build %q here; it has %s", engine, variant, strings.Join(names, ", "))
+	}
+	return nil, fmt.Errorf("no build of %s runs on this node: %s", engine, strings.Join(why, "; "))
+}
+
+// --- vLLM ---
+
+// VLLMVariants are the builds of vLLM a node can be given by choice: CUDA
+// 13 from PyPI, and CUDA 12.9 from the project's GitHub release — the same
+// version, built for drivers CUDA 13 does not run on.
+var VLLMVariants = []string{"cu13", "cu129"}
+
+// pytorchCU129 is where the CUDA 12.9 build's torch is published. PyPI's
+// torch is built for CUDA 13; the cu129 vLLM wheel needs this one.
+const pytorchCU129 = "https://download.pytorch.org/whl/cu129"
+
+// vllmCUDA13Since is the first vLLM whose PyPI build is CUDA 13, read from
+// its releases: 0.19.1 publishes "+cu130" beside a CUDA 12 default, 0.20.0
+// "+cu129" beside a CUDA 13 one.
+const vllmCUDA13Since = "0.20"
+
+// vllmMinCompute is the oldest GPU a vLLM version's builds have kernels for,
+// from its release pipeline: on x86, 7.0 (Volta) up to 0.19.1 — the
+// Dockerfile's default "7.0 7.5 8.0 8.9 9.0 10.0 12.0" — and 7.5 from 0.20.0,
+// CUDA 12.9 build included; on arm64, built for Grace machines, 8.7 before
+// 0.20 and 8.0 since.
+func vllmMinCompute(version, arch string) float64 {
+	old := versionBefore(version, vllmCUDA13Since)
+	switch {
+	case arch == "arm64" && old:
+		return 8.7
+	case arch == "arm64":
+		return 8.0
+	case old:
+		return 7.0
+	}
+	return 7.5
+}
+
+// vllmNewestFor is the newest vLLM built for GPUs of compute capability cc
+// on an architecture, for GPUs older than every build of the version asked
+// for: a V100 gets 0.19.1, where 0.20 and later have nothing for it. Nothing
+// for a GPU no vLLM was built for.
+var vllmNewestFor = []struct {
+	arch    string
+	from    float64 // the oldest GPU it covers
+	version string
+}{
+	{"amd64", 7.0, "0.19.1"},
+}
+
+// vllmCandidates are the vLLM builds for a machine.
+//
+// The version asked for, in the build the machine runs: CUDA 13 where the
+// driver allows, the same version's CUDA 12.9 build where it does not. One
+// version across the cluster wherever the GPUs allow it, because the flags
+// llmgw builds are checked against it. Where they do not — GPUs older than
+// every build of that version — the newest version built for them, said so
+// beside the installation: vLLM on a V100 at all, rather than not.
+func vllmCandidates(version, variant string, h discovery.Host) ([]Build, error) {
+	arch, err := gpuArch("vLLM", h)
+	if err != nil {
+		return nil, err
+	}
+	v, note := version, ""
+	if cc := h.MinComputeCapability; cc > 0 && cc < vllmMinCompute(version, h.Arch) {
+		older := ""
+		for _, n := range vllmNewestFor {
+			if n.arch == h.Arch && cc >= n.from {
+				older = n.version
+			}
+		}
+		if older == "" {
+			return nil, fmt.Errorf("no vLLM is built for GPUs of compute capability %.1f on %s; "+
+				"the node serves llama.cpp models", cc, arch)
+		}
+		v, note = older, fmt.Sprintf("vLLM %s, the newest built for compute capability %.1f, instead of %s", older, cc, version)
+	}
+	pypi := func(variant string) Build {
+		b := Build{
+			Variant:      variant,
+			Python:       pythonForEngines,
+			Requirements: []string{"vllm==" + v},
+			Binary:       "vllm",
+			Packages:     compilerPackages,
+			FreeGB:       25,
+			GPUCheck:     true,
+			Note:         note,
+		}
+		if v != version {
+			b.Version = v
+		}
+		return b
+	}
+	minCC := vllmMinCompute(v, h.Arch)
+	if versionBefore(v, vllmCUDA13Since) {
+		// PyPI's own build is CUDA 12, and is the one there is; asked for
+		// cu129, this is it.
+		if variant == "cu129" {
+			variant = "cu12"
+		}
+		return pick("vLLM "+v, variant, []candidate{
+			{pypi("cu12"), gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v},
+		}, h)
+	}
+	cu129 := pypi("cu129")
+	cu129.Requirements = nil
+	// Found in the release's assets rather than named: the manylinux tag
+	// in the file's name changes between versions (2_28, 2_31, 2_34).
+	cu129.Wheel = &Wheel{
+		Project: "vllm", Repo: "vllm-project/vllm", Tag: "v" + v,
+		Prefix: "vllm-" + v + "+cu129-", Suffix: "_" + map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[h.Arch] + ".whl",
+	}
+	cu129.ExtraIndex = pytorchCU129
+	return pick("vLLM "+v, variant, []candidate{
+		{pypi("cu13"), gpuNeeds{minDriverForCUDA13, minCC}, "vLLM " + v + " built for CUDA 13"},
+		{cu129, gpuNeeds{minDriverForCUDA12, minCC}, "vLLM " + v + " built for CUDA 12.9"},
+	}, h)
+}
+
+// gpuArch says whether a Python GPU engine is published for this machine
+// and whether it has an NVIDIA GPU at all.
+func gpuArch(engine string, h discovery.Host) (string, error) {
 	switch h.Arch {
 	case "amd64", "arm64":
 	default:
-		return fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
+		return "", fmt.Errorf("%s publishes no Linux build for %s", engine, orUnknown(h.Arch))
 	}
 	if h.NVIDIADriver == "" {
-		return NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
+		return "", NotHere{engine + " runs on NVIDIA GPUs and this node has none; it serves llama.cpp models"}
 	}
-	if major := leadingInt(h.NVIDIADriver); major > 0 && major < minDriverForCUDA13 {
-		return fmt.Errorf("%s is built for CUDA 13, which needs NVIDIA driver %d or newer; this node has %s",
-			engine, minDriverForCUDA13, h.NVIDIADriver)
+	return h.Arch, nil
+}
+
+// --- SGLang ---
+
+// sglangCUDA13Since is the first SGLang built for CUDA 13.
+const sglangCUDA13Since = "0.5"
+
+func sglangCandidates(version, variant string, h discovery.Host) ([]Build, error) {
+	if _, err := gpuArch("SGLang", h); err != nil {
+		return nil, err
 	}
-	if cc := h.MinComputeCapability; cc > 0 && cc < minComputeForCUDA13 {
-		return fmt.Errorf("%s is built for CUDA 13, which supports GPUs of compute capability %.1f and newer; this node has %.1f",
-			engine, minComputeForCUDA13, cc)
+	b := Build{
+		Variant:      "cu13",
+		Python:       pythonForEngines,
+		Requirements: []string{"sglang==" + version},
+		// SGLang is launched as a module, so what the agent is given is the
+		// environment's interpreter, and the proof is importing it.
+		Binary:      "python",
+		VersionArgs: []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
+		Packages:    compilerPackages,
+		FreeGB:      25,
+		GPUCheck:    true,
 	}
-	return nil
+	needs := gpuNeeds{minDriverForCUDA13, 7.5}
+	if versionBefore(version, sglangCUDA13Since) {
+		b.Variant, needs = "cu12", gpuNeeds{minDriverForCUDA12, 7.0}
+	}
+	return pick("SGLang "+version, variant, []candidate{{b, needs, "SGLang " + version}}, h)
 }
 
 // Python engines need a C compiler at run time: Triton builds a small
@@ -150,35 +385,27 @@ func cudaGate(engine string, h discovery.Host) error {
 // that load fails long after the installation looked finished.
 var compilerPackages = map[string][]string{"deb": {"gcc"}, "rpm": {"gcc"}}
 
-func chooseVLLM(version string, h discovery.Host) (Build, error) {
-	if err := cudaGate("vLLM", h); err != nil {
-		return Build{}, err
+// versionBefore reports whether version a is older than b, comparing the
+// dotted numbers in turn ("0.25.1" before "0.26"). A part that is not a
+// number counts as zero.
+func versionBefore(a, b string) bool {
+	if b == "" {
+		return false
 	}
-	return Build{
-		Variant:      "cu13",
-		Python:       pythonForEngines,
-		Requirements: []string{"vllm==" + version},
-		Binary:       "vllm",
-		Packages:     compilerPackages,
-		FreeGB:       25,
-	}, nil
-}
-
-func chooseSGLang(version string, h discovery.Host) (Build, error) {
-	if err := cudaGate("SGLang", h); err != nil {
-		return Build{}, err
+	pa, pb := strings.Split(strings.TrimPrefix(a, "v"), "."), strings.Split(b, ".")
+	for i := 0; i < max(len(pa), len(pb)); i++ {
+		x, y := 0, 0
+		if i < len(pa) {
+			x = leadingInt(pa[i])
+		}
+		if i < len(pb) {
+			y = leadingInt(pb[i])
+		}
+		if x != y {
+			return x < y
+		}
 	}
-	// SGLang is launched as a module, so what the agent is given is the
-	// environment's interpreter, and the proof is importing it.
-	return Build{
-		Variant:      "cu13",
-		Python:       pythonForEngines,
-		Requirements: []string{"sglang==" + version},
-		Binary:       "python",
-		VersionArgs:  []string{"-c", "import sglang; print('sglang', sglang.__version__)"},
-		Packages:     compilerPackages,
-		FreeGB:       25,
-	}, nil
+	return false
 }
 
 // leadingInt is the number a version string starts with: 580 of "580.65.06".
@@ -193,17 +420,30 @@ func leadingInt(s string) int {
 	return n
 }
 
-// cudaForLlamaCpp is the CUDA build taken on an NVIDIA node. The release also
-// carries a matching cudart archive, so the node needs a driver and nothing
-// else — no toolkit.
-const cudaForLlamaCpp = "12.8"
+// --- llama.cpp ---
 
-func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
-	// The published Linux builds are built on Ubuntu, but nothing about them
-	// is Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which
-	// Debian 12 and RHEL 10 have and RHEL 9 does not. So the distribution's
-	// name decides nothing here. The build is tried everywhere, and ldd says
-	// whether this system can load it — in the words of the missing symbol.
+// llamaCUDA12Real are the GPUs llama.cpp's CUDA 12.8 build carries machine
+// code for. The rest of what it covers — 5.0, 6.1, 7.0, 7.5, 8.0, 9.0 — it
+// carries as PTX, which a driver older than the CUDA it came from cannot
+// compile.
+var llamaCUDA12Real = []float64{8.6, 8.9, 12.0}
+
+// llamaCppCandidates are the llama.cpp builds for a machine.
+//
+// On an NVIDIA node, the GPU builds best first: CUDA 13.3 (driver 580,
+// Turing and newer), CUDA 12.8 on x64 (Maxwell to Blackwell; driver 570 for
+// the GPUs it ships as PTX, 525 for those it ships as machine code), and
+// Vulkan, which runs on any card whose driver has Vulkan — the generations
+// CUDA builds no longer cover. Never the CPU build on an NVIDIA node: that is
+// a GPU placement served on the CPU, which is what installing checks for.
+// The CPU build where there is no NVIDIA GPU.
+//
+// The published Linux builds are built on Ubuntu, but nothing about them is
+// Ubuntu's: they need glibc 2.34 and the libstdc++ of GCC 12, which Debian 12
+// and RHEL 10 have and RHEL 9 does not. So the distribution's name decides
+// nothing here. The build is tried everywhere, and ldd says whether this
+// system can load it — in the words of the missing symbol.
+func llamaCppCandidates(version, variant string, h discovery.Host) ([]Build, error) {
 	var arch string
 	switch h.Arch {
 	case "amd64":
@@ -211,35 +451,53 @@ func chooseLlamaCpp(version string, h discovery.Host) (Build, error) {
 	case "arm64":
 		arch = "arm64"
 	default:
-		return Build{}, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
+		return nil, fmt.Errorf("llama.cpp publishes no Linux build for %s", orUnknown(h.Arch))
 	}
-
-	b := Build{
-		Variant: "cpu",
-		Assets:  []string{fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)},
-		// Everything else the build links is in the archive or in any current
-		// glibc system; OpenMP's runtime is the one a cloud image lacks, and
-		// the server does not start without it.
-		Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
-		Binary:   "llama-server",
-	}
-	if arch == "x64" {
-		// The archive carries a backend per CPU generation and loads the best
-		// one, down to SSE4.2. Below AVX2 it runs, several times slower.
-		b.CPUFlags = []string{"avx2"}
-	}
-	// The CUDA build is published for x64 at this version; on arm64 it is a
-	// newer CUDA that the driver may not support, so an arm64 GPU node runs
-	// the CPU build until someone decides otherwise.
-	if h.NVIDIADriver != "" && arch == "x64" {
-		name := fmt.Sprintf("bin-ubuntu-cuda-%s-x64.tar.gz", cudaForLlamaCpp)
-		b.Variant = "cuda-" + cudaForLlamaCpp
-		b.Assets = []string{
-			fmt.Sprintf("llama-%s-%s", version, name),
-			fmt.Sprintf("cudart-llama-%s-%s", version, name),
+	base := func(variant string, assets ...string) Build {
+		b := Build{
+			Variant: variant,
+			Assets:  assets,
+			// Everything else the build links is in the archive or in any
+			// current glibc system; OpenMP's runtime is the one a cloud image
+			// lacks, and the server does not start without it.
+			Packages: map[string][]string{"deb": {"libgomp1"}, "rpm": {"libgomp"}},
+			Binary:   "llama-server",
 		}
+		if arch == "x64" {
+			// The archive carries a backend per CPU generation and loads the
+			// best one, down to SSE4.2. Below AVX2 it runs, several times
+			// slower.
+			b.CPUFlags = []string{"avx2"}
+		}
+		return b
 	}
-	return b, nil
+	if h.NVIDIADriver == "" {
+		return pick("llama.cpp", variant, []candidate{
+			{base("cpu", fmt.Sprintf("llama-%s-bin-ubuntu-%s.tar.gz", version, arch)), gpuNeeds{}, "llama.cpp"},
+		}, h)
+	}
+	cuda := func(cv string) Build {
+		name := fmt.Sprintf("bin-ubuntu-cuda-%s-%s.tar.gz", cv, arch)
+		// The release carries the CUDA runtime beside the build, so the node
+		// needs a driver and nothing else — no toolkit.
+		b := base("cuda-"+cv, fmt.Sprintf("llama-%s-%s", version, name), fmt.Sprintf("cudart-llama-%s-%s", version, name))
+		b.GPUBackend = "libggml-cuda.so"
+		return b
+	}
+	vulkan := base("vulkan", fmt.Sprintf("llama-%s-bin-ubuntu-vulkan-%s.tar.gz", version, arch))
+	vulkan.GPUBackend = "libggml-vulkan.so"
+	vulkan.Packages = map[string][]string{"deb": {"libgomp1", "libvulkan1"}, "rpm": {"libgomp", "vulkan-loader"}}
+
+	cuda12Driver := 570
+	if slices.Contains(llamaCUDA12Real, h.MinComputeCapability) {
+		cuda12Driver = minDriverForCUDA12
+	}
+	cands := []candidate{{cuda("13.3"), gpuNeeds{minDriverForCUDA13, 7.5}, "llama.cpp built for CUDA 13.3"}}
+	if arch == "x64" {
+		cands = append(cands, candidate{cuda("12.8"), gpuNeeds{cuda12Driver, 5.0}, "llama.cpp built for CUDA 12.8"})
+	}
+	cands = append(cands, candidate{vulkan, gpuNeeds{}, "llama.cpp built for Vulkan"})
+	return pick("llama.cpp", variant, cands, h)
 }
 
 func orUnknown(s string) string {

internal/agent/supervisor/health.go

@@ -53,8 +53,8 @@ func NewHTTPHealth(timeout time.Duration, opts ...HTTPHealthOption) *HTTPHealth
 }
 
 // Ready reports whether the engine at baseURL is serving model.
-func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
-	if err := h.get(ctx, baseURL+h.healthPath, nil); err != nil {
+func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model, apiKey string) error {
+	if err := h.get(ctx, baseURL+h.healthPath, apiKey, nil); err != nil {
 		return fmt.Errorf("health: %w", err)
 	}
 	if !h.requireName || model == "" {
@@ -66,7 +66,7 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
 			ID string `json:"id"`
 		} `json:"data"`
 	}
-	if err := h.get(ctx, baseURL+h.modelsPath, &list); err != nil {
+	if err := h.get(ctx, baseURL+h.modelsPath, apiKey, &list); err != nil {
 		return fmt.Errorf("models: %w", err)
 	}
 	for _, d := range list.Data {
@@ -77,11 +77,14 @@ func (h *HTTPHealth) Ready(ctx context.Context, baseURL, model string) error {
 	return fmt.Errorf("engine is up but does not serve %q yet", model)
 }
 
-func (h *HTTPHealth) get(ctx context.Context, url string, out any) error {
+func (h *HTTPHealth) get(ctx context.Context, url, apiKey string, out any) error {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return err
 	}
+	if apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+apiKey)
+	}
 	resp, err := h.client.Do(req)
 	if err != nil {
 		return err

internal/agent/supervisor/health_test.go

@@ -39,7 +39,7 @@ func engineStub(t *testing.T, healthy bool, models ...string) *httptest.Server {
 
 func TestReadyWhenServingModel(t *testing.T) {
 	srv := engineStub(t, true, "llama-3.3-70b")
-	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b"); err != nil {
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", ""); err != nil {
 		t.Errorf("a serving engine was reported unready: %v", err)
 	}
 }
@@ -48,7 +48,7 @@ func TestNotReadyWhileLoading(t *testing.T) {
 	// The case that matters: the process is up but the weights are not loaded.
 	// Routing on /health alone would send traffic into a loading model.
 	srv := engineStub(t, false, "llama-3.3-70b")
-	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
 	if err == nil {
 		t.Fatal("an unhealthy engine was reported ready")
 	}
@@ -61,7 +61,7 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
 	// /health passes but the model is not listed: the engine is up and serving
 	// something else, or still registering.
 	srv := engineStub(t, true, "some-other-model")
-	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b")
+	err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "llama-3.3-70b", "")
 	if err == nil {
 		t.Fatal("a model that is not served was reported ready")
 	}
@@ -73,14 +73,33 @@ func TestNotReadyWhenModelMissing(t *testing.T) {
 func TestModelCheckCanBeSkipped(t *testing.T) {
 	srv := engineStub(t, true)
 	h := NewHTTPHealth(time.Second, WithoutModelCheck())
-	if err := h.Ready(t.Context(), srv.URL, "anything"); err != nil {
+	if err := h.Ready(t.Context(), srv.URL, "anything", ""); err != nil {
 		t.Errorf("the model check was not skipped: %v", err)
 	}
 }
 
 func TestUnreachableEngine(t *testing.T) {
 	h := NewHTTPHealth(200 * time.Millisecond)
-	if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m"); err == nil {
+	if err := h.Ready(t.Context(), "http://127.0.0.1:1", "m", ""); err == nil {
 		t.Error("an unreachable engine was reported ready")
 	}
 }
+
+func TestTheHealthCheckSendsTheEnginesKey(t *testing.T) {
+	// An engine started with a key refuses /v1/models without it, and a
+	// shard that is serving would read as never ready.
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer k1" {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		_, _ = w.Write([]byte(`{"data":[{"id":"m"}]}`))
+	}))
+	defer srv.Close()
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", "k1"); err != nil {
+		t.Errorf("with the key: %v", err)
+	}
+	if err := NewHTTPHealth(time.Second).Ready(t.Context(), srv.URL, "m", ""); err == nil {
+		t.Error("without the key the engine refused, and that read as ready")
+	}
+}

internal/agent/supervisor/supervisor.go

@@ -79,7 +79,7 @@ func (c *Config) normalize() {
 // supervisor can be tested without an HTTP engine.
 type HealthChecker interface {
 	// Ready reports whether the shard at baseURL serves model.
-	Ready(ctx context.Context, baseURL, model string) error
+	Ready(ctx context.Context, baseURL, model, apiKey string) error
 }
 
 // Shard is one supervised engine process.
@@ -411,7 +411,7 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
 			last := strings.Join(sh.Logs(10), "\n")
 			return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
 		case <-t.C:
-			if err := s.health.Ready(ctx, baseURL, model); err != nil {
+			if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
 				continue
 			}
 			sh.setState(Ready, nil)

internal/agent/supervisor/supervisor_test.go

@@ -21,7 +21,7 @@ import (
 // scriptedHealth reports ready once the flag is set.
 type scriptedHealth struct{ ready atomic.Bool }
 
-func (h *scriptedHealth) Ready(context.Context, string, string) error {
+func (h *scriptedHealth) Ready(context.Context, string, string, string) error {
 	if h.ready.Load() {
 		return nil
 	}
@@ -243,7 +243,9 @@ func TestWaitReadyTimeoutIncludesLogs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	time.Sleep(100 * time.Millisecond) // let the line be captured
+	eventually(t, 3*time.Second, "the line was never captured", func() bool {
+		return strings.Contains(strings.Join(sh.Logs(10), "\n"), "CUDA out of memory")
+	})
 
 	err = s.WaitReady(t.Context(), sh, "http://x", "m", 200*time.Millisecond)
 	if err == nil {
@@ -295,7 +297,10 @@ func TestLogsAreCaptured(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	time.Sleep(400 * time.Millisecond)
+	eventually(t, 3*time.Second, "not every line arrived", func() bool {
+		l := strings.Join(sh.Logs(10), "\n")
+		return strings.Contains(l, "line1") && strings.Contains(l, "line2") && strings.Contains(l, "line3")
+	})
 
 	logs := strings.Join(sh.Logs(10), "\n")
 	for _, want := range []string{"line1", "line2", "line3"} {
@@ -372,11 +377,11 @@ func TestStateFileRemovedOnExit(t *testing.T) {
 
 	sh, _ := s.Start(t.Context(), "a", engines.ShardSpec{}, shellCommand("exit 0"))
 	waitState(t, sh, Failed, 3*time.Second)
-	time.Sleep(50 * time.Millisecond)
-
-	if _, err := os.Stat(s.statePath("a")); !os.IsNotExist(err) {
-		t.Error("the state file outlived the process; a later scan would find a dead orphan")
-	}
+	eventually(t, 3*time.Second,
+		"the state file outlived the process; a later scan would find a dead orphan", func() bool {
+			_, err := os.Stat(s.statePath("a"))
+			return os.IsNotExist(err)
+		})
 }
 
 // --- state machine and ring buffer ---
@@ -743,7 +748,9 @@ func TestWaitAliveFailsOnEarlyExit(t *testing.T) {
 	if err := s.WaitAlive(t.Context(), sh); err == nil {
 		t.Fatal("a worker that died immediately was reported ready")
 	}
-	if d := time.Since(start); d > 1500*time.Millisecond {
+	// Below AliveGrace (5s), which is what waiting it out would take; the
+	// headroom is for a loaded runner, where 1.5s was sometimes not enough.
+	if d := time.Since(start); d > 4*time.Second {
 		t.Errorf("waited %v for a process that had already exited", d)
 	}
 }
@@ -827,3 +834,17 @@ func TestAWorkersOnlyProcessWaitsForTheLeader(t *testing.T) {
 		t.Errorf("the worker started after %v, before the leader was listening", d)
 	}
 }
+
+// eventually waits for cond, failing with msg if it does not hold within
+// limit. Sleeping a fixed time and then checking measured the machine the
+// test ran on, and a loaded CI runner under the race detector lost that race.
+func eventually(t *testing.T, limit time.Duration, msg string, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(limit)
+	for !cond() {
+		if time.Now().After(deadline) {
+			t.Fatal(msg)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+}

internal/agent/upgrade.go

@@ -0,0 +1,58 @@
+package agent
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+)
+
+// upgradeExit is the exit status after an upgrade: non-zero, so systemd's
+// Restart=on-failure starts the new binary, and distinct, so the journal says
+// why the agent went.
+const upgradeExit = 75
+
+// exitForUpgrade ends the process; a variable so a test can see it called
+// rather than have its own process end.
+var exitForUpgrade = func() { os.Exit(upgradeExit) }
+
+// upgradeTarget is the binary an upgrade replaces; a variable so a test
+// replaces a file of its own rather than the test binary running it.
+var upgradeTarget = os.Executable
+
+// upgrade replaces this agent's binary with a release and restarts into it.
+//
+// Engines are not touched: an agent restart leaves them running and adopts
+// them again, which is what makes upgrading a node something that can be done
+// while it serves.
+func (a *Agent) upgrade(ctx context.Context, m *llmgwv1.UpgradeAgent, out chan<- *llmgwv1.AgentMessage) {
+	reply := &llmgwv1.UpgradeReply{RequestId: m.GetRequestId(), Version: m.GetVersion()}
+	send := func() {
+		select {
+		case out <- &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Upgrade{Upgrade: reply}}:
+		case <-ctx.Done():
+		}
+	}
+	a.log.Info("upgrading", "to", m.GetVersion(), "from", a.cfg.Version)
+	self, err := upgradeTarget()
+	if err == nil {
+		dctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+		_, err = release.Install(dctx, nil, m.GetReleases(), m.GetVersion(), runtime.GOOS, runtime.GOARCH, self)
+		cancel()
+	}
+	if err != nil {
+		reply.Error = err.Error()
+		a.log.Error("upgrade failed; still running the old version", "to", m.GetVersion(), "err", err)
+		send()
+		return
+	}
+	send()
+	a.log.Info("upgraded; restarting into the new version", "to", m.GetVersion())
+	// A moment for the reply to leave before the stream goes.
+	time.Sleep(500 * time.Millisecond)
+	exitForUpgrade()
+}

internal/agent/upgrade_test.go

@@ -0,0 +1,78 @@
+package agent
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+)
+
+func TestAnAgentUpgradesItselfAndSaysSoBeforeRestarting(t *testing.T) {
+	body := []byte("the new agent")
+	sum := sha256.Sum256(body)
+	asset := release.AssetName("v9.9.9", runtime.GOOS, runtime.GOARCH)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v9.9.9/checksums.txt":
+			_, _ = w.Write([]byte(hex.EncodeToString(sum[:]) + "  " + asset + "\n"))
+		case "/v9.9.9/" + asset:
+			_, _ = w.Write(body)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+
+	bin := filepath.Join(t.TempDir(), "llmgw")
+	_ = os.WriteFile(bin, []byte("the old agent"), 0o755)
+	exited := make(chan struct{}, 1)
+	oldTarget, oldExit := upgradeTarget, exitForUpgrade
+	upgradeTarget = func() (string, error) { return bin, nil }
+	exitForUpgrade = func() { exited <- struct{}{} }
+	t.Cleanup(func() { upgradeTarget, exitForUpgrade = oldTarget, oldExit })
+
+	cfg := Config{Server: "127.0.0.1:7443", NodeID: "n1", WeightsCache: t.TempDir(), EnginesDir: t.TempDir()}
+	cfg.Superv.WorkDir = t.TempDir()
+	a, err := New(cfg, nil, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := make(chan *llmgwv1.AgentMessage, 1)
+	go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u1", Version: "v9.9.9", Releases: srv.URL}, out)
+
+	select {
+	case m := <-out:
+		if r := m.GetUpgrade(); r.GetRequestId() != "u1" || r.GetError() != "" {
+			t.Fatalf("reply %+v", r)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatal("no reply")
+	}
+	select {
+	case <-exited:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the agent did not restart into the new binary")
+	}
+	if b, _ := os.ReadFile(bin); string(b) != "the new agent" {
+		t.Errorf("binary holds %q", b)
+	}
+
+	// A version with no release answers with why, and does not restart.
+	go a.upgrade(t.Context(), &llmgwv1.UpgradeAgent{RequestId: "u2", Version: "v9.9.8", Releases: srv.URL}, out)
+	if r := (<-out).GetUpgrade(); r.GetError() == "" {
+		t.Error("an upgrade to a release that does not exist reported success")
+	}
+	select {
+	case <-exited:
+		t.Error("a failed upgrade restarted the agent")
+	case <-time.After(time.Second):
+	}
+}

internal/agent/weights/set.go

@@ -2,6 +2,8 @@ package weights
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -26,8 +28,9 @@ type manifest struct {
 	Model   string `json:"model"`
 	Version string `json:"version"`
 	Files   []struct {
-		Name  string `json:"name"`
-		Bytes int64  `json:"bytes"`
+		Name   string `json:"name"`
+		Bytes  int64  `json:"bytes"`
+		SHA256 string `json:"sha256"`
 	} `json:"files"`
 	Bytes int64 `json:"bytes"`
 }
@@ -87,6 +90,25 @@ func (f *Fetcher) ensureSet(ctx context.Context, source string) (string, error)
 		}
 	}
 
+	// Each file against the checksum the mirror took when it fetched it,
+	// before the set is called complete: a shard damaged on the way loads as
+	// a model that answers nonsense, not as an error.
+	for _, file := range m.Files {
+		if file.SHA256 == "" {
+			continue
+		}
+		target := filepath.Join(dir, file.Name)
+		got, err := fileSHA256(target)
+		if err != nil {
+			return "", err
+		}
+		if !strings.EqualFold(got, file.SHA256) {
+			_ = os.Remove(target)
+			return "", fmt.Errorf("%s%s: sha256 %s, the mirror has %s; removed, to be fetched again",
+				source, file.Name, got, file.SHA256)
+		}
+	}
+
 	if err := os.WriteFile(done, []byte(source+"\n"), 0o644); err != nil {
 		return "", err
 	}
@@ -108,6 +130,12 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
 		return manifest{}, fmt.Errorf("asking %s what it holds: %w", source, err)
 	}
 	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusConflict {
+		// Still downloading on the mirror, or a shard missing: said in the
+		// mirror's words, which name what is not there yet.
+		why, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
+		return manifest{}, fmt.Errorf("%s is not complete on the mirror yet: %s", source, strings.TrimSpace(string(why)))
+	}
 	if resp.StatusCode != http.StatusOK {
 		return manifest{}, fmt.Errorf("%s answered %d", source, resp.StatusCode)
 	}
@@ -127,3 +155,16 @@ func (f *Fetcher) manifestFor(ctx context.Context, source string) (manifest, err
 	}
 	return m, nil
 }
+
+func fileSHA256(path string) (string, error) {
+	f, err := os.Open(path) //nolint:gosec // a file this fetcher wrote
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}

internal/agent/weights/set_test.go

@@ -1,6 +1,8 @@
 package weights
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -165,3 +167,40 @@ func TestOnlyATrailingSlashMeansASet(t *testing.T) {
 		}
 	}
 }
+
+func TestASetFileThatDoesNotMatchTheMirrorsChecksumIsRefused(t *testing.T) {
+	good := "the shard as mirrored"
+	sum := sha256.Sum256([]byte(good))
+	served := "the shard damaged on the way"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/") {
+			fmt.Fprintf(w, `{"model":"m","version":"v","files":[{"name":"s.safetensors","bytes":%d,"sha256":%q}]}`,
+				len(served), hex.EncodeToString(sum[:]))
+			return
+		}
+		_, _ = w.Write([]byte(served))
+	}))
+	t.Cleanup(srv.Close)
+	f := &Fetcher{CacheDir: t.TempDir()}
+	if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err == nil || !strings.Contains(err.Error(), "sha256") {
+		t.Fatalf("a damaged shard was accepted: %v", err)
+	}
+	served = good
+	// The size differs, so it is fetched again; this time it matches.
+	if _, err := f.Ensure(t.Context(), srv.URL+"/m/v/"); err != nil {
+		t.Fatalf("the shard as mirrored: %v", err)
+	}
+}
+
+func TestASetStillDownloadingOnTheMirrorSaysSo(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		http.Error(w, "m/v: this version is not complete: still downloading model-00002-of-00002.safetensors", http.StatusConflict)
+	}))
+	t.Cleanup(srv.Close)
+	f := &Fetcher{CacheDir: t.TempDir()}
+	_, err := f.Ensure(t.Context(), srv.URL+"/m/v/")
+	if err == nil || !strings.Contains(err.Error(), "not complete on the mirror yet") ||
+		!strings.Contains(err.Error(), "model-00002-of-00002") {
+		t.Fatalf("got %v", err)
+	}
+}

internal/bench/bench.go

@@ -225,6 +225,7 @@ func runStep(ctx context.Context, client *http.Client, cfg Config, concurrency i
 	)
 
 	sampler := newSampler(cfg.MetricsURL, cfg.Engine)
+	sampler.apiKey = cfg.APIKey
 	sampler.start(ctx)
 
 	start := time.Now()

internal/bench/sampler.go

@@ -19,6 +19,7 @@ import (
 // Optional, because a run pointed at the gateway has no single engine to read.
 type sampler struct {
 	url    string
+	apiKey string
 	engine string
 	client *http.Client
 	cancel context.CancelFunc
@@ -65,6 +66,9 @@ func (s *sampler) sample(ctx context.Context) {
 	if err != nil {
 		return
 	}
+	if s.apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+s.apiKey)
+	}
 	resp, err := s.client.Do(req)
 	if err != nil {
 		return

internal/cli/backup.go

@@ -0,0 +1,330 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+// DefaultBackupDir is where backups are written and where the control plane
+// looks for the newest one.
+//
+// Its own directory rather than /var/backups/llmgw, which a PostgreSQL move
+// had already made for its own dumps, owned by postgres and closed to
+// everyone else — so the control plane could not see a backup landing in it.
+const DefaultBackupDir = "/var/backups/llmgw-server"
+
+func newBackupCmd() *cobra.Command {
+	var (
+		configPath string
+		etcDir     string
+		dir        string
+		keep       int
+	)
+	c := &cobra.Command{
+		Use:   "backup",
+		Short: "Write the store and /etc/llmgw to one archive, and keep the last few",
+		Long: "Everything a control plane cannot be rebuilt without: the store — keys,\n" +
+			"people, quotas, usage, the models added at runtime, the secret engine\n" +
+			"keys derive from — and /etc/llmgw, which holds the configuration, the\n" +
+			"store's address and the certificate authority's key.\n\n" +
+			"The archive holds that key, so it is written readable by root only. It\n" +
+			"is on the same disk as what it backs up: copy it elsewhere, or it does\n" +
+			"not survive the loss of this machine.\n\n" +
+			"Run daily by llmgw-backup.timer (`llmgw server backup`). The control plane publishes the time of\n" +
+			"the newest archive as llmgw_backup_last_success_timestamp_seconds, and\n" +
+			"BackupTooOld fires when it is more than a day old.",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			cfg, err := config.Load(configPath)
+			if err != nil {
+				return err
+			}
+			path, err := writeBackup(cmd.Context(), cfg.Auth.Store, etcDir, dir)
+			if err != nil {
+				return err
+			}
+			if why := unlistable(dir); why != "" {
+				fmt.Fprintf(cmd.ErrOrStderr(), "warning: %s; the control plane will report no backup "+
+					"and BackupTooOld will fire\n", why)
+			}
+			removed, err := pruneBackups(dir, keep)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "wrote %s", path)
+			if removed > 0 {
+				fmt.Fprintf(cmd.OutOrStdout(), "; removed %d older", removed)
+			}
+			fmt.Fprintln(cmd.OutOrStdout())
+			return nil
+		},
+	}
+	c.Flags().StringVar(&configPath, "config", "/etc/llmgw/server.yaml", "the control plane's configuration")
+	c.Flags().StringVar(&etcDir, "etc", "/etc/llmgw", "configuration directory to include")
+	c.Flags().StringVar(&dir, "dir", DefaultBackupDir, "where archives are written")
+	c.Flags().IntVar(&keep, "keep", 14, "how many archives to keep")
+	return c
+}
+
+const backupPrefix = "llmgw-backup-"
+
+// writeBackup writes one archive and returns its path. Written beside its
+// final name and renamed, so a backup interrupted halfway is never the newest
+// one the control plane reports.
+func writeBackup(ctx context.Context, storeAddr, etcDir, dir string) (string, error) {
+	// Listable by anyone, so the control plane — not root — can see when the
+	// newest archive was written. The archives themselves are root's alone.
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	host, _ := os.Hostname()
+	name := backupPrefix + host + "-" + time.Now().UTC().Format("20060102T150405Z") + ".tar.gz"
+	final := filepath.Join(dir, name)
+	part := final + ".part"
+
+	f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600)
+	if err != nil {
+		return "", err
+	}
+	ok := false
+	defer func() {
+		if !ok {
+			_ = os.Remove(part)
+		}
+	}()
+	gz := gzip.NewWriter(f)
+	tw := tar.NewWriter(gz)
+
+	manifest := fmt.Sprintf("llmgw %s\nhost %s\ntime %s\n", version.Version, host,
+		time.Now().UTC().Format(time.RFC3339))
+	if err := addBytes(tw, "MANIFEST", []byte(manifest)); err != nil {
+		return "", err
+	}
+	if storeAddr != "" {
+		if err := addStore(ctx, tw, storeAddr, dir); err != nil {
+			return "", fmt.Errorf("the store: %w", err)
+		}
+	}
+	if err := addTree(tw, etcDir, "etc/llmgw"); err != nil {
+		return "", fmt.Errorf("%s: %w", etcDir, err)
+	}
+	if err := errors.Join(tw.Close(), gz.Close(), f.Sync(), f.Close()); err != nil {
+		return "", err
+	}
+	if err := os.Rename(part, final); err != nil {
+		return "", err
+	}
+	ok = true
+	return final, nil
+}
+
+// addStore dumps the store into the archive.
+func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error {
+	addr, err := store.ResolveAddress(addr)
+	if err != nil {
+		return err
+	}
+	// Inside a directory only its owner can enter. The archive directory is
+	// listable on purpose, and VACUUM INTO creates its file with the default
+	// mode: made there directly, a full copy of the store — password hashes,
+	// the engine-key secret — sat readable by any local user for as long as
+	// the backup took.
+	tmpDir, err := os.MkdirTemp(scratch, ".store-")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpPath := filepath.Join(tmpDir, "store")
+
+	name := "store.sqlite"
+	if store.IsPostgres(addr) {
+		name = "store.pgdump"
+		if err := pgDump(ctx, addr, tmpPath); err != nil {
+			return err
+		}
+	} else {
+		db, err := store.Open(ctx, addr)
+		if err != nil {
+			return err
+		}
+		err = db.SnapshotTo(ctx, tmpPath)
+		_ = db.Close()
+		if err != nil {
+			return err
+		}
+	}
+	return addFile(tw, tmpPath, name)
+}
+
+// pgDump runs pg_dump in its custom format, which pg_restore can restore
+// selectively. The connection is given through the environment, not the
+// command line: a DSN carries the password, and every user on the machine
+// can read another process's arguments.
+func pgDump(ctx context.Context, dsn, out string) error {
+	vars, err := pgEnv(dsn)
+	if err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, "pg_dump", "--format=custom", "--no-owner", "--file="+out) //nolint:gosec // fixed binary, our own file
+	cmd.Env = append(os.Environ(), vars...)
+	if b, err := cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("pg_dump: %v: %s", err, strings.TrimSpace(string(b)))
+	}
+	return nil
+}
+
+// pgEnv turns a store address into libpq's environment. Several hosts — the
+// members of a replicated cluster — become libpq's own lists in PGHOST and
+// PGPORT; read as one URL host, the list was one name libpq could not
+// resolve, and a backup worked only by falling through to the second.
+func pgEnv(dsn string) ([]string, error) {
+	hosts, ports, u, err := store.SplitPostgres(dsn)
+	if err != nil {
+		return nil, err
+	}
+	var env []string
+	set := func(k, v string) {
+		if v != "" {
+			env = append(env, k+"="+v)
+		}
+	}
+	set("PGHOST", strings.Join(hosts, ","))
+	set("PGPORT", strings.Join(ports, ","))
+	set("PGUSER", u.User.Username())
+	if pw, ok := u.User.Password(); ok {
+		set("PGPASSWORD", pw)
+	}
+	set("PGDATABASE", strings.TrimPrefix(u.Path, "/"))
+	set("PGSSLMODE", u.Query().Get("sslmode"))
+	// A standby if there is one, else whichever answers — not the address's
+	// read-write: a dump only reads, it need not load the primary, and one
+	// that insists on the primary fails exactly while the cluster is failing
+	// over.
+	if len(hosts) > 1 {
+		set("PGTARGETSESSIONATTRS", "prefer-standby")
+	}
+	return env, nil
+}
+
+func addTree(tw *tar.Writer, root, as string) error {
+	return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		rel, _ := filepath.Rel(root, p)
+		name := filepath.ToSlash(filepath.Join(as, rel))
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+		switch {
+		case d.IsDir():
+			return tw.WriteHeader(&tar.Header{Name: name + "/", Mode: int64(info.Mode().Perm()),
+				Typeflag: tar.TypeDir, ModTime: info.ModTime()})
+		case info.Mode().IsRegular():
+			return addFile(tw, p, name)
+		}
+		return nil // sockets, links: not configuration
+	})
+}
+
+func addFile(tw *tar.Writer, path, name string) error {
+	f, err := os.Open(path) //nolint:gosec // files we chose to back up
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: int64(info.Mode().Perm()),
+		Size: info.Size(), ModTime: info.ModTime(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err = io.Copy(tw, f)
+	return err
+}
+
+func addBytes(tw *tar.Writer, name string, b []byte) error {
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b)),
+		ModTime: time.Now(), Typeflag: tar.TypeReg}); err != nil {
+		return err
+	}
+	_, err := tw.Write(b)
+	return err
+}
+
+// pruneBackups keeps the newest keep archives. Only files this command
+// wrote are considered, so nothing else in the directory is ever removed.
+func pruneBackups(dir string, keep int) (int, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return 0, err
+	}
+	var names []string
+	for _, e := range entries {
+		if n := e.Name(); strings.HasPrefix(n, backupPrefix) && strings.HasSuffix(n, ".tar.gz") {
+			names = append(names, n)
+		}
+	}
+	// The name carries a UTC timestamp, so name order is age order.
+	slices.Sort(names)
+	removed := 0
+	for len(names) > max(keep, 1) {
+		if err := os.Remove(filepath.Join(dir, names[0])); err != nil {
+			return removed, err
+		}
+		names = names[1:]
+		removed++
+	}
+	return removed, nil
+}
+
+// NewestBackup is the modification time of the newest complete archive in
+// dir, or zero when there is none.
+func NewestBackup(dir string) time.Time {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return time.Time{}
+	}
+	var newest time.Time
+	for _, e := range entries {
+		if n := e.Name(); !strings.HasPrefix(n, backupPrefix) || !strings.HasSuffix(n, ".tar.gz") {
+			continue
+		}
+		if info, err := e.Info(); err == nil && info.ModTime().After(newest) {
+			newest = info.ModTime()
+		}
+	}
+	return newest
+}
+
+// unlistable says why the control plane, which does not run as root, could
+// not see the archives in dir, or "" when it can.
+func unlistable(dir string) string {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return err.Error()
+	}
+	if info.Mode().Perm()&0o005 != 0o005 {
+		return fmt.Sprintf("%s is %v, so only its owner can list it", dir, info.Mode().Perm())
+	}
+	return ""
+}

internal/cli/backup_test.go

@@ -0,0 +1,164 @@
+package cli
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func TestABackupHoldsTheStoreAndTheConfiguration(t *testing.T) {
+	tmp := t.TempDir()
+	dbPath := filepath.Join(tmp, "store.db")
+	db, err := store.Open(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secret, err := db.ClusterSecret(t.Context(), "engine-key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = db.Close()
+
+	etc := filepath.Join(tmp, "etc")
+	_ = os.MkdirAll(filepath.Join(etc, "pki"), 0o700)
+	_ = os.WriteFile(filepath.Join(etc, "server.yaml"), []byte("listen: :8080\n"), 0o644)
+	_ = os.WriteFile(filepath.Join(etc, "pki", "ca-key.pem"), []byte("KEY"), 0o600)
+
+	out := filepath.Join(tmp, "backups")
+	path, err := writeBackup(t.Context(), dbPath, etc, out)
+	if err != nil {
+		t.Fatal(err)
+	}
+	info, _ := os.Stat(path)
+	if info.Mode().Perm() != 0o600 {
+		t.Errorf("the archive holds the CA's key and is %v", info.Mode().Perm())
+	}
+
+	files := map[string][]byte{}
+	f, _ := os.Open(path)
+	defer f.Close()
+	gz, err := gzip.NewReader(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tr := tar.NewReader(gz)
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		b, _ := io.ReadAll(tr)
+		files[h.Name] = b
+	}
+	for _, want := range []string{"MANIFEST", "store.sqlite", "etc/llmgw/server.yaml", "etc/llmgw/pki/ca-key.pem"} {
+		if _, ok := files[want]; !ok {
+			t.Errorf("%s is not in the archive", want)
+		}
+	}
+
+	// And the store in it is a working store, with what was in it.
+	restored := filepath.Join(tmp, "restored.db")
+	_ = os.WriteFile(restored, files["store.sqlite"], 0o600)
+	rdb, err := store.Open(t.Context(), restored)
+	if err != nil {
+		t.Fatalf("the backed-up store does not open: %v", err)
+	}
+	defer rdb.Close()
+	if got, _ := rdb.ClusterSecret(t.Context(), "engine-key"); got != secret {
+		t.Error("the restored store lost the engine key secret; every engine would be locked out")
+	}
+}
+
+func TestOldBackupsArePrunedAndTheNewestIsFound(t *testing.T) {
+	dir := t.TempDir()
+	for i, stamp := range []string{"20260101T000000Z", "20260102T000000Z", "20260103T000000Z"} {
+		p := filepath.Join(dir, backupPrefix+"h-"+stamp+".tar.gz")
+		_ = os.WriteFile(p, []byte("x"), 0o600)
+		mt := time.Date(2026, 1, 1+i, 0, 0, 0, 0, time.UTC)
+		_ = os.Chtimes(p, mt, mt)
+	}
+	_ = os.WriteFile(filepath.Join(dir, "someone-elses-file"), []byte("x"), 0o600)
+	_ = os.WriteFile(filepath.Join(dir, backupPrefix+"h-20260104T000000Z.tar.gz.part"), []byte("x"), 0o600)
+
+	removed, err := pruneBackups(dir, 2)
+	if err != nil || removed != 1 {
+		t.Fatalf("removed %d, %v", removed, err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "someone-elses-file")); err != nil {
+		t.Error("a file the backup did not write was removed")
+	}
+	newest := NewestBackup(dir)
+	if !newest.Equal(time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC)) {
+		t.Errorf("newest %v; an interrupted .part must not count", newest)
+	}
+	if !NewestBackup(filepath.Join(dir, "none")).IsZero() {
+		t.Error("a missing directory reported a backup")
+	}
+}
+
+func TestABackupDirectoryTheControlPlaneCannotListIsSaid(t *testing.T) {
+	dir := t.TempDir()
+	_ = os.Chmod(dir, 0o700)
+	if unlistable(dir) == "" {
+		t.Error("a directory only its owner can list was taken as visible")
+	}
+	_ = os.Chmod(dir, 0o755)
+	if why := unlistable(dir); why != "" {
+		t.Errorf("a listable directory: %s", why)
+	}
+}
+
+func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) {
+	// The snapshot is made before it is archived. Made straight into the
+	// listable archive directory, it was readable by anyone for that time.
+	tmp := t.TempDir()
+	dbPath := filepath.Join(tmp, "store.db")
+	db, err := store.Open(t.Context(), dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = db.Close()
+	out := filepath.Join(tmp, "backups")
+	if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil {
+		t.Fatal(err)
+	}
+	entries, _ := os.ReadDir(out)
+	for _, e := range entries {
+		if strings.HasPrefix(e.Name(), ".store") {
+			t.Errorf("a scratch copy of the store was left behind: %s", e.Name())
+		}
+	}
+}
+
+func TestAStoreOfSeveralHostsIsGivenToPgDumpAsLibpqLists(t *testing.T) {
+	// The address of a Patroni cluster: url.Parse read its host as one name,
+	// and pg_dump reached the database only by falling through to the second.
+	env, err := pgEnv("postgres://llmgw:s3cret@10.0.0.149:5432,10.0.0.150/llmgw?sslmode=disable&target_session_attrs=read-write")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := strings.Join(env, " ")
+	for _, want := range []string{"PGHOST=10.0.0.149,10.0.0.150", "PGPORT=5432,5432", "PGUSER=llmgw",
+		"PGPASSWORD=s3cret", "PGDATABASE=llmgw", "PGSSLMODE=disable", "PGTARGETSESSIONATTRS=prefer-standby"} {
+		if !strings.Contains(got, want) {
+			t.Errorf("%s is not in %s", want, got)
+		}
+	}
+	one, err := pgEnv("postgres://u:p@db.internal:6432/llmgw")
+	if err != nil || !strings.Contains(strings.Join(one, " "), "PGHOST=db.internal PGPORT=6432") {
+		t.Errorf("one host: %v, %v", one, err)
+	}
+	if _, err := pgEnv("postgres://u:p@a:5432,:5432/x"); err == nil {
+		t.Error("an empty host in the list was accepted")
+	}
+}

internal/cli/bench.go

@@ -41,6 +41,11 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			cfg.MetricsURL = metrics
 			cfg.Engine = engine
 			cfg.APIKey = key
+			// A replica's engine demands its own key. Asked for rather than
+			// typed: the admin API knows which replica answers at the target.
+			if cfg.APIKey == "" {
+				cfg.APIKey = replicaKey(cmd, client, target)
+			}
 			if len(steps) > 0 {
 				cfg.Concurrency = steps
 			}
@@ -227,3 +232,18 @@ func ms(d time.Duration) string {
 		return d.Round(10 * time.Millisecond).String()
 	}
 }
+
+// replicaKey is the engine key of the replica at target, or "" when target is
+// not a replica (the gateway, say) or the admin API cannot be asked.
+func replicaKey(cmd *cobra.Command, client func() *ctlClient, target string) string {
+	var reps []adminapi.ReplicaView
+	if err := client().get(cmd.Context(), "/api/v1/replicas", &reps); err != nil {
+		return ""
+	}
+	for _, rep := range reps {
+		if sameEndpoint(rep.Endpoint, target) {
+			return rep.EngineKey
+		}
+	}
+	return ""
+}

internal/cli/ctl.go

@@ -67,7 +67,9 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	if err != nil {
 		return err
 	}
-	if body != nil {
+	// On every change, body or not: the admin listener refuses anything else,
+	// because a browser can send a bodiless POST across origins unasked.
+	if method != http.MethodGet {
 		req.Header.Set("Content-Type", "application/json")
 	}
 	// Who made the change, kept with a stored model so a catalog that grew
@@ -81,7 +83,10 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != http.StatusOK {
+	// Any 2xx. The API answers a change it has queued rather than made — a
+	// restart — with 202, and treating that as a failure printed
+	// "Error: 202 Accepted" over a request that had worked.
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		var e struct {
 			Error string `json:"error"`
 		}
@@ -117,9 +122,11 @@ func newCtlCmd() *cobra.Command {
 	cmd.AddCommand(
 		newNodesCmd(client, &asJSON),
 		newLogsCmd(client, &asJSON),
+		newRequestCmd(client, &asJSON),
 		newModelsCmd(client, &asJSON),
 		newReplicasCmd(client, &asJSON),
 		newExplainCmd(client, &asJSON),
+		newFabricTestCmd(client, &asJSON),
 		newNodeStateCmd(client, "cordon", "cordoned",
 			"Stop placing new work on a node; what runs there keeps running"),
 		newNodeStateCmd(client, "uncordon", "ready",
@@ -130,12 +137,17 @@ func newCtlCmd() *cobra.Command {
 		// apply to it; it lives here because this is where operators look.
 		newBenchCmd(client, &asJSON),
 		newStatusCmd(client, &asJSON),
-		newModelCmd(client),
+		newModelCmd(client, &asJSON),
 		newScaleCmd(client),
+		newRestartCmd(client),
+		newRetryCmd(client),
+		newUpgradeCmd(client),
 		newEngineCheckCmd(),
+		newEngineBuildCmd(client),
 		newToolCheckCmd(),
 		newWeightsCmd(client, &asJSON),
 		newCapacityCmd(client, &asJSON),
+		newQuotaCmd(client, &asJSON),
 		newUsageCmd(client, &asJSON),
 		newCertCmd(),
 	)
@@ -241,6 +253,14 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 // actually served, never what is merely catalogued.
 func modelStatus(m adminapi.ModelView) string {
 	switch {
+	case m.Parked:
+		return "parked"
+	case m.External != "":
+		return "external: " + m.External
+	case m.HeldFor != "":
+		return "gave room to " + m.HeldFor
+	case m.Unplaceable:
+		return "cannot place"
 	case m.Min == 0 && m.Total == 0:
 		return "stopped"
 	case m.Ready >= m.Min && m.Min > 0:
@@ -309,17 +329,27 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			}
 
 			w := tab()
-			fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tALIASES\tPROFILES\tFALLBACKS")
+			fmt.Fprintln(w, "MODEL\tSERVING\tREPLICAS\tSTATUS\tCLASS\tRESERVES\tALIASES\tPROFILES\tFALLBACKS")
+			var warnings []string
 			for _, m := range models {
 				serving := "no"
 				if m.Ready > 0 {
 					serving = "yes"
 				}
-				fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\n",
-					m.Name, serving, m.Ready, m.Min, modelStatus(m),
+				fmt.Fprintf(w, "%s\t%s\t%d/%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
+					m.Name, serving, m.Ready, m.Min, modelStatus(m), m.Class, reserves(m),
 					join(m.Aliases), join(m.Profiles), join(m.Fallbacks))
+				for _, x := range m.Warnings {
+					warnings = append(warnings, m.Name+": "+x)
+				}
 			}
-			return w.Flush()
+			if err := w.Flush(); err != nil {
+				return err
+			}
+			for _, x := range warnings {
+				fmt.Println("warning:", x)
+			}
+			return nil
 		},
 	}
 }
@@ -367,6 +397,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 				if r.Stale {
 					state += " (stale profile)"
 				}
+				if r.Restart != "" {
+					state += " (restart " + r.Restart + ")"
+				}
 				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
 					r.ID, r.Model, state, r.Pool, join(nodes),
 					(time.Duration(r.AgeSec) * time.Second).String(),
@@ -471,8 +504,12 @@ func newNodeStateCmd(client func() *ctlClient, use, state, short string) *cobra.
 
 // --- output helpers ---
 
-func tab() *tabwriter.Writer {
-	return tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+func tab() *tabwriter.Writer { return tabTo(os.Stdout) }
+
+// tabTo is tab writing where it is told: a summary drawn into the console
+// went to the terminal underneath it instead, every three seconds.
+func tabTo(w io.Writer) *tabwriter.Writer {
+	return tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
 }
 
 func emitJSON(v any) error {
@@ -494,3 +531,22 @@ func dash(s string) string {
 	}
 	return s
 }
+
+// reserves is what one replica of a model holds of a node, with what it was
+// measured using beside it: "6G 2c (5.1G used)".
+func reserves(m adminapi.ModelView) string {
+	if m.MemoryGB == 0 && m.CPUCores == 0 {
+		return "-"
+	}
+	out := fmt.Sprintf("%dG", m.MemoryGB)
+	if m.CPUCores > 0 {
+		out += fmt.Sprintf(" %dc", m.CPUCores)
+	}
+	if m.MeasuredMemoryGB > 0 {
+		out += fmt.Sprintf(" (%.1fG used)", m.MeasuredMemoryGB)
+	}
+	if len(m.Warnings) > 0 {
+		out += " !"
+	}
+	return out
+}

internal/cli/ctl_fabrictest.go

@@ -0,0 +1,75 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"text/tabwriter"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newFabricTestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	var (
+		fabric  string
+		seconds int
+	)
+	c := &cobra.Command{
+		Use:   "fabric-test NODE-A NODE-B",
+		Short: "Measure RDMA bandwidth between two nodes, rail by rail",
+		Long: `Runs ib_write_bw from each of NODE-A's adapters on a fabric to the same
+adapter on NODE-B — mlx5_0 to mlx5_0, and so on, the way NCCL pairs them —
+one rail at a time, and grades each against its link's rate: ok at 80% or
+more, slow at 50%, fail below or when nothing got through.
+
+Every check on one node can pass while two nodes still cannot reach each
+other: a cable into the wrong switch, a partition key, a RoCE network with no
+route. A replica spanning them then hangs on its first all-reduce. Run this
+before placing one.
+
+It goes through the agents, so it needs no ssh between nodes. perftest is
+installed on a node the first time it is asked. The traffic is real and
+takes a few seconds a rail; run it when the fabric is quiet.`,
+		Example: `  llmgw ctl fabric-test gpu01 gpu02
+  llmgw ctl fabric-test gpu01 gpu02 --fabric ib-compute --seconds 10`,
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			body, _ := json.Marshal(map[string]any{"a": args[0], "b": args[1], "fabric": fabric, "seconds": seconds})
+			cl := client()
+			// Rails one after another, each for its seconds and a settle; the
+			// client's usual timeout is for questions, not measurements.
+			cl.http.Timeout = 30 * time.Minute
+			var v adminapi.FabricTestView
+			if err := cl.post(cmd.Context(), "/api/v1/fabric-test", body, &v); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "%s → %s on %s, %s\n\n", v.A, v.B, v.Fabric, v.Took)
+			w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+			fmt.Fprintln(w, "RAIL\tLINK\tMEASURED\tVERDICT\tDETAIL")
+			for _, r := range v.Rails {
+				link := "-"
+				if r.RateGbps > 0 {
+					link = fmt.Sprintf("%d Gb/s", r.RateGbps)
+				}
+				fmt.Fprintf(w, "%s → %s\t%s\t%.1f Gb/s\t%s\t%s\n", r.A, r.B, link, r.Gbps, r.Verdict, r.Detail)
+			}
+			_ = w.Flush()
+			for _, n := range v.Notes {
+				fmt.Fprintf(out, "\nnote: %s\n", n)
+			}
+			if !v.OK {
+				return fmt.Errorf("not every rail passed")
+			}
+			return nil
+		},
+	}
+	c.Flags().StringVar(&fabric, "fabric", "", "the fabric to test (default: the one the two nodes share)")
+	c.Flags().IntVar(&seconds, "seconds", 3, "how long each rail runs, up to 30")
+	return c
+}

internal/cli/ctl_request.go

@@ -0,0 +1,45 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	return &cobra.Command{
+		Use:   "request ID",
+		Short: "What one request went through: replica, queue, first token, tokens",
+		Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
+			"own. Given one, this says which replica answered, how long admission held\n" +
+			"it, how long the first token took and what it cost — the answer to \"why\n" +
+			"was this slow\". The access log has the same fields under \"request\".",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var v adminapi.RequestView
+			if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
+			}
+			model := v.Model
+			if v.ServedModel != "" && v.ServedModel != v.Model {
+				model += " (served by " + v.ServedModel + ")"
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "request   %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
+			fmt.Fprintf(out, "model     %s\n", model)
+			fmt.Fprintf(out, "replica   %s\n", or(v.Replica, "- (none answered)"))
+			fmt.Fprintf(out, "status    %d\n", v.Status)
+			fmt.Fprintf(out, "time      %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
+			fmt.Fprintf(out, "tokens    %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
+			fmt.Fprintf(out, "who       key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
+			return nil
+		},
+	}
+}

internal/cli/ctl_restart.go

@@ -0,0 +1,81 @@
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newRestartCmd(client func() *ctlClient) *cobra.Command {
+	var (
+		models  []string
+		all     bool
+		inPlace bool
+	)
+	c := &cobra.Command{
+		Use:   "restart [REPLICA...]",
+		Short: "Replace replicas with new ones, starting each replacement before stopping the old",
+		Long: "How a running engine picks up what only applies at start: an engine key,\n" +
+			"an edited profile, a new engine version.\n\n" +
+			"Each replacement is placed beside the replica it replaces, and the old one\n" +
+			"keeps serving until the new one is ready. Replicas of one model are\n" +
+			"replaced one at a time.\n\n" +
+			"Where there is no room for a second copy and the old replica is all that\n" +
+			"is in the way — one GPU, every card that fits held — it is stopped at once\n" +
+			"and the new one starts in its place; a model with only that replica is\n" +
+			"unserved while it loads. Where the new one would not fit even then — a\n" +
+			"profile edited to need more than any node has — the restart is given up\n" +
+			"and the old replica keeps serving: llmgw ctl replicas says so, and why.\n\n" +
+			"--in-place stops each replica first, whatever room there is.\n\n" +
+			"  llmgw ctl restart qwen2.5-7b-3f2a91c0\n" +
+			"  llmgw ctl restart --model qwen2.5-7b\n" +
+			"  llmgw ctl restart --all",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 && len(models) == 0 && !all {
+				return errors.New("name replicas, --model, or --all")
+			}
+			body, _ := json.Marshal(adminapi.RestartRequest{Replicas: args, Models: models, All: all, InPlace: inPlace})
+			var res adminapi.RestartResult
+			if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
+				return err
+			}
+			fmt.Printf("%d replica(s) queued for restart; watch with: llmgw ctl replicas\n", len(res.Queued))
+			return nil
+		},
+	}
+	c.Flags().StringArrayVar(&models, "model", nil, "every replica of this model; repeatable")
+	c.Flags().BoolVar(&all, "all", false, "every ready replica in the cluster")
+	c.Flags().BoolVar(&inPlace, "in-place", false, "stop each replica before starting its replacement")
+	return c
+}
+
+func newRetryCmd(client func() *ctlClient) *cobra.Command {
+	return &cobra.Command{
+		Use:   "retry MODEL",
+		Short: "Try a parked model again, and clear its backoff",
+		Long: "A model whose replicas fail every time they start is parked after five\n" +
+			"failures in a row: it is not placed again until its definition changes\n" +
+			"or this is run. Run it once the cause — the weights, the engine, the\n" +
+			"profile — has been dealt with; `llmgw ctl status` lists parked models with\n" +
+			"their last error.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var res struct {
+				WasParked bool `json:"was_parked"`
+			}
+			if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/retry", []byte(`{}`), &res); err != nil {
+				return err
+			}
+			if res.WasParked {
+				fmt.Printf("%s unparked; it is placed on the next pass\n", args[0])
+			} else {
+				fmt.Printf("%s was not parked; its backoff is cleared\n", args[0])
+			}
+			return nil
+		},
+	}
+}

internal/cli/ctl_status_test.go

@@ -0,0 +1,86 @@
+package cli
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"github.com/spf13/cobra"
+)
+
+func TestAQueuedChangeIsNotReportedAsAnError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusAccepted)
+		_, _ = w.Write([]byte(`{"queued":["r1"]}`))
+	}))
+	defer srv.Close()
+	var out struct {
+		Queued []string `json:"queued"`
+	}
+	if err := newCtlClient(srv.URL).post(t.Context(), "/api/v1/restart", []byte(`{}`), &out); err != nil {
+		t.Fatalf("202 read as a failure: %v", err)
+	}
+	if len(out.Queued) != 1 {
+		t.Errorf("the body of a 202 was not read: %+v", out)
+	}
+}
+
+func TestStatusSaysWhoLeadsTheStoreAndWhetherTheRestKeepUp(t *testing.T) {
+	render := func(s adminapi.Summary) string {
+		cmd := &cobra.Command{}
+		var b bytes.Buffer
+		cmd.SetOut(&b)
+		printSummary(cmd, s)
+		return b.String()
+	}
+	out := render(adminapi.Summary{Store: []adminapi.StoreMember{
+		{Name: "llmgw-ctl02", Role: "replica", State: "streaming", Lag: 2048},
+		{Name: "llmgw-pg01", Role: "leader", State: "running"},
+	}})
+	if !strings.Contains(out, "STORE     llmgw-ctl02 streaming (2 KB behind), llmgw-pg01 leads") {
+		t.Errorf("status:\n%s", out)
+	}
+	out = render(adminapi.Summary{Store: []adminapi.StoreMember{{Name: "llmgw-pg01", Role: "replica", State: "stopped"}}})
+	if !strings.Contains(out, "NO LEADER") {
+		t.Errorf("a store with no leader was not called out:\n%s", out)
+	}
+	if out := render(adminapi.Summary{}); strings.Contains(out, "STORE") {
+		t.Error("a single database got a store line")
+	}
+}
+
+func TestTheSummaryIsWrittenWhereItIsToldNotToTheTerminal(t *testing.T) {
+	// The console draws the summary into its panel. Its GPU table went to
+	// the terminal instead, through the full screen, every three seconds —
+	// on a cluster with a GPU, which the test cluster did not have.
+	r, w, err := os.Pipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+	stdout := os.Stdout
+	os.Stdout = w
+	var b bytes.Buffer
+	cmd := &cobra.Command{}
+	cmd.SetOut(&b)
+	printSummary(cmd, adminapi.Summary{
+		GPUs:         []adminapi.ClassUse{{Class: "6000-24g", Total: 1, Used: 1}},
+		Fabrics:      []adminapi.FabricUse{{Fabric: "ib-a", Nodes: 2}},
+		IdleReplicas: []adminapi.IdleReplica{{ID: "r1", Model: "m", IdleSec: 3600, GPUs: 1}},
+	})
+	os.Stdout = stdout
+	_ = w.Close()
+	leaked, _ := io.ReadAll(r)
+	if len(leaked) > 0 {
+		t.Errorf("written to the terminal: %q", leaked)
+	}
+	for _, want := range []string{"6000-24g", "ib-a", "r1"} {
+		if !strings.Contains(b.String(), want) {
+			t.Errorf("the summary lacks %q:\n%s", want, b.String())
+		}
+	}
+}

internal/cli/ctl_upgrade.go

@@ -0,0 +1,201 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"runtime"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+func newUpgradeCmd(client func() *ctlClient) *cobra.Command {
+	var (
+		nodes     []string
+		skipCP    bool
+		releases  string
+		perNode   time.Duration
+		serverSvc string
+	)
+	c := &cobra.Command{
+		Use:   "upgrade VERSION",
+		Short: "Upgrade this control plane, then every node, one at a time",
+		Long: "Moves the cluster to a release: v0.10.4, or a candidate like v0.10.4-rc.2.\n\n" +
+			"This machine first — run it with sudo on the control plane, since it\n" +
+			"replaces /usr/local/bin/llmgw and restarts llmgw-server. Then each node, in\n" +
+			"turn: its agent fetches the release for its own platform, checks it\n" +
+			"against the release's checksums, swaps it in and restarts. Engines keep\n" +
+			"serving through all of it and are adopted again; no ssh into the nodes is\n" +
+			"needed.\n\n" +
+			"A node counts as done when it is back on the new version, ready, with as\n" +
+			"many ready replicas as before. One that is not within --timeout stops the\n" +
+			"upgrade there, so a bad release reaches one node rather than all of them.\n" +
+			"Nodes already on the version are skipped, so running it again resumes.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := args[0]
+			if !release.Tag.MatchString(v) {
+				return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
+			}
+			out := cmd.OutOrStdout()
+			ctx := cmd.Context()
+			if !skipCP {
+				if err := upgradeControlPlane(ctx, out, client, v, releases, serverSvc); err != nil {
+					return err
+				}
+			}
+			return upgradeNodes(ctx, out, client, v, nodes, perNode)
+		},
+	}
+	c.Flags().StringSliceVar(&nodes, "nodes", nil, "only these nodes (default: all)")
+	c.Flags().BoolVar(&skipCP, "skip-control-plane", false, "leave this machine as it is and upgrade only nodes")
+	c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from, for this machine")
+	c.Flags().DurationVar(&perNode, "timeout", 3*time.Minute, "how long a node may take to come back")
+	c.Flags().StringVar(&serverSvc, "service", "llmgw-server", "this machine's systemd unit")
+	return c
+}
+
+func upgradeControlPlane(ctx context.Context, out io.Writer, client func() *ctlClient, v, releases, svc string) error {
+	if version.Version == v {
+		fmt.Fprintf(out, "this machine already runs %s\n", v)
+		return nil
+	}
+	if os.Geteuid() != 0 {
+		return errors.New("replacing this machine's binary needs root: run it with sudo, " +
+			"or pass --skip-control-plane to upgrade only the nodes")
+	}
+	self, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(out, "== this machine: %s -> %s\n", version.Version, v)
+	backup, err := release.Install(ctx, nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(out, "   installed, checksum verified; the old binary is %s\n", backup)
+	var before []adminapi.NodeView
+	_ = client().get(ctx, "/api/v1/nodes", &before)
+	if b, err := exec.CommandContext(ctx, "systemctl", "restart", svc).CombinedOutput(); err != nil { //nolint:gosec // fixed command
+		return fmt.Errorf("restarting %s: %v: %s", svc, err, strings.TrimSpace(string(b)))
+	}
+	fmt.Fprintf(out, "   %s restarted; waiting for its nodes to reconnect\n", svc)
+	// Back when the admin API answers and every node that was ready is ready
+	// again: agents reconnect within seconds of the control plane returning.
+	want := 0
+	for _, n := range before {
+		if n.State == "ready" {
+			want++
+		}
+	}
+	deadline := time.Now().Add(2 * time.Minute)
+	for {
+		var now []adminapi.NodeView
+		if err := client().get(ctx, "/api/v1/nodes", &now); err == nil {
+			ready := 0
+			for _, n := range now {
+				if n.State == "ready" {
+					ready++
+				}
+			}
+			if ready >= want {
+				fmt.Fprintf(out, "   back: %d of %d nodes ready\n", ready, len(now))
+				return nil
+			}
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("%s did not come back with its %d nodes within two minutes; "+
+				"see journalctl -u %s, and %s holds the previous binary", svc, want, svc, backup)
+		}
+		time.Sleep(2 * time.Second)
+	}
+}
+
+func upgradeNodes(ctx context.Context, out io.Writer, client func() *ctlClient, v string, only []string, perNode time.Duration) error {
+	var nodes []adminapi.NodeView
+	if err := client().get(ctx, "/api/v1/nodes", &nodes); err != nil {
+		return err
+	}
+	slices.SortFunc(nodes, func(a, b adminapi.NodeView) int { return strings.Compare(a.ID, b.ID) })
+	done, skipped := 0, 0
+	for _, n := range nodes {
+		if len(only) > 0 && !slices.Contains(only, n.ID) {
+			continue
+		}
+		if n.AgentVersion == v {
+			skipped++
+			continue
+		}
+		start := time.Now()
+		before := readyReplicasOn(ctx, client, n.ID)
+		fmt.Fprintf(out, "== %s: %s -> %s\n", n.ID, n.AgentVersion, v)
+		body, _ := json.Marshal(adminapi.UpgradeRequest{Version: v})
+		if err := client().post(ctx, "/api/v1/nodes/"+n.ID+"/upgrade", body, nil); err != nil {
+			return fmt.Errorf("%s: %w\nstopped here; `llmgw ctl logs %s` says more", n.ID, err, n.ID)
+		}
+		fmt.Fprintf(out, "   new binary in place; waiting for it to come back with its %d replica(s)\n", before)
+		deadline := time.Now().Add(perNode)
+		for {
+			var now []adminapi.NodeView
+			_ = client().get(ctx, "/api/v1/nodes", &now)
+			var cur adminapi.NodeView
+			for _, x := range now {
+				if x.ID == n.ID {
+					cur = x
+				}
+			}
+			after := readyReplicasOn(ctx, client, n.ID)
+			if cur.AgentVersion == v && cur.State == "ready" && after >= before {
+				fmt.Fprintf(out, "   done in %s: ready, replicas %d/%d\n",
+					time.Since(start).Round(time.Second), after, before)
+				done++
+				break
+			}
+			if time.Now().After(deadline) {
+				return fmt.Errorf("%s did not come back within %s (version %q, state %q, replicas %d/%d)\n"+
+					"stopped here, before the next node; `llmgw ctl logs %s` says why, and the node "+
+					"keeps its previous binary beside the new one", n.ID, perNode, cur.AgentVersion,
+					cur.State, after, before, n.ID)
+			}
+			time.Sleep(2 * time.Second)
+		}
+	}
+	fmt.Fprintf(out, "%d node(s) upgraded to %s, %d already on it\n", done, v, skipped)
+	// The one machine this does not reach. Said every time, because it is
+	// exactly the kind of thing that stays three versions behind unnoticed.
+	fmt.Fprintf(out, "The model mirror's host is not upgraded by this; on it, run:\n"+
+		"  sudo llmgw upgrade %s --service llmgw-mirror\n", v)
+	return nil
+}
+
+// readyReplicasOn counts the ready replicas with a shard on a node.
+func readyReplicasOn(ctx context.Context, client func() *ctlClient, node string) int {
+	var reps []adminapi.ReplicaView
+	if err := client().get(ctx, "/api/v1/replicas", &reps); err != nil {
+		return 0
+	}
+	n := 0
+	for _, r := range reps {
+		if r.State != "ready" {
+			continue
+		}
+		for _, s := range r.Shards {
+			if s.Node == node {
+				n++
+				break
+			}
+		}
+	}
+	return n
+}

internal/cli/ctl_upgrade_test.go

@@ -0,0 +1,88 @@
+package cli
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+// fakeFleet is an admin API whose nodes come back on the version they are told
+// to upgrade to — except those listed in stuck.
+type fakeFleet struct {
+	mu       sync.Mutex
+	versions map[string]string
+	stuck    map[string]bool
+	told     []string
+}
+
+func (f *fakeFleet) server(t *testing.T) *httptest.Server {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		f.mu.Lock()
+		defer f.mu.Unlock()
+		switch {
+		case r.URL.Path == "/api/v1/nodes":
+			var out []adminapi.NodeView
+			for id, v := range f.versions {
+				out = append(out, adminapi.NodeView{ID: id, State: "ready", AgentVersion: v})
+			}
+			_ = json.NewEncoder(w).Encode(out)
+		case r.URL.Path == "/api/v1/replicas":
+			var out []adminapi.ReplicaView
+			for id := range f.versions {
+				out = append(out, adminapi.ReplicaView{ID: "r-" + id, State: "ready",
+					Shards: []adminapi.ShardView{{Node: id}}})
+			}
+			_ = json.NewEncoder(w).Encode(out)
+		case strings.HasSuffix(r.URL.Path, "/upgrade"):
+			id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/nodes/"), "/upgrade")
+			var req adminapi.UpgradeRequest
+			_ = json.NewDecoder(r.Body).Decode(&req)
+			f.told = append(f.told, id)
+			if !f.stuck[id] {
+				f.versions[id] = req.Version
+			}
+			_, _ = w.Write([]byte(`{}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func TestAnUpgradeGoesNodeByNodeAndSkipsTheDone(t *testing.T) {
+	f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.1", "c": "v1.0.0"}}
+	srv := f.server(t)
+	var out bytes.Buffer
+	err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 5*time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(f.told, ",") != "a,c" {
+		t.Errorf("told %v; b was already on the version", f.told)
+	}
+	if !strings.Contains(out.String(), "2 node(s) upgraded to v1.0.1, 1 already on it") {
+		t.Errorf("output:\n%s", out.String())
+	}
+}
+
+func TestAnUpgradeStopsAtTheFirstNodeThatDoesNotComeBack(t *testing.T) {
+	// A bad release reaches one node, not all of them.
+	f := &fakeFleet{versions: map[string]string{"a": "v1.0.0", "b": "v1.0.0"}, stuck: map[string]bool{"a": true}}
+	srv := f.server(t)
+	var out bytes.Buffer
+	err := upgradeNodes(t.Context(), &out, func() *ctlClient { return newCtlClient(srv.URL) }, "v1.0.1", nil, 3*time.Second)
+	if err == nil || !strings.Contains(err.Error(), "a did not come back") {
+		t.Fatalf("err %v", err)
+	}
+	if len(f.told) != 1 {
+		t.Errorf("went on to %v after a node failed to come back", f.told)
+	}
+}

internal/cli/dispatcherref_test.go

@@ -5,6 +5,7 @@ import (
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
 )
 
 func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
@@ -23,10 +24,17 @@ func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
 		t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
 	}
 
+	if _, ok := ref.(fabrictest.Asker); !ok {
+		t.Error("dispatcherRef drops the bandwidth test, so ctl fabric-test cannot reach the agents")
+	}
+
 	// And the thing it forwards to must have it, or the forwarding has nothing
 	// to reach.
 	var svc any = &agentsvc.Service{}
 	if _, ok := svc.(control.WeightRemover); !ok {
 		t.Error("the agent service does not implement WeightRemover")
 	}
+	if _, ok := svc.(fabrictest.Asker); !ok {
+		t.Error("the agent service does not implement the bandwidth test")
+	}
 }

internal/cli/doctor.go

@@ -23,6 +23,7 @@ func newDoctorCmd() *cobra.Command {
 		opts    doctor.Options
 		asJSON  bool
 		verbose bool
+		fabrics []string
 	)
 	cmd := &cobra.Command{
 		Use:   "doctor",
@@ -38,8 +39,21 @@ that file when the report is not enough to go on.`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			doctor.SetConfigLoader(config.LoadAgent)
-			if opts.VLLMVersion == "" {
+			for _, spec := range fabrics {
+				name, hcas, _ := strings.Cut(spec, ":")
+				if name == "" || hcas == "" {
+					return fmt.Errorf("--fabric %q: give it as name:mlx5_0,mlx5_1", spec)
+				}
+				if opts.Fabrics == nil {
+					opts.Fabrics = map[string][]string{}
+				}
+				opts.Fabrics[name] = strings.Split(hcas, ",")
+			}
+			switch opts.VLLMVersion {
+			case "":
 				opts.VLLMVersion = config.DefaultEngines["vllm"]
+			case "none":
+				opts.VLLMVersion = ""
 			}
 			ctx, cancel := context.WithTimeout(cmd.Context(), 60*time.Second)
 			defer cancel()
@@ -66,7 +80,8 @@ that file when the report is not enough to go on.`,
 	f.StringVar(&opts.ConfigPath, "config", "", "agent configuration (default /etc/llmgw/agent.yaml when present)")
 	f.StringVar(&opts.IBRoot, "ib-root", discovery.SysfsRoot, "sysfs InfiniBand directory")
 	f.StringVar(&opts.NvidiaSMI, "nvidia-smi", "", "path to nvidia-smi")
-	f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs)")
+	f.StringVar(&opts.VLLMVersion, "vllm", "", "the vLLM version to check for (default: the one this llmgw installs; none to skip)")
+	f.StringArrayVar(&fabrics, "fabric", nil, "a fabric about to be declared, name:mlx5_0,mlx5_1, checked as if it were; repeatable")
 	f.BoolVar(&asJSON, "json", false, "write every fact and finding as JSON, to send back")
 	f.BoolVar(&verbose, "verbose", false, "also print the raw nvidia-smi topology")
 	return cmd

internal/cli/enginebuild.go

@@ -0,0 +1,39 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+)
+
+func newEngineBuildCmd(client func() *ctlClient) *cobra.Command {
+	return &cobra.Command{
+		Use:   "engine-build <node> <engine> <build>",
+		Short: "Choose which build of an engine a GPU node installs",
+		Long: "A node installs the build of vLLM its hardware runs: CUDA 13 where the\n" +
+			"driver and GPUs allow, else the same version built for CUDA 12.9. This\n" +
+			"chooses instead, as the cluster page's vLLM build column does:\n\n" +
+			"  auto    what the hardware runs (the default)\n" +
+			"  cu13    the CUDA 13 build, from PyPI\n" +
+			"  cu129   the CUDA 12.9 build, from vLLM's GitHub release\n" +
+			"  none    no vLLM on this node\n\n" +
+			"The node is told at once and installs it in the background; a build it\n" +
+			"cannot compute with on its GPU fails to install, saying why, and the\n" +
+			"cluster page shows it. A V100 runs no build of vLLM 0.20 or later, and\n" +
+			"gets 0.19.1, the newest built for it, by itself.\n\n" +
+			"  llmgw ctl engine-build gpu07 vllm cu129",
+		Args: cobra.ExactArgs(3),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			body, _ := json.Marshal(adminapi.NodeEngineRequest{Engine: args[1], Build: args[2]})
+			if err := client().post(cmd.Context(), "/api/v1/nodes/"+args[0]+"/engines", body, nil); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "%s: %s build %s; the node installs it in the background (llmgw ctl nodes shows it)\n",
+				args[0], args[1], args[2])
+			return nil
+		},
+	}
+}

internal/cli/enginecheck.go

@@ -3,6 +3,7 @@ package cli
 import (
 	"context"
 	"fmt"
+	"io"
 	"os/exec"
 	"path/filepath"
 	"sort"
@@ -126,6 +127,14 @@ func newEngineCheckCmd() *cobra.Command {
 			}
 			sort.Strings(unknown)
 
+			// Flags accepted is not the same as running where it is placed: a
+			// GPU build whose backend does not load accepts every flag and
+			// computes on the CPU. Asked of the engine here, not of a help.
+			var deviceErr error
+			if fb, ok := eng.(engines.CPUFallback); ok && help == "" {
+				deviceErr = checkDevices(cmd.Context(), name, fb, out)
+			}
+
 			fmt.Fprintf(out, "%s: %d flags checked", name, checked)
 			if skipped > 0 {
 				fmt.Fprintf(out, ", %d commands skipped", skipped)
@@ -137,7 +146,7 @@ func newEngineCheckCmd() *cobra.Command {
 					return fmt.Errorf("nothing was checked; install the engine here or pass --help-output")
 				}
 				fmt.Fprintln(out, "every flag llmgw builds is accepted")
-				return nil
+				return deviceErr
 			}
 			fmt.Fprintf(out, "\n%d flags not mentioned by the program they are passed to:\n", len(unknown))
 			for _, f := range unknown {
@@ -249,3 +258,22 @@ func isFlagChar(b byte) bool {
 	return b == '-' || b == '_' || (b >= 'a' && b <= 'z') ||
 		(b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9')
 }
+
+// checkDevices says which GPUs an engine that can fall back to the CPU sees on
+// this machine, and fails when the machine has NVIDIA GPUs and it sees none.
+func checkDevices(ctx context.Context, name string, fb engines.CPUFallback, out io.Writer) error {
+	if _, err := exec.LookPath("nvidia-smi"); err != nil {
+		fmt.Fprintln(out, "no NVIDIA GPU here: which devices it computes on was not checked")
+		return nil
+	}
+	devs, err := fb.Devices(ctx)
+	if err != nil {
+		return err
+	}
+	if len(devs) == 0 {
+		return fmt.Errorf("this machine has NVIDIA GPUs and %s sees none: it would serve a GPU placement on the CPU. "+
+			"Its GPU backend did not load; ldd on its libggml-cuda.so names the missing library", name)
+	}
+	fmt.Fprintf(out, "computes on: %s\n", strings.Join(devs, "; "))
+	return nil
+}

internal/cli/join.go

@@ -0,0 +1,61 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/spf13/cobra"
+
+	joinpkg "gitea.mynodes.xyz/ilker/llmgw/internal/join"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+func newJoinCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "join <code>",
+		Short: "Let a setup on another machine set this one up, with the code it gave",
+		Long: "For a machine nothing may log into: the setup screens (sudo llmgw on\n" +
+			"the control plane) show a code for each machine they expect. Run this\n" +
+			"with it, as root, on that machine. It dials out to the setup — nothing\n" +
+			"dials in — checks it is the setup the code was made by, and runs what\n" +
+			"the setup sends until it is done: the same checks and installs ssh\n" +
+			"would carry. Each step is printed here as it runs.\n\n" +
+			"A code is good for one machine, for an hour, and only while the setup\n" +
+			"that made it is waiting.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if runtime.GOOS != "linux" {
+				return fmt.Errorf("join sets up a Linux machine; this is %s", runtime.GOOS)
+			}
+			if os.Geteuid() != 0 {
+				return errors.New("run it as root — sudo llmgw join … — since the setup installs software here")
+			}
+			code, err := joinpkg.Parse(args[0])
+			if err != nil {
+				return err
+			}
+			out := cmd.OutOrStdout()
+			role := code.Role
+			if role == "" {
+				role = "part of a cluster"
+			}
+			fmt.Fprintf(out, "Joining the setup at %s, as %s. It runs as root here until it is done.\n", code.Addr, role)
+			return joinpkg.Serve(cmd.Context(), code, func(s string) { fmt.Fprintln(out, s) })
+		},
+	}
+}
+
+// joinInstall is what to run on a machine that has no llmgw yet, to fetch
+// this version for its architecture: short lines, since it is copied from a
+// screen that cuts long ones off.
+func joinInstall() []string {
+	v := version.Version
+	return []string{
+		`a=$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/')`,
+		fmt.Sprintf("sudo curl -fsSL -o /usr/local/bin/llmgw \\\n    %s/%s/llmgw-%s-linux-$a", release.DefaultBase, v, v),
+		"sudo chmod 755 /usr/local/bin/llmgw",
+	}
+}

internal/cli/mirror.go

@@ -2,10 +2,14 @@ package cli
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
+	"net"
 	"net/http"
+	"net/netip"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -44,6 +48,7 @@ func newMirrorCmd() *cobra.Command {
 		newMirrorUnusedCmd(&dir),
 		newMirrorRemoveCmd(&dir),
 		newMirrorServeCmd(&dir),
+		newMirrorInitCmd(&dir),
 	)
 	return c
 }
@@ -52,52 +57,69 @@ func newMirrorAddCmd(dir *string) *cobra.Command {
 	var model, version, as string
 
 	c := &cobra.Command{
-		Use:   "add <url>",
+		Use:   "add <url>...",
 		Short: "Download a model into the mirror",
 		Long: "Downloads to a temporary name and renames when complete, so an\n" +
 			"interrupted download never leaves something a catalog could be pointed\n" +
 			"at. Reports the checksum: a mirror is the one place in this system where\n" +
 			"a corrupted file is copied to every node rather than caught once.\n\n" +
+			"Several URLs go into one version, as the files of one model: the shards\n" +
+			"of a safetensors set with its config.json and tokenizer, for vLLM. The\n" +
+			"version is offered to nodes only once it is whole — nothing still\n" +
+			"downloading, every shard its index names present — and its source is\n" +
+			"the version, ending in a slash.\n\n" +
 			"Refuses to replace a version that is already here. Add a new one beside\n" +
 			"it instead — that is what makes the old one still available when the new\n" +
 			"one turns out to be worse.",
-		Args: cobra.ExactArgs(1),
+		Args: cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			source := args[0]
-			file := as
-			if file == "" {
-				file = mirror.LastPathElement(source)
-			}
 			if model == "" || version == "" {
 				return fmt.Errorf("--model and --version are required: they are the layout")
 			}
-
-			e := mirror.Entry{Model: model, Version: version, File: file}
-			out := cmd.OutOrStdout()
-			fmt.Fprintf(out, "fetching %s\n", source)
-
-			added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
-				if total > 0 {
-					fmt.Fprintf(out, "  %s of %s (%d%%)\n",
-						fileSize(done), fileSize(total), done*100/total)
-					return
-				}
-				fmt.Fprintf(out, "  %s so far\n", fileSize(done))
-			})
-			if err != nil {
-				return err
+			if as != "" && len(args) > 1 {
+				return fmt.Errorf("--as names one file; with %d URLs each keeps its own name", len(args))
 			}
-			fmt.Fprintf(out, "\n%s/%s/%s  %s\n  sha256 %s\n",
-				added.Entry.Model, added.Entry.Version, added.Entry.File,
-				fileSize(added.Entry.Bytes), added.SHA256)
-			fmt.Fprintf(out, "\nPoint a model at it with:\n  source: %s\n",
-				added.Entry.URL("http://<this-host>:8081"))
+			out := cmd.OutOrStdout()
+			var last mirror.Added
+			for _, source := range args {
+				file := as
+				if file == "" {
+					file = mirror.LastPathElement(source)
+				}
+				e := mirror.Entry{Model: model, Version: version, File: file}
+				fmt.Fprintf(out, "fetching %s\n", source)
+				added, err := mirror.Add(cmd.Context(), *dir, source, e, func(done, total int64) {
+					if total > 0 {
+						fmt.Fprintf(out, "  %s of %s (%d%%)\n",
+							fileSize(done), fileSize(total), done*100/total)
+						return
+					}
+					fmt.Fprintf(out, "  %s so far\n", fileSize(done))
+				})
+				if err != nil {
+					return err
+				}
+				fmt.Fprintf(out, "%s/%s/%s  %s\n  sha256 %s\n",
+					added.Entry.Model, added.Entry.Version, added.Entry.File,
+					fileSize(added.Entry.Bytes), added.SHA256)
+				last = added
+			}
+			src := last.Entry.URL("http://<this-host>:8081")
+			if len(args) > 1 {
+				src = strings.TrimSuffix(mirror.Entry{Model: model, Version: version}.URL("http://<this-host>:8081"), "/") + "/"
+				if m, err := mirror.ManifestFor(*dir, model, version); err != nil {
+					fmt.Fprintf(out, "\nNot yet offered to nodes: %v\n", err)
+				} else {
+					fmt.Fprintf(out, "\n%d files, %s, whole.\n", len(m.Files), fileSize(m.Bytes))
+				}
+			}
+			fmt.Fprintf(out, "\nPoint a model at it with:\n  source: %s\n", src)
 			return nil
 		},
 	}
 	c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
 	c.Flags().StringVar(&version, "version", "", "the version, one directory level; a date works well")
-	c.Flags().StringVar(&as, "as", "", "the filename to save as; taken from the URL when omitted")
+	c.Flags().StringVar(&as, "as", "", "the filename to save as, for one URL; taken from the URL when omitted")
 	return c
 }
 
@@ -229,6 +251,8 @@ func newMirrorRemoveCmd(dir *string) *cobra.Command {
 func newMirrorServeCmd(dir *string) *cobra.Command {
 	var listen, apiListen, base string
 	var tlsc tlsconf.Config
+	var allow, clients []string
+	var allowPrivate bool
 
 	c := &cobra.Command{
 		Use:   "serve",
@@ -254,10 +278,16 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
 			if err := tlsc.Validate(); err != nil {
 				return err
 			}
+			if len(clients) > 0 && !tlsc.Enabled() {
+				return fmt.Errorf("--api-client names a certificate, and needs --tls-ca, --tls-cert and --tls-key")
+			}
 
 			out := cmd.OutOrStdout()
 			log := slog.Default()
 			mgr := mirror.NewManager(*dir, log)
+			if allowPrivate {
+				mgr.AllowPrivateSources()
+			}
 			api := mirror.NewAPI(mgr, strings.TrimSuffix(base, "/"))
 
 			// http.Dir refuses paths that escape the root, which is the whole
@@ -268,9 +298,13 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
 				Handler:           readOnly(withManifest(*dir, http.FileServer(http.Dir(*dir)))),
 				ReadHeaderTimeout: 10 * time.Second,
 			}
+			allowed, err := parseAllow(allow)
+			if err != nil {
+				return err
+			}
 			admin := &http.Server{
 				Addr:              apiListen,
-				Handler:           api.Handler(),
+				Handler:           onlyClients(clients, apiGate(tlsc.Enabled(), allowed, api.Handler())),
 				ReadHeaderTimeout: 10 * time.Second,
 			}
 
@@ -279,14 +313,23 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
 			fmt.Fprintf(out, "serving %s on %s\n", *dir, listen)
 
 			go func() { errs <- fmt.Errorf("serving the API: %w", serveAPI(admin, tlsc)) }()
-			if tlsc.Enabled() {
+			switch {
+			case tlsc.Enabled():
 				fmt.Fprintf(out, "management API on %s, mutual TLS against %s\n", apiListen, tlsc.CA)
-			} else {
-				// Said plainly rather than left to be discovered. An API that
-				// can delete models and start downloads, open to anyone who can
-				// reach the port, is a choice the operator should have made
-				// rather than inherited from a default.
-				fmt.Fprintf(out, "management API on %s, UNAUTHENTICATED — keep it on a trusted network\n", apiListen)
+				if len(clients) == 0 {
+					// The same root signs every node's certificate, so without a
+					// name any node could delete models here.
+					log.Warn("the management API takes any certificate from " + tlsc.CA +
+						", every node's included; name the control plane's with --api-client")
+				}
+			case len(allowed) > 0:
+				fmt.Fprintf(out, "management API on %s, for %s and this machine only\n", apiListen, strings.Join(allow, ", "))
+			default:
+				// This machine only, which was once "anyone": an API that can
+				// delete models and start downloads, open to every node that
+				// could reach the port, is a choice the operator should make
+				// with --api-allow rather than inherit.
+				fmt.Fprintf(out, "management API on %s, for this machine only (--api-allow to add the control plane)\n", apiListen)
 			}
 			fmt.Fprintf(out, "catalog sources should point at %s\n", base)
 
@@ -300,6 +343,12 @@ func newMirrorServeCmd(dir *string) *cobra.Command {
 	c.Flags().StringVar(&apiListen, "api-listen", ":8082", "address to serve the management API on")
 	c.Flags().StringVar(&base, "base", "http://localhost:8081",
 		"the address nodes use to reach this mirror, as it should appear in catalog sources")
+	c.Flags().StringArrayVar(&allow, "api-allow", nil,
+		"an address or network the management API answers, besides this machine; repeatable (default: this machine only)")
+	c.Flags().StringArrayVar(&clients, "api-client", nil,
+		"with mutual TLS, the common name of a certificate the management API answers; repeatable (default: any from --tls-ca)")
+	c.Flags().BoolVar(&allowPrivate, "allow-private-sources", false,
+		"let downloads reach loopback, private and link-local addresses")
 	c.Flags().StringVar(&tlsc.CA, "tls-ca", "", "CA bundle that management clients must present a certificate from")
 	c.Flags().StringVar(&tlsc.Cert, "tls-cert", "", "this mirror's certificate")
 	c.Flags().StringVar(&tlsc.Key, "tls-key", "", "this mirror's private key")
@@ -374,7 +423,14 @@ func withManifest(root string, next http.Handler) http.Handler {
 			return
 		}
 		m, err := mirror.ManifestFor(root, model, version)
-		if err != nil {
+		switch {
+		case errors.Is(err, mirror.ErrIncomplete):
+			// Said, not hidden behind a 404: a node asking now is told why
+			// it cannot have the version yet, and tries again later.
+			w.Header().Set("Retry-After", "60")
+			http.Error(w, err.Error(), http.StatusConflict)
+			return
+		case err != nil:
 			http.NotFound(w, r)
 			return
 		}
@@ -398,3 +454,78 @@ func versionPath(p string) (model, version string, ok bool) {
 	}
 	return parts[0], parts[1], true
 }
+
+// parseAllow reads addresses and networks: 10.0.0.140, 10.0.0.0/24.
+func parseAllow(list []string) ([]netip.Prefix, error) {
+	var out []netip.Prefix
+	for _, s := range list {
+		for _, one := range strings.Split(s, ",") {
+			one = strings.TrimSpace(one)
+			if one == "" {
+				continue
+			}
+			if p, err := netip.ParsePrefix(one); err == nil {
+				out = append(out, p.Masked())
+				continue
+			}
+			a, err := netip.ParseAddr(one)
+			if err != nil {
+				return nil, fmt.Errorf("--api-allow %q is not an address or a network", one)
+			}
+			out = append(out, netip.PrefixFrom(a.Unmap(), a.Unmap().BitLen()))
+		}
+	}
+	return out, nil
+}
+
+// apiGate decides who reaches the management API. With mutual TLS the
+// certificate decides, narrowed by --api-allow when given. Without it, only
+// this machine and the addresses allowed: nothing allowed used to mean anyone,
+// and every node that could reach the port could delete models.
+func apiGate(mtls bool, allowed []netip.Prefix, next http.Handler) http.Handler {
+	if mtls && len(allowed) == 0 {
+		return next
+	}
+	return onlyFrom(allowed, next)
+}
+
+// onlyClients answers only certificates with one of these common names, when
+// any are given. The handshake has already checked the certificate against
+// the root; this narrows which of the root's certificates may manage.
+func onlyClients(names []string, next http.Handler) http.Handler {
+	if len(names) == 0 {
+		return next
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 ||
+			!slices.Contains(names, r.TLS.VerifiedChains[0][0].Subject.CommonName) {
+			http.Error(w, "this mirror is managed by its control plane only", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+// onlyFrom answers only this machine and the addresses allowed: the management
+// API can delete models and start downloads, and a mirror on its own machine
+// is reached over the network by one control plane and no one else.
+func onlyFrom(allowed []netip.Prefix, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		host, _, err := net.SplitHostPort(r.RemoteAddr)
+		a, perr := netip.ParseAddr(host)
+		if err != nil || perr != nil {
+			http.Error(w, "forbidden", http.StatusForbidden)
+			return
+		}
+		a = a.Unmap()
+		ok := a.IsLoopback()
+		for _, p := range allowed {
+			ok = ok || p.Contains(a)
+		}
+		if !ok {
+			http.Error(w, "this mirror is managed by its control plane only", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}

internal/cli/mirror_gate_test.go

@@ -0,0 +1,66 @@
+package cli
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"testing"
+)
+
+func TestTheMirrorAPIAnswersOnlyThisMachineUnlessTold(t *testing.T) {
+	ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
+	call := func(h http.Handler, from string) int {
+		r := httptest.NewRequest("DELETE", "/v1/models/m/v", nil)
+		r.RemoteAddr = from + ":5000"
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, r)
+		return w.Code
+	}
+	cp := []netip.Prefix{netip.MustParsePrefix("10.0.0.140/32")}
+
+	// Nothing allowed and no TLS used to mean anyone: every node could delete
+	// models.
+	if got := call(apiGate(false, nil, ok), "10.0.0.141"); got != http.StatusForbidden {
+		t.Errorf("a node reached an unconfigured mirror API: %d", got)
+	}
+	if got := call(apiGate(false, nil, ok), "127.0.0.1"); got != http.StatusOK {
+		t.Errorf("the mirror's own machine was refused: %d", got)
+	}
+	if got := call(apiGate(false, cp, ok), "10.0.0.140"); got != http.StatusOK {
+		t.Errorf("the allowed control plane was refused: %d", got)
+	}
+	if got := call(apiGate(false, cp, ok), "10.0.0.141"); got != http.StatusForbidden {
+		t.Errorf("a node outside the allowlist got through: %d", got)
+	}
+	// With mutual TLS the certificate decides.
+	if got := call(apiGate(true, nil, ok), "10.0.0.141"); got != http.StatusOK {
+		t.Errorf("mTLS without an allowlist refused a client: %d", got)
+	}
+}
+
+func TestOnlyANamedCertificateManagesTheMirror(t *testing.T) {
+	ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})
+	with := func(cn string) *http.Request {
+		r := httptest.NewRequest("GET", "/", nil)
+		if cn != "" {
+			r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{{Subject: pkix.Name{CommonName: cn}}}}}
+		}
+		return r
+	}
+	h := onlyClients([]string{"control-plane"}, ok)
+	for cn, want := range map[string]int{"control-plane": 200, "node01": 403, "": 403} {
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, with(cn))
+		if w.Code != want {
+			t.Errorf("%q: %d, expected %d", cn, w.Code, want)
+		}
+	}
+	w := httptest.NewRecorder()
+	onlyClients(nil, ok).ServeHTTP(w, with("node01"))
+	if w.Code != 200 {
+		t.Error("no names given narrowed it anyway")
+	}
+}

internal/cli/mirrorinit.go

@@ -0,0 +1,125 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
+)
+
+// mirror init sets a machine up as the cluster's model mirror, on its own:
+// the bytes for every node on :8081, the management API on :8082 for the
+// control plane alone. server init does the same beside the control plane;
+// this is for when the mirror wants its own disk.
+
+func newMirrorInitCmd(dir *string) *cobra.Command {
+	var (
+		controlPlanes []string
+		base          string
+	)
+	cmd := &cobra.Command{
+		Use:   "init",
+		Short: "Set this machine up as the cluster's model mirror and start it",
+		Long: `Creates the llmgw user and the mirror's directory, and a systemd unit that
+serves the model files to every node on :8081 and the management API on
+:8082, and starts it.
+
+The management API can fetch and delete models, so it answers this machine
+and the control planes given with --control-plane, and nobody else. Then on
+the control plane:
+
+  sudo llmgw server init --mirror http://<this machine>:8082
+
+Safe to run again: it changes only what differs.`,
+		Example: `  sudo llmgw mirror init --control-plane 10.0.0.10`,
+		Args:    cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if runtime.GOOS != "linux" {
+				return fmt.Errorf("mirror init sets up a Linux machine; this is %s", runtime.GOOS)
+			}
+			if os.Geteuid() != 0 {
+				return errors.New("mirror init creates a user, writes /etc and starts a service: run it as root")
+			}
+			if len(controlPlanes) == 0 {
+				return errors.New("--control-plane: which control plane manages this mirror (its address)")
+			}
+			if _, err := parseAllow(controlPlanes); err != nil {
+				return err
+			}
+			return mirrorInit(cmd.Context(), cmd.OutOrStdout(), *dir, base, controlPlanes)
+		},
+	}
+	cmd.Flags().StringArrayVar(&controlPlanes, "control-plane", nil,
+		"address of the control plane that manages this mirror; repeatable")
+	cmd.Flags().StringVar(&base, "base", "",
+		"the address nodes fetch from, http://host:8081 (default: this machine's first address)")
+	return cmd
+}
+
+func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPlanes []string) error {
+	in := &serverInit{user: "llmgw", stateDir: "/var/lib/llmgw", out: out}
+	if err := in.ensureUser(ctx); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	if err := in.chown(dir, true); err != nil {
+		return err
+	}
+	addr := ""
+	if ips := localAddrs(); len(ips) > 0 {
+		addr = ips[0].String()
+	}
+	if base == "" {
+		if addr == "" {
+			return errors.New("this machine has no address to give nodes; give one with --base")
+		}
+		base = "http://" + net.JoinHostPort(addr, "8081")
+	}
+	unit := mirrorUnitFor(dir, ":8082", base, controlPlanes,
+		"'llmgw mirror init'. Bytes on :8081 for every node; the\n# management API on :8082, for the control plane only.")
+	changed, err := in.writeUnit(mirrorUnitPath, unit)
+	if err != nil {
+		return err
+	}
+	for _, args := range [][]string{{"daemon-reload"}, {"enable", "--now", "llmgw-mirror"}} {
+		if err := in.systemctl(ctx, args...); err != nil {
+			return err
+		}
+	}
+	if changed {
+		if err := in.systemctl(ctx, "restart", "llmgw-mirror"); err != nil {
+			return err
+		}
+		in.done("model mirror in %s: files on :8081 at %s, management on :8082 for %s", dir, base, strings.Join(controlPlanes, ", "))
+	} else {
+		in.kept("model mirror in %s", dir)
+	}
+	m, err := mirror.NewClient("http://127.0.0.1:8082", tlsconf.Config{}, 5*time.Second)
+	if err != nil {
+		return err
+	}
+	if err := waitFor(ctx, 30*time.Second, "the mirror to answer", func() (bool, error) {
+		return m.Health(ctx) == nil, nil
+	}); err != nil {
+		return err
+	}
+	in.kept("answering")
+	api := "http://<this machine>:8082"
+	if addr != "" {
+		api = "http://" + net.JoinHostPort(addr, "8082")
+	}
+	fmt.Fprintf(out, "\nThe mirror is up. On the control plane:\n\n  sudo llmgw server init --mirror %s\n\n", api)
+	return nil
+}

internal/cli/model.go

@@ -3,7 +3,11 @@ package cli
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/user"
+	"slices"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/knadh/koanf/parsers/yaml"
@@ -23,12 +27,15 @@ import (
 // minutes of outage. For a fixed production set that is a reasonable trade; for
 // a platform whose promise is that people can try things, it is the difference
 // between trying one and not bothering.
-func newModelCmd(client func() *ctlClient) *cobra.Command {
+func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 	c := &cobra.Command{
 		Use:   "model",
 		Short: "Add or remove models while the cluster is running",
 	}
-	c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
+	c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
+		newResourcesCmd(client), newClassCmd(client), newOwnerCmd(client),
+		newTargetCmd(client, asJSON), newAdaptersCmd(client),
+		newModelServeCmd(client))
 	return c
 }
 
@@ -126,6 +133,10 @@ func newModelRemoveCmd(client func() *ctlClient) *cobra.Command {
 
 // actor names who made a change, for the audit trail on a stored model.
 func actor() string {
+	// Under sudo, the person who ran it rather than root.
+	if u := os.Getenv("SUDO_USER"); u != "" && os.Geteuid() == 0 {
+		return u
+	}
 	if u, err := user.Current(); err == nil && u.Username != "" {
 		return u.Username
 	}
@@ -186,3 +197,139 @@ func newScaleCmd(client func() *ctlClient) *cobra.Command {
 		"how long an on-demand model stays up unasked-for; zero takes the cluster default")
 	return c
 }
+
+func newResourcesCmd(client func() *ctlClient) *cobra.Command {
+	var memory, cores int
+	var profile string
+	c := &cobra.Command{
+		Use:   "resources <model> --memory GB [--cores N]",
+		Short: "Set what a model reserves of a node: memory, and CPU cores",
+		Long: "--memory is the host RAM a replica needs: for a model with no GPU, its\n" +
+			"weights, KV cache and engine together. Placement reserves it on the node,\n" +
+			"so a node holds only as many replicas as it has memory for.\n\n" +
+			"--cores is the CPU cores a replica with no GPU computes on. They are\n" +
+			"reserved the same way, and become the engine's thread count, so two\n" +
+			"replicas on one node stop competing for the same cores. Zero reserves\n" +
+			"none and leaves the thread count to the node.\n\n" +
+			"Running replicas keep what they were started with; `llmgw ctl restart\n" +
+			"--model M` moves them to the new figures.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if !cmd.Flags().Changed("memory") {
+				return fmt.Errorf("--memory is required")
+			}
+			body, err := json.Marshal(adminapi.ResourcesRequest{Profile: profile, MemoryGB: memory, CPUCores: cores})
+			if err != nil {
+				return err
+			}
+			var m catalog.Model
+			if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/resources", body, &m); err != nil {
+				return err
+			}
+			for _, p := range m.Profiles {
+				fmt.Fprintf(cmd.OutOrStdout(), "%s/%s: %d GB memory, %d cores\n", m.Name, p.Name, p.MinMemoryGB, p.CPUCores)
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "Running replicas keep their old figures until restarted: llmgw ctl restart --model %s\n", m.Name)
+			return nil
+		},
+	}
+	c.Flags().IntVar(&memory, "memory", 0, "host RAM a replica needs, in GB")
+	c.Flags().IntVar(&cores, "cores", 0, "CPU cores a replica with no GPU computes on; 0 reserves none")
+	c.Flags().StringVar(&profile, "profile", "", "one profile; every profile when left out")
+	return c
+}
+
+func newClassCmd(client func() *ctlClient) *cobra.Command {
+	return &cobra.Command{
+		Use:   "class <model> critical|standard|opportunistic|default",
+		Short: "Set how much a model's capacity is worth against another's",
+		Long: "When a model cannot be placed, replicas of a lower class are stopped to\n" +
+			"make room for it:\n\n" +
+			"  critical       never gives up the replicas of its minimum\n" +
+			"  standard       gives way to critical\n" +
+			"  opportunistic  gives way to both; the default for an on-demand model\n\n" +
+			"Within a class, a model below its minimum may take what another grew into\n" +
+			"under load, but not its minimum. \"default\" goes back to what the model's\n" +
+			"scaling implies: opportunistic when on demand, standard otherwise.",
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			class := args[1]
+			if class == "default" {
+				class = ""
+			}
+			body, _ := json.Marshal(map[string]string{"class": class})
+			var res map[string]string
+			if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/class", body, &res); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "%s is %s\n", res["model"], res["class"])
+			return nil
+		},
+	}
+}
+
+func newAdaptersCmd(client func() *ctlClient) *cobra.Command {
+	var add, remove []string
+	c := &cobra.Command{
+		Use:   "adapters <model> [--add NAME=URL[@SCALE]] [--remove NAME]",
+		Short: "List, add or remove a model's LoRA adapters",
+		Long: "An adapter is asked for by its NAME as though it were a model, and is served\n" +
+			"by this model's replicas with it applied. URL is the adapter file, usually in\n" +
+			"the mirror; it must be a GGUF LoRA adapter for this model's architecture,\n" +
+			"which is checked before it is attached. Replicas load a changed set when\n" +
+			"restarted: llmgw ctl restart --model MODEL.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			name := args[0]
+			var models []adminapi.ModelView
+			if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
+				return err
+			}
+			var current []catalog.Adapter
+			found := false
+			for _, m := range models {
+				if m.Name == name {
+					current, found = m.Adapters, true
+				}
+			}
+			if !found {
+				return fmt.Errorf("no model %q", name)
+			}
+			out := cmd.OutOrStdout()
+			if len(add) == 0 && len(remove) == 0 {
+				if len(current) == 0 {
+					fmt.Fprintf(out, "%s has no adapters\n", name)
+				}
+				for _, a := range current {
+					fmt.Fprintf(out, "%s\tscale %g\t%s\n", a.Name, a.EffectiveScale(), a.Source)
+				}
+				return nil
+			}
+			next := slices.DeleteFunc(slices.Clone(current), func(a catalog.Adapter) bool { return slices.Contains(remove, a.Name) })
+			for _, spec := range add {
+				n, rest, ok := strings.Cut(spec, "=")
+				if !ok || n == "" || rest == "" {
+					return fmt.Errorf("--add %q is not NAME=URL[@SCALE]", spec)
+				}
+				ad := catalog.Adapter{Name: n, Source: rest}
+				if i := strings.LastIndex(rest, "@"); i > 0 {
+					if s, err := strconv.ParseFloat(rest[i+1:], 64); err == nil {
+						ad.Source, ad.Scale = rest[:i], s
+					}
+				}
+				next = append(next, ad)
+			}
+			body, _ := json.Marshal(adminapi.AdaptersRequest{Adapters: next})
+			var res adminapi.AdaptersRequest
+			if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/adapters", body, &res); err != nil {
+				return err
+			}
+			fmt.Fprintf(out, "%s serves %d adapter(s); restart its replicas to load them: llmgw ctl restart --model %s\n",
+				name, len(res.Adapters), name)
+			return nil
+		},
+	}
+	c.Flags().StringArrayVar(&add, "add", nil, "NAME=URL[@SCALE]; repeatable")
+	c.Flags().StringArrayVar(&remove, "remove", nil, "an adapter's name; repeatable")
+	return c
+}

internal/cli/model_serve.go

@@ -0,0 +1,81 @@
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
+)
+
+func newModelServeCmd(client func() *ctlClient) *cobra.Command {
+	var (
+		req      modelserve.Request
+		onDemand bool
+	)
+	c := &cobra.Command{
+		Use:   "serve OWNER/REPO",
+		Short: "Take a model from the registry to a replica that answers, in one command",
+		Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
+			"fetch it and checks what arrived against the registry's checksum, checks\n" +
+			"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
+			"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
+			"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
+			"  llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
+			"  llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
+			"  llmgw ctl model serve owner/big-GGUF --on-demand",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			req.Repo = args[0]
+			if onDemand {
+				req.Replicas = 0
+			}
+			body, _ := json.Marshal(req)
+			var job modelserve.Job
+			if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
+				return err
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "job %s\n", job.ID)
+			shown := map[string]string{}
+			for {
+				for _, s := range job.Steps {
+					line := s.State + " " + s.Detail
+					if shown[s.Name] == line {
+						continue
+					}
+					shown[s.Name] = line
+					mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
+						modelserve.Failed: "!!"}[s.State]
+					fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
+				}
+				switch job.State {
+				case modelserve.Done:
+					fmt.Fprintf(out, "\n%s is served.\n", job.Model)
+					return nil
+				case modelserve.Failed:
+					return errors.New(job.Error)
+				}
+				time.Sleep(2 * time.Second)
+				if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
+					return err
+				}
+			}
+		},
+	}
+	f := c.Flags()
+	f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
+	f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
+	f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
+	f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
+	f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
+	f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
+	f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
+	f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
+	f.IntVar(&req.CPUCores, "cores", 0, "CPU cores a replica computes on; 0 takes half the smallest node it fits")
+	f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
+	return c
+}

internal/cli/node.go

@@ -6,11 +6,13 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
@@ -64,12 +66,15 @@ type nodeAdd struct {
 	noWait    bool
 	plain     bool
 	serverCfg string
+	// named are the fabric names given in this run, by what the adapters
+	// report the fabric to be.
+	named map[string]string
 }
 
 func newNodeAddCmd() *cobra.Command {
 	var na nodeAdd
 	cmd := &cobra.Command{
-		Use:   "add [user@]address",
+		Use:   "add [user@]address...",
 		Short: "Make a machine a node: check it, give it an identity, start its agent",
 		Long: `Run on the machine holding the certificate authority — the control plane,
 after 'llmgw server init' — as root. It logs in to the machine with ssh,
@@ -87,14 +92,18 @@ Nothing else needs giving, except --fabric for InfiniBand or RoCE: which
 fabric an adapter is on has a name only you can choose, and it must be the
 same on every node attached to it. Doctor suggests one when it sees adapters.
 
+Several addresses add several nodes, one after another; a fabric named for
+one is recognised on the next and not asked again.
+
 The user needs sudo on the machine without a password. Run it again at any
 time: the node gets a fresh certificate and configuration, and its running
 engines are kept.`,
 		Example: `  sudo llmgw node add 10.0.0.21
+  sudo llmgw node add ubuntu@10.0.0.21 ubuntu@10.0.0.22 ubuntu@10.0.0.23
   sudo llmgw node add ubuntu@gpu01 --fabric ib-a:mlx5_0,mlx5_1,mlx5_2,mlx5_3`,
-		Args: cobra.ExactArgs(1),
+		Args: cobra.MinimumNArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return na.run(cmd.Context(), args[0], cmd.OutOrStdout())
+			return na.run(cmd.Context(), args, cmd.OutOrStdout())
 		},
 	}
 	f := cmd.Flags()
@@ -105,13 +114,14 @@ engines are kept.`,
 	f.StringVar(&na.opts.MgmtAddr, "mgmt-addr", "", "address engines are reached at (default: found by the agent)")
 	f.StringVar(&na.opts.MgmtIface, "mgmt-if", "", "interface NCCL bootstraps over (default: the one holding that address)")
 	f.StringVar(&na.opts.LlamaCppBin, "llamacpp-bin", "", "a llama-server you installed, instead of the one the agent installs")
-	f.StringVar(&na.opts.VLLMBin, "vllm-bin", "", "a vllm you installed; also set engines.vllm: \"\" in server.yaml")
+	f.StringVar(&na.opts.VLLMBin, "vllm-bin", "", "a vllm you installed, instead of the one the agent installs")
 	f.StringVar(&na.opts.SGLangPython, "sglang-python", "", "python of an SGLang virtualenv you installed")
 	f.StringVar(&na.opts.NvidiaSMI, "nvidia-smi", "", "where nvidia-smi is, when it is not on PATH")
 	f.StringVar(&na.opts.IBRoot, "ib-root", "", "the sysfs InfiniBand directory, when not /sys/class/infiniband")
 	f.StringVar(&na.opts.PKI, "pki", defaultPKIDir, "directory holding the certificate authority")
 	f.StringVar(&na.opts.Releases, "releases", "https://gitea.mynodes.xyz/ilker/llmgw/releases/download", "where release files are downloaded from")
 	f.BoolVar(&na.opts.IgnoreChecks, "ignore-checks", false, "add the node even when doctor finds a problem")
+	f.StringArrayVar(&na.opts.DisableEngines, "disable-engine", nil, "an engine this node never installs, e.g. vllm on GPUs it does not run on; repeatable")
 	f.StringVar(&na.admin, "admin", "", "the control plane's admin API, to watch the node join (default: from server.yaml, else 127.0.0.1:9090)")
 	f.StringVar(&na.serverCfg, "config", "/etc/llmgw/server.yaml", "the control plane's configuration, for the admin API's address")
 	f.BoolVar(&na.waitAll, "wait", false, "wait for every engine, vLLM included, not only llama.cpp")
@@ -120,18 +130,9 @@ engines are kept.`,
 	return cmd
 }
 
-func (na *nodeAdd) prepare(addr string) error {
+// prepare works out what every node added in this run shares.
+func (na *nodeAdd) prepare() error {
 	o := &na.opts
-	o.Addr = addr
-	if user, host, ok := strings.Cut(addr, "@"); ok {
-		o.User, o.Addr = user, host
-	}
-	if o.User == "" {
-		// Under sudo, the person who ran it — their account is usually the
-		// one that exists on the nodes too. root's own keys still do the
-		// logging in, since that is who is running ssh.
-		o.User = os.Getenv("SUDO_USER")
-	}
 	for _, s := range na.fabrics {
 		fab, err := onboard.ParseFabric(s)
 		if err != nil {
@@ -149,6 +150,23 @@ func (na *nodeAdd) prepare(addr string) error {
 			}
 		}
 	}
+	for _, e := range o.DisableEngines {
+		if !slices.Contains(provision.Installable(), e) {
+			return fmt.Errorf("--disable-engine %s: the engines are %s", e, strings.Join(provision.Installable(), ", "))
+		}
+	}
+	// What the control plane will ask the node to install, so doctor checks
+	// that vLLM rather than its own default. Its overrides for a GPU class
+	// are not known until the node says what it has, and are not applied
+	// here; doctor's verdict on vLLM is a warning either way.
+	if o.VLLMVersion == "" {
+		if cfg, err := config.Load(na.serverCfg); err == nil {
+			o.VLLMVersion = cfg.WantedEngines()["vllm"]
+			if o.VLLMVersion == "" {
+				o.VLLMVersion = "none"
+			}
+		}
+	}
 	o.Version = version.Version
 	if o.Binary == "" {
 		exe, err := os.Executable()
@@ -158,32 +176,124 @@ func (na *nodeAdd) prepare(addr string) error {
 		o.Binary = exe
 	}
 	onboard.AgentUnit = agentUnit
+	na.named = map[string]string{}
 	return nil
 }
 
-func (na *nodeAdd) run(ctx context.Context, addr string, out io.Writer) error {
-	if err := na.prepare(addr); err != nil {
+// target is the options for one address, [user@]host.
+func (na *nodeAdd) target(addr string) onboard.Options {
+	o := na.opts
+	o.Fabrics = slices.Clone(na.opts.Fabrics)
+	o.Addr = addr
+	if user, host, ok := strings.Cut(addr, "@"); ok {
+		o.User, o.Addr = user, host
+	}
+	if o.User == "" {
+		// Under sudo, the person who ran it — their account is usually the
+		// one that exists on the nodes too. root's own keys still do the
+		// logging in, since that is who is running ssh.
+		o.User = os.Getenv("SUDO_USER")
+	}
+	return o
+}
+
+// rememberFabrics wraps the question of which fabric a node's adapters are
+// on, so a name given for one node is used for the next whose adapters
+// report the same fabric, without asking again. ask nil is a script, which
+// is asked nothing and gets only what an earlier answer said.
+func (na *nodeAdd) rememberFabrics(ask func([]onboard.Fabric) []onboard.Fabric) func([]onboard.Fabric) []onboard.Fabric {
+	return func(s []onboard.Fabric) []onboard.Fabric {
+		var known []onboard.Fabric
+		all := true
+		for i, f := range s {
+			if name, ok := na.named[f.Key]; ok && f.Key != "" && f.Key != "unknown" {
+				f.Name = name
+				s[i].Name = name
+				known = append(known, f)
+			} else {
+				all = false
+			}
+		}
+		if all || ask == nil {
+			return known
+		}
+		got := ask(s)
+		for _, f := range got {
+			if f.Key != "" && f.Key != "unknown" {
+				na.named[f.Key] = f.Name
+			}
+		}
+		return got
+	}
+}
+
+func (na *nodeAdd) run(ctx context.Context, addrs []string, out io.Writer) error {
+	if len(addrs) > 1 && na.opts.Node != "" {
+		return errors.New("--node names one node; give one address with it")
+	}
+	if err := na.prepare(); err != nil {
 		return err
 	}
-	if !na.plain && isTerminal(out) {
-		return na.runTUI(ctx)
+	tui := !na.plain && isTerminal(out)
+	type outcome struct {
+		addr, node string
+		err        error
 	}
+	var results []outcome
+	for i, addr := range addrs {
+		o := na.target(addr)
+		title := "llmgw · adding " + o.Addr
+		if len(addrs) > 1 {
+			title += fmt.Sprintf(" (%d of %d)", i+1, len(addrs))
+		}
+		var node string
+		var err error
+		if tui {
+			node, err = na.runTUI(ctx, o, title)
+		} else {
+			node, err = na.runPlain(ctx, o, title, out)
+		}
+		results = append(results, outcome{addr, node, err})
+		if ctx.Err() != nil {
+			break
+		}
+	}
+	if len(addrs) == 1 {
+		return results[0].err
+	}
+	fmt.Fprintln(out)
+	failed := 0
+	for _, r := range results {
+		if r.err != nil {
+			failed++
+			fmt.Fprintf(out, "  FAIL  %-20s %s\n", r.addr, r.err)
+		} else {
+			fmt.Fprintf(out, "  ok    %-20s %s\n", r.addr, r.node)
+		}
+	}
+	if failed > 0 {
+		return fmt.Errorf("%d of %d not added; running the same command again finishes them and leaves the rest as they are", failed, len(addrs))
+	}
+	return nil
+}
+
+func (na *nodeAdd) runPlain(ctx context.Context, o onboard.Options, title string, out io.Writer) (string, error) {
 	p := &plainProgress{out: out}
-	fmt.Fprintf(out, "adding %s\n", na.opts.Addr)
-	r := onboard.New(na.opts, p.event)
-	res, err := r.Run(ctx)
+	fmt.Fprintf(out, "%s\n", title)
+	o.AskFabrics = na.rememberFabrics(nil)
+	res, err := onboard.New(o, p.event).Run(ctx)
 	if err != nil {
-		return err
+		return "", err
 	}
 	if na.noWait {
-		fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n", res.Node)
-		return nil
+		fmt.Fprintf(out, "\n%s is added. 'llmgw ctl nodes' shows it once it joins.\n\n", res.Node)
+		return res.Node, nil
 	}
 	if err := na.watch(ctx, res.Node, p.event); err != nil {
-		return err
+		return res.Node, err
 	}
-	fmt.Fprintf(out, "\n%s is in the cluster.\n", res.Node)
-	return nil
+	fmt.Fprintf(out, "\n%s is in the cluster.\n\n", res.Node)
+	return res.Node, nil
 }
 
 // adminAPI is the control plane's admin API, from its configuration when

internal/cli/quota.go

@@ -0,0 +1,234 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func newQuotaCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "quota",
+		Short: "What parts of the tenancy tree are guaranteed of the cluster, and may hold",
+		Long: "A capacity quota belongs to an organization, team or project, and counts the\n" +
+			"GPUs, CPU cores and memory held by replicas of the models it owns (see\n" +
+			"`llmgw ctl model owner`), and of those its teams and projects own.\n\n" +
+			"The guarantee is kept for it: when it holds less and a model of its needs\n" +
+			"room, replicas other scopes hold beyond their own guarantees are stopped to\n" +
+			"make it — whatever their class. What it holds within its guarantee is\n" +
+			"never taken by another scope's model. Beyond it, it borrows: free room is\n" +
+			"used, and given back when asked for.\n\n" +
+			"The limit is the most it may hold, borrowing included; a model that would\n" +
+			"take it past is not given another replica.\n\n" +
+			"Scopes are written org:Acme, team:Acme/ml or project:Acme/ml/training, or\n" +
+			"with IDs. Changes take effect within half a minute.",
+	}
+	c.AddCommand(newQuotaListCmd(client, asJSON), newQuotaSetCmd(client), newQuotaClearCmd(client))
+	return c
+}
+
+func newQuotaListCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	return &cobra.Command{
+		Use:   "list",
+		Short: "Every quota, with what its scope holds now",
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			var qs []adminapi.QuotaView
+			if err := client().get(cmd.Context(), "/api/v1/quotas", &qs); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(qs)
+			}
+			if len(qs) == 0 {
+				fmt.Fprintln(cmd.OutOrStdout(), "No capacity quotas: everything shares the cluster by class alone.")
+				return nil
+			}
+			w := tab()
+			fmt.Fprintln(w, "SCOPE\tNAME\tGPUS\tCORES\tMEMORY GB")
+			for _, q := range qs {
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", q.Scope, q.Name,
+					quotaCell(q.Holds.GPUs, q.Guarantee.GPUs, q.Limit.GPUs),
+					quotaCell(q.Holds.CPUCores, q.Guarantee.CPUCores, q.Limit.CPUCores),
+					quotaCell(q.Holds.MemoryGB, q.Guarantee.MemoryGB, q.Limit.MemoryGB))
+			}
+			if err := w.Flush(); err != nil {
+				return err
+			}
+			fmt.Fprintln(cmd.OutOrStdout(), "\nEach cell: holds / guaranteed (at most limit). - is not governed.")
+			return nil
+		},
+	}
+}
+
+// quotaCell is "6 / 8 (≤ 16)": what is held, what is guaranteed, the limit.
+func quotaCell(holds int, guarantee, limit *int) string {
+	if guarantee == nil && limit == nil {
+		return "-"
+	}
+	out := strconv.Itoa(holds) + " / "
+	if guarantee != nil {
+		out += strconv.Itoa(*guarantee)
+	} else {
+		out += "-"
+	}
+	if limit != nil {
+		out += fmt.Sprintf(" (≤ %d)", *limit)
+	}
+	return out
+}
+
+func newQuotaSetCmd(client func() *ctlClient) *cobra.Command {
+	var gpus, cores, mem, lgpus, lcores, lmem int
+	c := &cobra.Command{
+		Use:   "set SCOPE [--gpus N] [--cores N] [--memory GB] [--limit-gpus N] ...",
+		Short: "Set a scope's guarantee and limit, replacing what it had",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			f := cmd.Flags()
+			pick := func(name string, v int) *int {
+				if !f.Changed(name) {
+					return nil
+				}
+				return &v
+			}
+			req := adminapi.QuotaRequest{Scope: args[0],
+				Guarantee: store.Resources{GPUs: pick("gpus", gpus), CPUCores: pick("cores", cores), MemoryGB: pick("memory", mem)},
+				Limit:     store.Resources{GPUs: pick("limit-gpus", lgpus), CPUCores: pick("limit-cores", lcores), MemoryGB: pick("limit-memory", lmem)},
+			}
+			body, _ := json.Marshal(req)
+			var res map[string]string
+			if err := client().put(cmd.Context(), "/api/v1/quotas", body, &res); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "%s: set; it takes effect within half a minute\n", res["scope"])
+			return nil
+		},
+	}
+	f := c.Flags()
+	f.IntVar(&gpus, "gpus", 0, "GPUs guaranteed")
+	f.IntVar(&cores, "cores", 0, "CPU cores guaranteed")
+	f.IntVar(&mem, "memory", 0, "memory guaranteed, in GB")
+	f.IntVar(&lgpus, "limit-gpus", 0, "the most GPUs it may hold")
+	f.IntVar(&lcores, "limit-cores", 0, "the most CPU cores it may hold")
+	f.IntVar(&lmem, "limit-memory", 0, "the most memory it may hold, in GB")
+	return c
+}
+
+func newQuotaClearCmd(client func() *ctlClient) *cobra.Command {
+	return &cobra.Command{
+		Use:   "clear SCOPE",
+		Short: "Remove a scope's quota",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var res map[string]string
+			if err := client().delete(cmd.Context(), "/api/v1/quotas/"+url.PathEscape(args[0]), &res); err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "%s: quota removed\n", res["removed"])
+			return nil
+		},
+	}
+}
+
+func newOwnerCmd(client func() *ctlClient) *cobra.Command {
+	return &cobra.Command{
+		Use:   "owner <model> SCOPE|shared",
+		Short: "Set whose capacity quota a model counts against",
+		Long: "SCOPE is org:Acme, team:Acme/ml or project:Acme/ml/training. A shared\n" +
+			"model belongs to the cluster and counts against nobody's quota.",
+		Args: cobra.ExactArgs(2),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			owner := args[1]
+			if owner == "shared" {
+				owner = ""
+			}
+			body, _ := json.Marshal(map[string]string{"owner": owner})
+			var res map[string]string
+			if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/owner", body, &res); err != nil {
+				return err
+			}
+			if res["owner"] == "" {
+				fmt.Fprintf(cmd.OutOrStdout(), "%s is shared\n", res["model"])
+			} else {
+				fmt.Fprintf(cmd.OutOrStdout(), "%s counts against %s\n", res["model"], res["owner"])
+			}
+			return nil
+		},
+	}
+}
+
+func newTargetCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	var tps float64
+	var apply bool
+	c := &cobra.Command{
+		Use:   "target <model> --tokens-per-sec N [--apply]",
+		Short: "How many replicas a model needs for a throughput, and whether they fit",
+		Long: "Sized from what one replica was measured to serve while meeting its SLO\n" +
+			"(`llmgw ctl bench MODEL --save`), on the profile its replicas run on.\n" +
+			"--apply makes that the number it always runs, if the cluster has room.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if tps <= 0 {
+				return fmt.Errorf("--tokens-per-sec is required")
+			}
+			var v adminapi.TargetView
+			path := "/api/v1/models/" + url.PathEscape(args[0]) + "/target?tokens_per_sec=" +
+				strconv.FormatFloat(tps, 'f', -1, 64)
+			if err := client().get(cmd.Context(), path, &v); err != nil {
+				return err
+			}
+			if *asJSON {
+				return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
+			}
+			out := cmd.OutOrStdout()
+			fmt.Fprintf(out, "%s: %.0f tokens/s needs %d replicas at %.1f each (measured on %s); %d running\n",
+				v.Model, v.TokPerSec, v.Needed, v.PerReplica, v.Profile, v.Current)
+			switch {
+			case v.More == 0:
+				fmt.Fprintln(out, "It already has them.")
+			case v.Fits && len(v.Preempts) > 0:
+				fmt.Fprintf(out, "%d more; there is room once replicas of %s are stopped, which its class may do.\n",
+					v.More, strings.Join(v.Preempts, ", "))
+			case v.Fits:
+				fmt.Fprintf(out, "%d more, and there is room for them.\n", v.More)
+			default:
+				fmt.Fprintf(out, "%d more, and there is room for %d:\n", v.More, v.Room)
+				for _, b := range v.Blocked {
+					fmt.Fprintln(out, "  "+b)
+				}
+			}
+			if !apply || v.More == 0 {
+				return nil
+			}
+			if !v.Fits {
+				return fmt.Errorf("not applied: the cluster has no room for them")
+			}
+			var models []adminapi.ModelView
+			if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
+				return err
+			}
+			top := v.Needed
+			for _, m := range models {
+				if m.Name == v.Model {
+					top = max(top, m.Max)
+				}
+			}
+			body, _ := json.Marshal(adminapi.ScaleRequest{Min: v.Needed, Max: top})
+			if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(v.Model)+"/scale", body, nil); err != nil {
+				return err
+			}
+			fmt.Fprintf(out, "Applied: %s runs %d (up to %d under load).\n", v.Model, v.Needed, top)
+			return nil
+		},
+	}
+	c.Flags().Float64Var(&tps, "tokens-per-sec", 0, "the throughput to size for")
+	c.Flags().BoolVar(&apply, "apply", false, "set it as the number of replicas the model always runs")
+	return c
+}

internal/cli/release.go

@@ -0,0 +1,67 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+)
+
+func newReleaseCmd() *cobra.Command {
+	c := &cobra.Command{
+		Use:   "release",
+		Short: "Sign releases, and make the key they are signed with",
+	}
+	c.AddCommand(&cobra.Command{
+		Use:   "keygen",
+		Short: "Make a release signing key pair",
+		Long: "Prints a public key and a private key. The private key goes into the CI\n" +
+			"secret RELEASE_SIGNING_KEY and nowhere else; the public key into\n" +
+			"internal/release/trusted.pub, which every build from then on carries\n" +
+			"and checks releases against. Run once, on a machine you trust.",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			pub, priv, err := release.GenerateKey()
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(cmd.OutOrStdout(), "public:  %s\nprivate: %s\n", pub, priv)
+			fmt.Fprintln(cmd.ErrOrStderr(), "\nThe private key signs every release: put it in the CI secret "+
+				"RELEASE_SIGNING_KEY, then clear this terminal's scrollback.")
+			return nil
+		},
+	})
+	c.AddCommand(&cobra.Command{
+		Use:   "sign FILE",
+		Short: "Sign a release's checksums, writing FILE.sig",
+		Long: "Reads the private key from RELEASE_SIGNING_KEY, never from an argument,\n" +
+			"so it does not land in a shell history or a process list.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			key := os.Getenv("RELEASE_SIGNING_KEY")
+			if key == "" {
+				return errors.New("RELEASE_SIGNING_KEY is not set")
+			}
+			data, err := os.ReadFile(args[0])
+			if err != nil {
+				return err
+			}
+			sig, err := release.Sign(key, data)
+			if err != nil {
+				return err
+			}
+			if pub := release.TrustedKey(); pub != "" {
+				// Signed with a key this build would not accept is a release
+				// every upgraded machine refuses: stopped here instead.
+				if err := release.Verify(pub, data, sig); err != nil {
+					return fmt.Errorf("the signing key does not match this build's trusted key: %w", err)
+				}
+			}
+			return os.WriteFile(args[0]+".sig", []byte(sig+"\n"), 0o644) //nolint:gosec // public
+		},
+	})
+	return c
+}

internal/cli/root.go

@@ -30,6 +30,14 @@ func Root() *cobra.Command {
 		PersistentPreRun: func(_ *cobra.Command, _ []string) {
 			obs.Setup(obs.Options{Level: global.logLevel, Format: global.logFormat})
 		},
+		// Bare, in a terminal: the setup screens or the console. Otherwise
+		// the help, as before.
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			if interactive(cmd) {
+				return runHome(cmd)
+			}
+			return cmd.Help()
+		},
 	}
 
 	pf := root.PersistentFlags()
@@ -47,6 +55,9 @@ func Root() *cobra.Command {
 		newSimEngineCmd(),
 		newDoctorCmd(),
 		newNodeCmd(),
+		newSelfUpgradeCmd(),
+		newReleaseCmd(),
+		newJoinCmd(),
 	)
 	return root
 }

internal/cli/selfupgrade.go

@@ -0,0 +1,87 @@
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/release"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+// newSelfUpgradeCmd upgrades this machine's llmgw on its own, for the machines
+// `llmgw ctl upgrade` does not reach: a mirror host, above all, whose service
+// runs unprivileged with the filesystem read-only and so cannot replace its own
+// binary the way an agent does.
+func newSelfUpgradeCmd() *cobra.Command {
+	var releases, service string
+	c := &cobra.Command{
+		Use:   "upgrade VERSION [--service UNIT]",
+		Short: "Upgrade this machine's llmgw, and restart the service that runs it",
+		Long: "For a machine the control plane does not upgrade — the model mirror's host.\n" +
+			"The control plane and the nodes are upgraded with `llmgw ctl upgrade`.\n\n" +
+			"Fetches the release for this platform, checks it against the release's\n" +
+			"checksums, keeps the old binary beside the new one, and restarts --service,\n" +
+			"waiting for it to be running again. Run it with sudo.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			v := args[0]
+			out := cmd.OutOrStdout()
+			if !release.Tag.MatchString(v) {
+				return fmt.Errorf("%q is not a release version (v1.2.3 or v1.2.3-rc.4)", v)
+			}
+			if version.Version == v {
+				fmt.Fprintf(out, "this machine already runs %s\n", v)
+				return nil
+			}
+			if os.Geteuid() != 0 {
+				return errors.New("replacing this machine's binary needs root: run it with sudo")
+			}
+			self, err := os.Executable()
+			if err != nil {
+				return err
+			}
+			backup, err := release.Install(cmd.Context(), nil, releases, v, runtime.GOOS, runtime.GOARCH, self)
+			if err != nil {
+				return err
+			}
+			fmt.Fprintf(out, "%s -> %s: installed, checksum verified; the old binary is %s\n",
+				version.Version, v, backup)
+			if service == "" {
+				return nil
+			}
+			//nolint:gosec // the unit name is the operator's own flag
+			if b, err := exec.CommandContext(cmd.Context(), "systemctl", "restart", service).CombinedOutput(); err != nil {
+				return fmt.Errorf("restarting %s: %v: %s", service, err, strings.TrimSpace(string(b)))
+			}
+			// Running, and still running a few seconds on: a unit that starts
+			// and dies on a bad flag is "active" for a moment first.
+			deadline := time.Now().Add(30 * time.Second)
+			steady := 0
+			for time.Now().Before(deadline) {
+				//nolint:gosec // as above
+				b, _ := exec.CommandContext(cmd.Context(), "systemctl", "is-active", service).Output()
+				if strings.TrimSpace(string(b)) == "active" {
+					if steady++; steady >= 3 {
+						fmt.Fprintf(out, "%s restarted and running\n", service)
+						return nil
+					}
+				} else {
+					steady = 0
+				}
+				time.Sleep(2 * time.Second)
+			}
+			return fmt.Errorf("%s is not running after the upgrade; see journalctl -u %s, and %s holds the previous binary",
+				service, service, backup)
+		},
+	}
+	c.Flags().StringVar(&releases, "releases", release.DefaultBase, "where release binaries are fetched from")
+	c.Flags().StringVar(&service, "service", "", "the systemd unit to restart afterwards, e.g. llmgw-mirror")
+	return c
+}

internal/cli/server.go

@@ -5,35 +5,46 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
-	"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
 	"log/slog"
+	"maps"
 	"net"
 	"net/http"
 	"os/signal"
+	"slices"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/spf13/cobra"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 	"golang.org/x/sync/errgroup"
 
+	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/router"
 	storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/web"
 )
@@ -85,6 +96,21 @@ func newServerCmd() *cobra.Command {
 					"or give --endpoint for a router-only run")
 			}
 
+			// Before the router, so its tracer is the configured one.
+			shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
+				Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
+				Service: "llmgw-server", Version: version.Version,
+			})
+			if err != nil {
+				return err
+			}
+			defer func() {
+				// Bounded: a collector that is gone must not hold up a shutdown.
+				sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				defer cancel()
+				_ = shutdownTracing(sctx)
+			}()
+
 			rt, err := router.New(cfg, log)
 			if err != nil {
 				return err
@@ -102,6 +128,9 @@ func newServerCmd() *cobra.Command {
 			var agentHandler http.Handler
 			var agentPath string
 			var admin *control.Controller
+			// The store's cluster, when it is one; read by the metrics and
+			// by `ctl status`.
+			var storeHA *pgha.Collector
 			if controlMode {
 				cat, err := cfg.Catalog()
 				if err != nil {
@@ -129,9 +158,42 @@ func newServerCmd() *cobra.Command {
 				// On the same /metrics as the traffic figures: correlating
 				// "the cluster was full" with "requests were queueing" should
 				// not need two systems.
+				// When the newest backup was written, so its absence alerts
+				// rather than being found on the day it is needed.
+				backupDir := cfg.BackupDir
+				if backupDir == "" {
+					backupDir = DefaultBackupDir
+				}
+				if err := rt.RegisterMetrics(prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+					Name: "llmgw_backup_last_success_timestamp_seconds",
+					Help: "When the newest backup archive was written, as a Unix time; 0 when there is none.",
+				}, func() float64 {
+					if t := NewestBackup(backupDir); !t.IsZero() {
+						return float64(t.Unix())
+					}
+					return 0
+				})); err != nil {
+					return err
+				}
+				if err := rt.RegisterMetrics(tlsconf.ExpiryCollector(map[string]string{
+					"agent-server": cfg.AgentTLS.Cert, "agent-ca": cfg.AgentTLS.CA,
+				})); err != nil {
+					return err
+				}
 				if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
 					return err
 				}
+				// A store of several hosts is a replicated cluster: what state
+				// it is in is published beside everything else, read from its
+				// Patroni. One host is a single PostgreSQL, which has none.
+				if addr, err := storepkg.ResolveAddress(cfg.Auth.Store); err == nil && storepkg.IsPostgres(addr) {
+					if hosts, _, _, err := storepkg.SplitPostgres(strings.TrimSpace(addr)); err == nil && len(hosts) > 1 {
+						storeHA = pgha.NewCollector(hosts)
+						if err := rt.RegisterMetrics(storeHA); err != nil {
+							return err
+						}
+					}
+				}
 				// So a request for a model that is configured but not running
 				// starts it, rather than being told nothing serves it.
 				rt.SetActivator(ctl.Reconciler())
@@ -145,8 +207,27 @@ func newServerCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
+			// Kept past this block: serving a model from the registry needs
+			// both the admin API and the mirror, and the mirror is set up
+			// below.
+			var (
+				adminAPI   *adminapi.API
+				catalogSvc *modelsvc.Service
+			)
 			if admin != nil {
 				api := adminapi.New(admin)
+				if storeHA != nil {
+					api.SetStoreStatus(func() ([]adminapi.StoreMember, error) {
+						c, err := storeHA.Read()
+						var out []adminapi.StoreMember
+						for _, m := range c.Members {
+							out = append(out, adminapi.StoreMember{Name: m.Name, Host: m.Host, Role: m.Role, State: m.State, Lag: m.Lag})
+						}
+						return out, err
+					})
+				}
+				adminAPI = api
+				api.SetReleases(cfg.Releases)
 				// Catalog changes need somewhere to persist, so this is wired
 				// only when there is a store. Without one, the endpoints say so
 				// rather than accepting a model that vanishes on restart.
@@ -163,6 +244,7 @@ func newServerCmd() *cobra.Command {
 						admin.ReplaceCatalog(ctx, built)
 					}
 					api.SetModels(svc)
+					catalogSvc = svc
 					// And the mirror page, so a model that has just been
 					// fetched can be served without writing YAML by hand.
 					rt.SetCatalogAdmin(svc)
@@ -173,6 +255,8 @@ func newServerCmd() *cobra.Command {
 					// And what each group of the tenancy tree spent, which is
 					// the other half of a quota being reviewable.
 					api.SetUsage(st)
+					// Capacity quotas, set and read over this API.
+					api.SetQuotas(st)
 				}
 				api.Register(adminMux)
 
@@ -232,7 +316,16 @@ func newServerCmd() *cobra.Command {
 				// Browsing the registry is what turns "paste a resolve URL"
 				// into "find the model". It needs a way out to the internet
 				// from this machine, which the mirror's own fetching does not.
-				rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
+				registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
+				rt.SetRegistry(registry)
+				// One command from a registry repository to a replica that
+				// answers: `llmgw ctl model serve`.
+				if adminAPI != nil && catalogSvc != nil {
+					adminAPI.SetServe(modelserve.New(modelserve.Deps{
+						Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
+						LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
+					}))
+				}
 				log.Info("model mirror configured",
 					"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
 			}
@@ -244,7 +337,7 @@ func newServerCmd() *cobra.Command {
 				// enforces its own per-phase deadlines.
 			}
 			adminSrv := &http.Server{
-				Handler:           adminMux,
+				Handler:           router.AdminGuard(adminMux),
 				ReadHeaderTimeout: 10 * time.Second,
 			}
 
@@ -358,7 +451,8 @@ func newServerCmd() *cobra.Command {
 	f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose")
 	f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
 
-	cmd.AddCommand(newServerInitCmd())
+	cmd.AddCommand(newServerInitCmd(), newServerAddCmd())
+	cmd.AddCommand(newBackupCmd())
 	return cmd
 }
 
@@ -383,6 +477,20 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
 		ccfg.Reconciler.PortRange = [2]int{cfg.Control.PortRange[0], cfg.Control.PortRange[1]}
 	}
 
+	// Each replica's engine demands a key only the router has. The secret it
+	// is derived from lives in the store, so every control plane against it —
+	// and this one after a restart — computes the same key for a replica.
+	if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
+		secret, err := h.Store().ClusterSecret(ctx, "engine-key")
+		if err != nil {
+			return nil, nil, fmt.Errorf("reading the engine key secret: %w", err)
+		}
+		ccfg.Reconciler.EngineKey = control.EngineKeys([]byte(secret))
+	} else {
+		log.Warn("engines start without a key: there is no store to keep the secret " +
+			"they are derived from, so anyone who can reach a node's ports reaches its models")
+	}
+
 	// The service needs the inventory and the status sink, both of which the
 	// controller owns; the controller needs the service as its dispatcher. The
 	// cycle is broken by giving the controller a dispatcher that forwards to
@@ -405,14 +513,31 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
 	if h, ok := sink.(interface{ Store() *storepkg.DB }); ok {
 		if st := h.Store(); st != nil {
 			lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log)
+			if r, ok := sink.(interface{ SetLeader(func() bool) }); ok {
+				r.SetLeader(lead.IsLeader)
+			}
 			ctl.SetLeadership(lead)
+			// The tenancy tree's capacity quotas: what each part is
+			// guaranteed and may hold.
+			ctl.Reconciler().SetQuotas(control.QuotaPolicy(st, log))
+			// And what each model was measured to serve, for scaling and for
+			// choosing between profiles.
+			ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
+			// The engine builds chosen per node on the cluster page.
+			ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log))
 			log.Info("competing to be the control plane that places work",
 				"owner", lead.Owner())
 		}
 	}
 	svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
-	svc.SetEngines(cfg.WantedEngines())
-	log.Info("nodes install these engines", "engines", cfg.WantedEngines())
+	svc.SetEnginesFor(func(n inventory.Node) map[string]string {
+		return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels)
+	})
+	log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides))
+	if ch := ctl.EngineChoices(); ch != nil {
+		svc.SetEngineChoices(ch.For)
+		ch.OnChange(svc.Reprovision)
+	}
 	// A standby must refuse agents as well as refusing to place. Commands
 	// travel down the stream the agent opened, so a standby holding one is a
 	// node the leader cannot command — and the leader's placements would fail
@@ -420,6 +545,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
 	if lead != nil {
 		svc.SetLeadership(lead)
 	}
+	// The control planes agents may dial, from the store every control plane
+	// shares: told to each agent as it registers, and to all of them when
+	// `server add` records another.
+	if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
+		st := h.Store()
+		svc.SetControlPlanes(func(ctx context.Context) ([]string, error) {
+			cps, err := st.ControlPlanes(ctx)
+			urls := make([]string, len(cps))
+			for i, c := range cps {
+				urls[i] = c.URL
+			}
+			return urls, err
+		})
+		go svc.WatchControlPlanes(ctx, 30*time.Second)
+	}
 	fwd.set(svc)
 	// Every InfiniBand fabric mapped every five minutes, through one node.
 	go svc.ScanFabrics(ctx, 5*time.Minute)
@@ -493,6 +633,31 @@ func (r *dispatcherRef) Logs(ctx context.Context, nodeID, source string, lines i
 	return lr.Logs(ctx, nodeID, source, lines)
 }
 
+func (r *dispatcherRef) FabricTest(ctx context.Context, nodeID string, req *llmgwv1.FabricTestRequest) (*llmgwv1.FabricTestReply, error) {
+	d, err := r.get()
+	if err != nil {
+		return nil, err
+	}
+	ft, ok := d.(fabrictest.Asker)
+	if !ok {
+		return nil, errors.New("this dispatcher cannot run a bandwidth test")
+	}
+	return ft.FabricTest(ctx, nodeID, req)
+}
+
+// Upgrade forwards a node upgrade, as Logs does.
+func (r *dispatcherRef) Upgrade(ctx context.Context, nodeID, version, releases string) error {
+	d, err := r.get()
+	if err != nil {
+		return err
+	}
+	u, ok := d.(control.Upgrader)
+	if !ok {
+		return errors.New("this dispatcher cannot upgrade nodes")
+	}
+	return u.Upgrade(ctx, nodeID, version, releases)
+}
+
 func (r *dispatcherRef) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
 	d, err := r.get()
 	if err != nil {

internal/cli/serveradd.go

@@ -0,0 +1,459 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/version"
+)
+
+// serverAdd makes another machine a control plane of this cluster: a standby
+// that takes over when this one is lost. Run on a control plane that holds
+// the authority, as root, like `node add`.
+type serverAdd struct {
+	etcDir, pkiDir string
+	// hosts are more names to put in the new control plane's certificate.
+	hosts []string
+	// vip is a floating address, "10.0.0.160/24", to keep on whichever of the
+	// two leads; "" leaves client traffic to whatever is in front.
+	vip string
+
+	remote onboard.Remote
+	// addr is the machine's address as given; ip what the others reach it at.
+	addr, ip, name string
+	out            io.Writer
+}
+
+func newServerAddCmd() *cobra.Command {
+	sa := serverAdd{etcDir: "/etc/llmgw", pkiDir: defaultPKIDir}
+	cmd := &cobra.Command{
+		Use:   "add [user@]address",
+		Short: "Make another machine a control plane of this cluster: a standby that takes over",
+		Long: "Run on a control plane, as root. It logs in to the machine with ssh, as\n" +
+			"node add does, and gives it this llmgw, a certificate from this\n" +
+			"cluster's authority, this control plane's configuration and the store's\n" +
+			"address, lets it into the store, and starts it. It stands by while this\n" +
+			"one leads, and takes over within seconds when this one is lost; every\n" +
+			"node is told of it and fails over to it without being touched.\n\n" +
+			"The store must be one both can reach — replicated with 'server init\n" +
+			"--patroni' — not a database on this machine.\n\n" +
+			"--vip keeps a floating address on whichever control plane leads, with\n" +
+			"keepalived on both: the address clients use for the gateway.\n\n" +
+			"Run it again at any time: it changes only what is missing.",
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if os.Geteuid() != 0 {
+				return errors.New("run it as root: it reads the certificate authority and the store's address")
+			}
+			user, host, ok := strings.Cut(args[0], "@")
+			if !ok {
+				user, host = "", args[0]
+			}
+			sa.remote = &onboard.SSH{Addr: host, User: user}
+			sa.addr = host
+			sa.out = cmd.OutOrStdout()
+			return sa.run(cmd.Context())
+		},
+	}
+	f := cmd.Flags()
+	f.StringArrayVar(&sa.hosts, "host", nil, "another name for the new control plane's certificate; repeatable")
+	f.StringVar(&sa.vip, "vip", "", "a floating address for whichever control plane leads, as 10.0.0.160/24")
+	f.StringVar(&sa.pkiDir, "pki", sa.pkiDir, "directory holding the certificate authority")
+	return cmd
+}
+
+func (sa *serverAdd) done(format string, a ...any) {
+	fmt.Fprintf(sa.out, "  ok    "+format+"\n", a...)
+}
+
+func (sa *serverAdd) kept(format string, a ...any) {
+	fmt.Fprintf(sa.out, "  kept  "+format+"\n", a...)
+}
+
+func (sa *serverAdd) sh(ctx context.Context, script string, stdin []byte) (string, error) {
+	out, err := sa.remote.Run(ctx, script, stdin)
+	if err != nil {
+		return string(out), fmt.Errorf("on %s: %v: %s", sa.addr, err, strings.TrimSpace(string(out)))
+	}
+	return string(out), nil
+}
+
+// put writes a file there, through standard input, with its mode and owner.
+func (sa *serverAdd) put(ctx context.Context, path, mode, owner string, body []byte) error {
+	_, err := sa.sh(ctx, fmt.Sprintf(`set -e
+sudo install -d -m 0755 %[1]s
+sudo sh -c 'umask 077; cat > %[2]s.new'
+sudo chmod %[3]s %[2]s.new
+sudo chown %[4]s %[2]s.new
+sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode, owner), body)
+	return err
+}
+
+func (sa *serverAdd) run(ctx context.Context) error {
+	steps := []struct {
+		name string
+		fn   func(context.Context) error
+	}{
+		{"connect", sa.connect},
+		{"binary", sa.binary},
+		{"directories", sa.dirs},
+		{"certificate", sa.certificate},
+		{"configuration", sa.configuration},
+		{"store", sa.storeAccess},
+		{"service", sa.service},
+	}
+	if sa.vip != "" {
+		steps = append(steps, struct {
+			name string
+			fn   func(context.Context) error
+		}{"floating address", sa.floating})
+	}
+	for _, s := range steps {
+		if err := s.fn(ctx); err != nil {
+			return fmt.Errorf("%s: %w", s.name, err)
+		}
+	}
+	fmt.Fprintf(sa.out, "\n%s is a control plane of this cluster, standing by while this one leads.\n", sa.name)
+	return nil
+}
+
+func (sa *serverAdd) connect(ctx context.Context) error {
+	out, err := sa.sh(ctx, "hostname; sudo -n true && echo sudo-ok", nil)
+	if err != nil {
+		return fmt.Errorf("%w — it needs ssh and sudo without a password", err)
+	}
+	lines := strings.Split(strings.TrimSpace(out), "\n")
+	sa.name = strings.TrimSpace(lines[0])
+	if sa.ip == "" {
+		if sa.ip, err = ipv4Of(sa.addr); err != nil {
+			return err
+		}
+	}
+	sa.done("%s (%s)", sa.name, sa.ip)
+	return nil
+}
+
+// binary gives it this very llmgw, so the two control planes never differ.
+func (sa *serverAdd) binary(ctx context.Context) error {
+	have, _ := sa.sh(ctx, "llmgw --version 2>/dev/null || true", nil)
+	if strings.TrimSpace(have) == "llmgw version "+version.Version {
+		sa.kept("llmgw %s", version.Version)
+		return nil
+	}
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	b, err := os.ReadFile(exe)
+	if err != nil {
+		return err
+	}
+	if _, err := sa.sh(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
+		return err
+	}
+	sa.done("llmgw %s, copied from this machine", version.Version)
+	return nil
+}
+
+func (sa *serverAdd) dirs(ctx context.Context) error {
+	_, err := sa.sh(ctx, `set -e
+id llmgw >/dev/null 2>&1 || sudo useradd --system --home /var/lib/llmgw --shell /usr/sbin/nologin llmgw
+sudo install -d -m 0755 /etc/llmgw
+sudo install -d -o llmgw -g llmgw -m 0700 /etc/llmgw/secrets
+sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil)
+	if err == nil {
+		sa.done("user llmgw, /etc/llmgw and /var/lib/llmgw")
+	}
+	return err
+}
+
+// certificate issues it a server certificate from this cluster's authority,
+// for its name and addresses, unless it holds one that already names them.
+func (sa *serverAdd) certificate(ctx context.Context) error {
+	ca, err := loadCA(sa.pkiDir)
+	if err != nil {
+		return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err)
+	}
+	ips, _ := sa.sh(ctx, "hostname -I", nil)
+	want := []string{sa.name, sa.ip}
+	for _, ip := range strings.Fields(ips) {
+		if net.ParseIP(ip).To4() != nil && !slices.Contains(want, ip) {
+			want = append(want, ip)
+		}
+	}
+	want = append(want, sa.hosts...)
+
+	if have, _ := sa.sh(ctx, "sudo cat /etc/llmgw/server.pem 2>/dev/null || true", nil); have != "" {
+		if blk, _ := pem.Decode([]byte(have)); blk != nil {
+			if cert, err := x509.ParseCertificate(blk.Bytes); err == nil && cert.CheckSignatureFrom(ca.Cert) == nil {
+				names := strings.Split(sanNames(cert), ", ")
+				missing := false
+				for _, w := range want {
+					missing = missing || !slices.Contains(names, w)
+				}
+				if !missing {
+					sa.kept("certificate for %s", strings.Join(names, ", "))
+					return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM)
+				}
+			}
+		}
+	}
+	leaf, err := ca.IssueServer(sa.name, want)
+	if err != nil {
+		return err
+	}
+	if err := sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM); err != nil {
+		return err
+	}
+	if err := sa.put(ctx, "/etc/llmgw/server.pem", "0644", "root:root", leaf.CertPEM); err != nil {
+		return err
+	}
+	if err := sa.put(ctx, "/etc/llmgw/server-key.pem", "0640", "root:llmgw", leaf.KeyPEM); err != nil {
+		return err
+	}
+	sa.done("certificate for %s, from this cluster's authority", strings.Join(want, ", "))
+	return nil
+}
+
+// configuration copies this control plane's, and the secrets beside it: the
+// two must agree on everything, the store above all.
+func (sa *serverAdd) configuration(ctx context.Context) error {
+	if have, _ := sa.sh(ctx, "sudo test -f /etc/llmgw/server.yaml && echo yes || true", nil); strings.TrimSpace(have) == "yes" {
+		sa.kept("/etc/llmgw/server.yaml (never overwritten)")
+	} else {
+		b, err := os.ReadFile(filepath.Join(sa.etcDir, "server.yaml"))
+		if err != nil {
+			return err
+		}
+		if err := sa.put(ctx, "/etc/llmgw/server.yaml", "0644", "root:root", b); err != nil {
+			return err
+		}
+		sa.done("/etc/llmgw/server.yaml, as this control plane's")
+	}
+	secrets, err := os.ReadDir(filepath.Join(sa.etcDir, "secrets"))
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	for _, e := range secrets {
+		if e.IsDir() || strings.Contains(e.Name(), ".pre-") || strings.HasSuffix(e.Name(), ".new") {
+			continue
+		}
+		path := "/etc/llmgw/secrets/" + e.Name()
+		mine, err := os.ReadFile(filepath.Join(sa.etcDir, "secrets", e.Name()))
+		if err != nil {
+			return err
+		}
+		theirs, _ := sa.sh(ctx, "sudo cat "+path+" 2>/dev/null || true", nil)
+		switch {
+		case theirs == string(mine):
+			sa.kept("%s", path)
+		case theirs != "":
+			// Two control planes on two stores are two clusters.
+			return fmt.Errorf("%s there differs from this one's; remove it there if that machine is to join this cluster", path)
+		default:
+			if err := sa.put(ctx, path, "0600", "llmgw:llmgw", mine); err != nil {
+				return err
+			}
+			sa.done("%s", path)
+		}
+	}
+	return nil
+}
+
+// storeAccess lets it into the store, and records both control planes for
+// the nodes to be told.
+func (sa *serverAdd) storeAccess(ctx context.Context) error {
+	cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml"))
+	if err != nil {
+		return err
+	}
+	addr, err := store.ResolveAddress(cfg.Auth.Store)
+	if err != nil {
+		return err
+	}
+	addr = strings.TrimSpace(addr)
+	if !store.IsPostgres(addr) {
+		return errors.New("the store is SQLite on this machine; a second control plane needs one both reach — run 'llmgw server init --patroni' first")
+	}
+	hosts, _, _, err := store.SplitPostgres(addr)
+	if err != nil {
+		return err
+	}
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil && ip.IsLoopback() || h == "localhost" {
+			return fmt.Errorf("the store is at %s, on this machine only; replicate it first with 'llmgw server init --patroni'", h)
+		}
+	}
+	switch {
+	case slices.Contains(hosts, sa.ip):
+		sa.kept("store: %s is one of its members", sa.ip)
+	case len(hosts) > 1:
+		changed, err := pgha.AllowClient(ctx, hosts, sa.ip)
+		if err != nil {
+			return err
+		}
+		if changed {
+			sa.done("store: %s let in, through Patroni", sa.ip)
+		} else {
+			sa.kept("store: %s already let in", sa.ip)
+		}
+	default:
+		sa.kept("store at %s: make sure it lets %s in", hosts[0], sa.ip)
+	}
+
+	st, err := store.Open(ctx, addr)
+	if err != nil {
+		return err
+	}
+	defer st.Close()
+	_, port, _ := net.SplitHostPort(cfg.AgentListen)
+	if port == "" {
+		port = "7443"
+	}
+	self, err := sourceIPToward(sa.ip)
+	if err != nil {
+		return err
+	}
+	me, _ := os.Hostname()
+	for _, cp := range []store.ControlPlane{
+		{URL: "https://" + net.JoinHostPort(self, port), Name: me},
+		{URL: "https://" + net.JoinHostPort(sa.ip, port), Name: sa.name, AddedAt: time.Now().Add(time.Second)},
+	} {
+		if err := st.PutControlPlane(ctx, cp); err != nil {
+			return err
+		}
+	}
+	sa.done("nodes will be told of both control planes within half a minute")
+	return nil
+}
+
+// service installs the unit and starts it, then waits for it to answer.
+func (sa *serverAdd) service(ctx context.Context) error {
+	if err := sa.put(ctx, "/etc/systemd/system/llmgw-server.service", "0644", "root:root", []byte(serverUnit)); err != nil {
+		return err
+	}
+	if _, err := sa.sh(ctx, "sudo systemctl daemon-reload && sudo systemctl enable llmgw-server >/dev/null 2>&1 && sudo systemctl restart llmgw-server", nil); err != nil {
+		return err
+	}
+	deadline := time.Now().Add(90 * time.Second)
+	for {
+		code, _ := sa.sh(ctx, "curl -s -o /dev/null -m 2 -w '%{http_code}' http://127.0.0.1:8080/leader || true", nil)
+		switch strings.TrimSpace(code) {
+		case "200":
+			sa.done("llmgw-server running, and leading")
+			return nil
+		case "503":
+			sa.done("llmgw-server running, standing by")
+			return nil
+		}
+		if time.Now().After(deadline) {
+			logs, _ := sa.sh(ctx, "sudo journalctl -u llmgw-server -n 15 --no-pager", nil)
+			return fmt.Errorf("it did not answer within 90s:\n%s", logs)
+		}
+		time.Sleep(2 * time.Second)
+	}
+}
+
+// floating keeps an address on whichever of the two leads, with keepalived
+// on both, following /leader.
+func (sa *serverAdd) floating(ctx context.Context) error {
+	vip, nw, err := net.ParseCIDR(sa.vip)
+	if err != nil || vip.To4() == nil {
+		return fmt.Errorf("--vip %q is not an IPv4 address with its prefix, like 10.0.0.160/24", sa.vip)
+	}
+	prefix, _ := nw.Mask.Size()
+	pass, err := vrrpPassword(sa.etcDir)
+	if err != nil {
+		return err
+	}
+	here := pgha.Local{}
+	for _, m := range []struct {
+		r        onboard.Remote
+		name     string
+		priority int
+	}{{here, "this machine", 110}, {sa.remote, sa.name, 100}} {
+		out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
+command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
+ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
+		if err != nil {
+			return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
+		}
+		iface := strings.TrimSpace(string(out))
+		conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
+		if out, err := m.r.Run(ctx, `set -e
+sudo install -d -m 0755 /etc/keepalived
+sudo sh -c 'umask 077; cat > /etc/keepalived/keepalived.conf'
+sudo systemctl enable keepalived >/dev/null 2>&1
+sudo systemctl restart keepalived`, []byte(conf)); err != nil {
+			return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
+		}
+	}
+	sa.done("%s follows whichever control plane leads (keepalived on both)", sa.vip)
+	return nil
+}
+
+// vrrpPassword is the shared secret of the two keepalived, made once and
+// kept with the other secrets, so running this again does not split them.
+func vrrpPassword(etcDir string) (string, error) {
+	p := filepath.Join(etcDir, "secrets", "vrrp")
+	if b, err := os.ReadFile(p); err == nil && len(bytes.TrimSpace(b)) > 0 {
+		return string(bytes.TrimSpace(b)), nil
+	}
+	b := make([]byte, 4)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	pass := hex.EncodeToString(b) // keepalived reads eight characters
+	if err := os.WriteFile(p, []byte(pass+"\n"), 0o600); err != nil {
+		return "", err
+	}
+	return pass, nil
+}
+
+func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
+	return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
+vrrp_script llmgw_leader {
+  script "/usr/bin/curl -fsS -o /dev/null -m 2 http://127.0.0.1:8080/leader"
+  interval 2
+  fall 2
+  rise 2
+}
+
+vrrp_instance llmgw {
+  state BACKUP
+  interface %s
+  virtual_router_id 51
+  priority %d
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass %s
+  }
+  virtual_ipaddress {
+    %s/%d
+  }
+  track_script {
+    llmgw_leader
+  }
+}
+`, iface, priority, pass, vip, prefix)
+}

internal/cli/serverinit.go

@@ -23,6 +23,7 @@ import (
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/account"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
 )
@@ -69,10 +70,19 @@ type serverInit struct {
 	// postgres keeps the store in a PostgreSQL on this machine, installed if
 	// need be, moving whatever SQLite already holds.
 	postgres bool
+	// patroni are user@host of the machines to keep the store on under
+	// Patroni, replicated; this machine is the third etcd vote for two.
+	patroni []string
+	// patroniNodes are the same machines already connected to, which the
+	// setup screen does with the password it was given; when set, patroni
+	// only names them.
+	patroniNodes []pgha.Node
 	// noMirror, noNode and noModel turn off the parts of a first install that
 	// make it serve something: a mirror, this machine as a node, and a small
 	// model on it.
 	noMirror, noNode, noModel bool
+	// mirrorURL is a mirror on another machine to use instead of running one.
+	mirrorURL string
 	// pgSetup prepares PostgreSQL and returns the store address; a seam for
 	// tests. Nil does it for real.
 	pgSetup func(context.Context) (string, error)
@@ -84,14 +94,20 @@ type serverInit struct {
 	changed  bool
 }
 
-func newServerInitCmd() *cobra.Command {
-	in := serverInit{
+// newDefaultServerInit is init as it sets up a real machine.
+func newDefaultServerInit() serverInit {
+	return serverInit{
 		etcDir:   "/etc/llmgw",
 		stateDir: "/var/lib/llmgw",
 		pkiDir:   defaultPKIDir,
 		unitPath: "/etc/systemd/system/llmgw-server.service",
 		user:     "llmgw",
 	}
+}
+
+func newServerInitCmd() *cobra.Command {
+	in := newDefaultServerInit()
+	var plain bool
 	cmd := &cobra.Command{
 		Use:   "init",
 		Short: "Set this machine up as a control plane and start it",
@@ -108,6 +124,12 @@ func newServerInitCmd() *cobra.Command {
 			"installed if there is none, with its own role and a generated password\n" +
 			"kept in /etc/llmgw/secrets. An existing SQLite store is copied into it,\n" +
 			"so accounts and keys survive, and server.yaml is pointed at it.\n\n" +
+			"With --patroni, given two or more machines, the store is PostgreSQL\n" +
+			"replicated across them under Patroni, which promotes a standby when the\n" +
+			"primary is lost. etcd holds its leader lock on those machines — and on\n" +
+			"this one, when they are two, as the third vote. Installed over ssh with\n" +
+			"passwordless sudo, as 'llmgw node add' does; the store already in use,\n" +
+			"SQLite or PostgreSQL, is moved in. Run again, it changes nothing.\n\n" +
 			"Each step is skipped when its result already exists, so it is safe to run\n" +
 			"again: to finish an install that stopped, or with another --host to\n" +
 			"reissue the certificate. It never overwrites server.yaml or the authority.\n\n" +
@@ -122,19 +144,28 @@ func newServerInitCmd() *cobra.Command {
 				return errors.New("init creates a user, writes /etc and starts a service: run it as root")
 			}
 			in.out = cmd.OutOrStdout()
+			if !plain && isTerminal(in.out) {
+				return in.runInitTUI(cmd.Context(), in.out)
+			}
 			return in.run(cmd.Context())
 		},
 	}
 	f := cmd.Flags()
+	f.BoolVar(&plain, "plain", false, "plain output, even in a terminal")
 	f.StringArrayVar(&in.hosts, "host", nil,
 		"an address agents dial besides the ones found on this machine; repeatable")
 	f.StringVar(&in.admin, "admin", "", "email of the first administrator; a password is generated and printed once")
 	f.StringVar(&in.pkiDir, "pki", in.pkiDir, "directory holding the certificate authority")
 	f.BoolVar(&in.noMirror, "no-mirror", false, "do not run a model mirror on this machine")
+	f.StringVar(&in.mirrorURL, "mirror", "",
+		"use the mirror on another machine, set up there by 'llmgw mirror init': its management API, http://host:8082")
 	f.BoolVar(&in.noNode, "no-node", false, "do not make this machine a node (and serve no model on it)")
 	f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
 	f.BoolVar(&in.postgres, "postgres", false,
 		"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
+	f.StringArrayVar(&in.patroni, "patroni", nil,
+		"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
+			"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
 	return cmd
 }
 
@@ -163,7 +194,12 @@ func (in *serverInit) run(ctx context.Context) error {
 		hasSystemd = false
 	}
 	// Before the service starts, because the mirror goes into server.yaml.
-	if hasSystemd && !in.noMirror {
+	switch {
+	case in.mirrorURL != "":
+		if err := in.useMirror(ctx); err != nil {
+			return fmt.Errorf("mirror: %w", err)
+		}
+	case hasSystemd && !in.noMirror:
 		if err := in.ensureMirror(ctx); err != nil {
 			return fmt.Errorf("mirror: %w", err)
 		}
@@ -175,14 +211,14 @@ func (in *serverInit) run(ctx context.Context) error {
 		if _, err := in.ensureNode(ctx); err != nil {
 			return fmt.Errorf("node: %w", err)
 		}
-		switch {
-		case in.noModel:
-		case in.noMirror:
-			fmt.Fprintln(in.out, "  skip  no starter model without the mirror")
-		default:
-			if err := in.ensureStarter(ctx); err != nil {
-				return fmt.Errorf("starter model: %w", err)
-			}
+	}
+	switch {
+	case !hasSystemd || in.noModel:
+	case in.noMirror && in.mirrorURL == "":
+		fmt.Fprintln(in.out, "  skip  no starter model without the mirror")
+	default:
+		if err := in.ensureStarter(ctx); err != nil {
+			return fmt.Errorf("starter model: %w", err)
 		}
 	}
 	in.nextSteps()

internal/cli/serverinit_cluster.go

@@ -111,15 +111,25 @@ func (in *serverInit) writeUnit(path, body string) (bool, error) {
 
 func (in *serverInit) mirrorUnit() string {
 	base := "http://" + net.JoinHostPort(in.reachableAddr(), "8081")
+	return mirrorUnitFor(mirrorDir, mirrorAPI, base, nil,
+		"'llmgw server init'. Bytes on :8081 for every node; the\n# management API on loopback only, for the control plane beside it.")
+}
+
+// mirrorUnitFor is a mirror's systemd unit. allow is who, besides the
+// machine itself, may use the management API.
+func mirrorUnitFor(dir, apiListen, base string, allow []string, about string) string {
+	exec := "/usr/local/bin/llmgw mirror serve --dir " + dir + " --listen :8081 --api-listen " + apiListen + " --base " + base
+	for _, a := range allow {
+		exec += " --api-allow " + a
+	}
 	return `[Unit]
 Description=llmgw model mirror
 After=network-online.target
 Wants=network-online.target
 
 [Service]
-# Written by 'llmgw server init'. Bytes on :8081 for every node; the
-# management API on loopback only, for the control plane beside it.
-ExecStart=/usr/local/bin/llmgw mirror serve --dir ` + mirrorDir + ` --listen :8081 --api-listen ` + mirrorAPI + ` --base ` + base + `
+# Written by ` + about + `
+ExecStart=` + exec + `
 User=llmgw
 Group=llmgw
 Restart=on-failure
@@ -128,13 +138,53 @@ NoNewPrivileges=true
 ProtectSystem=strict
 ProtectHome=true
 PrivateTmp=true
-ReadWritePaths=` + mirrorDir + `
+ReadWritePaths=` + dir + `
 
 [Install]
 WantedBy=multi-user.target
 `
 }
 
+// mirrorAPIURL is the mirror's management API: one on its own machine, or
+// the one beside this control plane.
+func (in *serverInit) mirrorAPIURL() string {
+	if in.mirrorURL != "" {
+		return in.mirrorURL
+	}
+	return "http://" + mirrorAPI
+}
+
+// useMirror points the control plane at a mirror on another machine, set up
+// there by 'llmgw mirror init'.
+func (in *serverInit) useMirror(ctx context.Context) error {
+	m, err := mirror.NewClient(in.mirrorURL, tlsconf.Config{}, 5*time.Second)
+	if err != nil {
+		return err
+	}
+	if err := m.Health(ctx); err != nil {
+		return fmt.Errorf("the mirror at %s does not answer: %v. On that machine: sudo llmgw mirror init --control-plane <this machine's address>", in.mirrorURL, err)
+	}
+	cfg, err := config.Load(in.configPath())
+	if err != nil {
+		return err
+	}
+	switch cfg.Mirror.API {
+	case in.mirrorURL:
+		in.kept("mirror at %s", in.mirrorURL)
+		return nil
+	case "":
+	default:
+		in.kept("mirror at %s, as server.yaml says; change it there to use %s", cfg.Mirror.API, in.mirrorURL)
+		return nil
+	}
+	if err := appendMirrorBlockFor(in.configPath(), in.mirrorURL,
+		"# The model mirror, on its own machine, added by 'llmgw server init'."); err != nil {
+		return err
+	}
+	in.done("server.yaml points at the mirror at %s", in.mirrorURL)
+	return nil
+}
+
 // ensureMirror runs a mirror on this machine and points the control plane at
 // it. Its management API is on loopback, which is why it needs no certificate:
 // nothing but this machine can reach it.
@@ -174,7 +224,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
 	}
 
 	if cfg.Mirror.API == "" {
-		if err := appendMirrorBlock(in.configPath()); err != nil {
+		if err := appendMirrorBlockFor(in.configPath(), "http://"+mirrorAPI,
+			"# The model mirror on this machine, added by 'llmgw server init'. Its\n"+
+				"# management API is on loopback, so it needs no certificate."); err != nil {
 			return err
 		}
 		in.done("server.yaml points at the mirror")
@@ -189,9 +241,9 @@ func (in *serverInit) ensureMirror(ctx context.Context) error {
 	})
 }
 
-// appendMirrorBlock adds the mirror to server.yaml without touching the rest,
-// keeping the file as it was beside it.
-func appendMirrorBlock(path string) error {
+// appendMirrorBlockFor adds the mirror to server.yaml without touching the
+// rest, keeping the file as it was beside it.
+func appendMirrorBlockFor(path, api, comment string) error {
 	b, err := os.ReadFile(path)
 	if err != nil {
 		return err
@@ -200,9 +252,7 @@ func appendMirrorBlock(path string) error {
 	if err := os.WriteFile(backup, b, 0o640); err != nil {
 		return err
 	}
-	block := "\n# The model mirror on this machine, added by 'llmgw server init'. Its\n" +
-		"# management API is on loopback, so it needs no certificate.\n" +
-		"mirror:\n  api: http://" + mirrorAPI + "\n"
+	block := "\n" + comment + "\nmirror:\n  api: " + api + "\n"
 	info, err := os.Stat(path)
 	if err != nil {
 		return err
@@ -356,8 +406,11 @@ func hasNVIDIADevice() bool {
 
 // starterModel is the catalog entry: a GPU profile first when there is a card,
 // and a CPU profile always, both pinned to this machine.
-func starterModel(source string, gpuClasses []string) catalog.Model {
-	pin := map[string]string{localLabelKey: localLabelValue}
+func starterModel(source string, gpuClasses []string, pinned bool) catalog.Model {
+	var pin map[string]string
+	if pinned {
+		pin = map[string]string{localLabelKey: localLabelValue}
+	}
 	var profiles []catalog.Profile
 	for _, class := range gpuClasses {
 		profiles = append(profiles, catalog.Profile{
@@ -401,7 +454,7 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
 	if !ok {
 		return fmt.Errorf("cannot turn %s into a download", starterSource)
 	}
-	m, err := mirror.NewClient("http://"+mirrorAPI, tlsconf.Config{}, 15*time.Second)
+	m, err := mirror.NewClient(in.mirrorAPIURL(), tlsconf.Config{}, 15*time.Second)
 	if err != nil {
 		return err
 	}
@@ -444,14 +497,24 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
 	}
 	in.done("%s in the mirror", starterName)
 
-	classes, _ := gpuClasses(ctx)
-	body, err := json.Marshal(starterModel(st.URL, classes))
+	// Served here when this machine is a node, pinned so it does not wander
+	// to the first GPU node that joins; otherwise on whichever node joins.
+	pinned := !in.noNode
+	var classes []string
+	if pinned {
+		classes, _ = gpuClasses(ctx)
+	}
+	body, err := json.Marshal(starterModel(st.URL, classes, pinned))
 	if err != nil {
 		return err
 	}
 	if err := client.post(ctx, "/api/v1/models", body, &map[string]string{}); err != nil {
 		return fmt.Errorf("adding %s to the catalog: %w", starterName, err)
 	}
+	if !pinned {
+		in.done("%s in the catalog; it starts on the first node that joins", starterName)
+		return nil
+	}
 	where := "on CPU"
 	if len(classes) > 0 {
 		where = "on " + strings.Join(classes, ", ") + ", or CPU if the card is busy"

internal/cli/serverinit_cluster_test.go

@@ -1,6 +1,8 @@
 package cli
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"strings"
@@ -21,7 +23,7 @@ func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) {
 }
 
 func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) {
-	m := starterModel("http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"})
+	m := starterModel("http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true)
 	c, err := catalog.New([]catalog.Model{m})
 	if err != nil {
 		t.Fatalf("the starter is not a valid catalog entry: %v", err)
@@ -46,7 +48,7 @@ func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) {
 		t.Error("the starter should be running, not waiting to be asked for")
 	}
 
-	if m := starterModel("http://x/q.gguf", nil); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" {
+	if m := starterModel("http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" {
 		t.Errorf("without a GPU only the CPU profile: %+v", m.Profiles)
 	}
 }
@@ -56,7 +58,7 @@ func TestTheMirrorBlockIsAddedAndReads(t *testing.T) {
 	if err := in.run(t.Context()); err != nil {
 		t.Fatalf("%v\n%s", err, out)
 	}
-	if err := appendMirrorBlock(in.configPath()); err != nil {
+	if err := appendMirrorBlockFor(in.configPath(), "http://"+mirrorAPI, "# the mirror"); err != nil {
 		t.Fatal(err)
 	}
 	cfg, err := config.Load(in.configPath())
@@ -87,3 +89,35 @@ func TestTheMirrorUnitAdvertisesAnAddressNodesCanReach(t *testing.T) {
 		t.Errorf("the management API is not on loopback: %s", u)
 	}
 }
+
+func TestWithNoNodeHereTheStarterGoesToTheFirstNodeThatJoins(t *testing.T) {
+	m := starterModel("http://x/q.gguf", nil, false)
+	for _, p := range m.Profiles {
+		if len(p.NodeSelector) > 0 {
+			t.Errorf("profile %s is pinned to a machine that is not a node: %v", p.Name, p.NodeSelector)
+		}
+	}
+}
+
+func TestTheMirrorAPIOnlyAnswersItsControlPlane(t *testing.T) {
+	allowed, err := parseAllow([]string{"10.0.0.140", "10.1.0.0/24"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	h := onlyFrom(allowed, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {}))
+	for addr, want := range map[string]int{
+		"10.0.0.140:5000": 200, "10.1.0.7:5000": 200, "127.0.0.1:5000": 200, "[::1]:5000": 200,
+		"10.0.0.141:5000": 403, "192.168.1.9:5000": 403,
+	} {
+		r := httptest.NewRequest("GET", "/api/v1/health", nil)
+		r.RemoteAddr = addr
+		w := httptest.NewRecorder()
+		h.ServeHTTP(w, r)
+		if w.Code != want {
+			t.Errorf("%s: %d, want %d", addr, w.Code, want)
+		}
+	}
+	if _, err := parseAllow([]string{"not-an-address"}); err == nil {
+		t.Error("nonsense was accepted")
+	}
+}

internal/cli/serverinit_patroni.go

@@ -0,0 +1,182 @@
+package cli
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+// ensurePatroni puts the store in PostgreSQL under Patroni on the machines
+// given with --patroni, and moves whatever store server.yaml names into it:
+// SQLite, or a PostgreSQL on one machine. This machine is the third etcd vote
+// when they are two.
+func (in *serverInit) ensurePatroni(ctx context.Context) (err error) {
+	setup := pgha.Setup{Emit: func(step, detail string, changed bool) {
+		if changed {
+			in.done("%s: %s", step, detail)
+		} else {
+			in.kept("%s: %s", step, detail)
+		}
+	}}
+	setup.DB = in.patroniNodes
+	for _, t := range in.patroni {
+		if len(in.patroniNodes) > 0 {
+			break
+		}
+		user, host, ok := strings.Cut(t, "@")
+		if !ok {
+			user, host = "", t
+		}
+		remote := &onboard.SSH{Addr: host, User: user}
+		out, err := remote.Run(ctx, "hostname && sudo -n true", nil)
+		if err != nil {
+			return fmt.Errorf("%s: cannot log in and use sudo without a password: %v: %s",
+				t, err, strings.TrimSpace(string(out)))
+		}
+		ip, err := ipv4Of(host)
+		if err != nil {
+			return err
+		}
+		setup.DB = append(setup.DB, pgha.Node{
+			Member: pgha.Member{Name: strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0]), IP: ip},
+			Remote: remote,
+		})
+	}
+	self, err := sourceIPToward(setup.DB[0].IP)
+	if err != nil {
+		return err
+	}
+	name, _ := os.Hostname()
+	setup.Witness = &pgha.Node{Member: pgha.Member{Name: name, IP: self}, Remote: pgha.Local{}}
+	setup.Clients = []string{self}
+
+	dsn, err := setup.Run(ctx)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(in.configPath())
+	if err != nil {
+		return err
+	}
+	current := cfg.Auth.Store
+	if current == "" {
+		current = in.storeAddress()
+	}
+	resolved, _ := store.ResolveAddress(current)
+	if strings.TrimSpace(resolved) == dsn {
+		in.kept("store in the Patroni cluster (%s)", in.dsnPath())
+		return nil
+	}
+
+	// Still while it moves, and started again on the old store if the move
+	// fails: a gateway down over a database that did not work out is worse
+	// than one left where it was.
+	stopped, err := in.stopService(ctx)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err != nil && stopped {
+			_ = exec.Command("systemctl", "start", "llmgw-server").Run()
+			fmt.Fprintf(in.out, "  ..    llmgw-server started again on its old store\n")
+		}
+	}()
+	if err := in.moveInto(ctx, current, dsn); err != nil {
+		return err
+	}
+
+	// The old address is kept beside the new one: the way back, if the
+	// cluster is not what was wanted.
+	if b, err := os.ReadFile(in.dsnPath()); err == nil && strings.TrimSpace(string(b)) != dsn {
+		if err := os.WriteFile(in.dsnPath()+".pre-patroni", b, 0o600); err != nil {
+			return err
+		}
+		_ = in.chown(in.dsnPath()+".pre-patroni", true)
+	}
+	if err := in.writeDSN(dsn); err != nil {
+		return err
+	}
+	if current != "@"+in.dsnPath() {
+		if err := setAuthStore(in.configPath(), "@"+in.dsnPath()); err != nil {
+			return err
+		}
+	}
+	// Started again by the service step that follows, on the new address.
+	in.done("server.yaml now keeps the store in the Patroni cluster")
+	return nil
+}
+
+// moveInto copies a store into the cluster, unless the cluster already holds
+// one — from a run that copied and stopped before changing the address.
+func (in *serverInit) moveInto(ctx context.Context, from, to string) error {
+	src, err := store.Open(ctx, from)
+	if err != nil {
+		// Nothing there yet: a first install, with nothing to move.
+		in.kept("no store at %s to move", from)
+		return nil
+	}
+	defer src.Close()
+	dst, err := store.Open(ctx, to)
+	if err != nil {
+		return fmt.Errorf("opening the cluster: %w", err)
+	}
+	defer dst.Close()
+	empty, err := dst.Empty(ctx)
+	if err != nil {
+		return err
+	}
+	if !empty {
+		in.kept("the cluster already holds a store; %s is left as it is", from)
+		return nil
+	}
+	res, err := src.CopyTo(ctx, dst)
+	if err != nil {
+		return fmt.Errorf("copying the store into the cluster: %w", err)
+	}
+	var rows int64
+	for _, r := range res {
+		rows += r.Rows
+	}
+	in.done("copied %d rows into the cluster (the old store is kept as it was)", rows)
+	return nil
+}
+
+// sourceIPToward is this machine's address on the route to another: the one
+// the other machine sees it connect from, which is what etcd advertises and
+// what the database lets in. Not the first --host, which names the gateway
+// for its clients and may not resolve yet, or resolve to a floating address.
+func sourceIPToward(ip string) (string, error) {
+	c, err := net.Dial("udp", net.JoinHostPort(ip, "5432"))
+	if err != nil {
+		return "", fmt.Errorf("no route from this machine to %s: %w", ip, err)
+	}
+	defer c.Close()
+	return c.LocalAddr().(*net.UDPAddr).IP.String(), nil
+}
+
+// ipv4Of is the address a host name or address stands for, as the other
+// machines will be told to reach it.
+func ipv4Of(host string) (string, error) {
+	if ip := net.ParseIP(host); ip != nil {
+		return ip.String(), nil
+	}
+	ips, err := net.LookupIP(host)
+	if err != nil {
+		return "", fmt.Errorf("%s does not resolve: %w", host, err)
+	}
+	for _, ip := range ips {
+		if v4 := ip.To4(); v4 != nil {
+			return v4.String(), nil
+		}
+	}
+	return "", fmt.Errorf("%s has no IPv4 address", host)
+}